From 401084bab3184c9b6948d96cf2f3632df500f0fc Mon Sep 17 00:00:00 2001 From: Alex Maldonado Date: Thu, 12 Nov 2026 22:23:54 -0500 Subject: [PATCH] Initial commit --- .editorconfig | 100 + .gitattributes | 2 + .gitea/workflows/nightly.yml | 55 + .gitea/workflows/test.yml | 85 + .gitignore | 14 + .gitmodules | 4 + AGENTS.md | 61 + CHANGELOG.md | 68 + CITATION.cff | 21 + CODE_OF_CONDUCT.md | 84 + CONTRIBUTING.md | 35 + LICENSE.md | 167 + NOTICE.md | 16 + README.md | 152 + SECURITY.md | 16 + THIRD_PARTY.md | 50 + docs/asm-coverage.md | 51 + docs/ci.md | 48 + docs/engines.md | 121 + docs/exercises.md | 38 + docs/library.md | 43 + docs/limits.md | 84 + docs/mojo-corpus.md | 179 + docs/sync-coverage.md | 117 + docs/trust.md | 79 + library | 1 + package-lock.json | 33 + package.json | 20 + pixi.lock | 1395 +++++++ pixi.toml | 78 + scripts/asm-census.mjs | 70 + scripts/ci-fuzz.mjs | 66 + scripts/cli.mjs | 109 + scripts/dafny.py | 114 + scripts/engines-diff.mjs | 74 + scripts/engines-export.mjs | 45 + scripts/engines-files.mjs | 27 + scripts/library.mjs | 718 ++++ scripts/library_godbolt.py | 71 + scripts/mojo/corpus.py | 149 + scripts/mojo/fuzz-programs.mjs | 667 ++++ scripts/mojo/fuzz.mjs | 596 +++ scripts/mojo/listings.mjs | 151 + scripts/mojo/local.mjs | 345 ++ scripts/mojo/mojocorpus/__init__.py | 3 + scripts/mojo/mojocorpus/common.py | 271 ++ scripts/mojo/mojocorpus/emit_ts.py | 179 + scripts/mojo/mojocorpus/fetch.py | 252 ++ scripts/mojo/mojocorpus/godbolt.py | 127 + scripts/mojo/mojocorpus/harvest.py | 130 + scripts/mojo/mojocorpus/kde.py | 124 + scripts/mojo/mojocorpus/notices.py | 200 + .../mojo/mojocorpus/processors/__init__.py | 60 + .../mojo/mojocorpus/processors/diagnostics.py | 117 + scripts/mojo/mojocorpus/processors/editor.py | 66 + scripts/mojo/mojocorpus/processors/manual.py | 88 + .../mojo/mojocorpus/processors/messages.py | 150 + scripts/mojo/mojocorpus/processors/quest.py | 98 + scripts/mojo/mojocorpus/processors/release.py | 122 + scripts/mojo/mojocorpus/processors/skill.py | 60 + .../mojo/mojocorpus/processors/stdindex.py | 151 + .../mojo/mojocorpus/processors/stdtypes.py | 188 + scripts/mojo/mojocorpus/processors/syntax.py | 92 + scripts/mojo/mojocorpus/processors/tokens.py | 118 + scripts/mojo/mojocorpus/verify.py | 378 ++ scripts/mojo/new-program.mjs | 61 + scripts/mojo/tests/test_pipeline.py | 439 +++ scripts/new-exercise.mjs | 77 + scripts/package.py | 49 + scripts/play.mjs | 187 + scripts/prove-fuzz.mjs | 467 +++ scripts/tests/test_dafny.py | 49 + src/common/fuzz.ts | 482 +++ src/common/machine.ts | 35 + src/common/problems.ts | 47 + src/common/reference.ts | 478 +++ src/common/runview.ts | 244 ++ src/common/types.ts | 153 + src/common/workspace.ts | 316 ++ src/curriculum/advanced.ts | 307 ++ src/curriculum/concepts.ts | 58 + src/curriculum/core.ts | 388 ++ src/curriculum/foundations.ts | 311 ++ src/curriculum/index.ts | 75 + src/curriculum/science.ts | 87 + src/curriculum/types.ts | 103 + src/engines/asm86/cache.js | 250 ++ src/engines/asm86/dom.js | 35 + src/engines/asm86/encode.js | 590 +++ src/engines/asm86/hwviews.js | 1347 +++++++ src/engines/asm86/interp.js | 2591 +++++++++++++ src/engines/asm86/machine.js | 332 ++ src/engines/asm86/ooo.js | 370 ++ src/engines/asm86/paging.js | 130 + src/engines/asm86/pipeline.js | 169 + src/engines/asm86/sandbox.js | 420 +++ src/engines/asm86/specs-hw.js | 670 ++++ src/engines/asm86/specs.js | 844 +++++ src/engines/asm86/uops.js | 401 ++ src/engines/asm86/views.js | 1312 +++++++ src/engines/pseudo/analyze.js | 738 ++++ src/engines/pseudo/ast.ts | 472 +++ src/engines/pseudo/check.ts | 1507 ++++++++ src/engines/pseudo/coach.js | 421 +++ src/engines/pseudo/dom.js | 35 + src/engines/pseudo/editor.js | 147 + src/engines/pseudo/english.js | 3297 +++++++++++++++++ src/engines/pseudo/exercise.js | 393 ++ src/engines/pseudo/highlight.js | 49 + src/engines/pseudo/interp.ts | 2016 ++++++++++ src/engines/pseudo/lab.js | 226 ++ src/engines/pseudo/lang.ts | 593 +++ src/engines/pseudo/logic.ts | 706 ++++ src/engines/pseudo/numeric.ts | 198 + src/engines/pseudo/parse.ts | 1518 ++++++++ src/engines/pseudo/plan.js | 501 +++ src/engines/pseudo/planner.js | 143 + src/engines/pseudo/sharpen.js | 793 ++++ src/engines/pseudo/solver.ts | 628 ++++ src/engines/pseudo/specs.js | 854 +++++ src/engines/pseudo/study.js | 83 + src/engines/pseudo/testing.js | 424 +++ src/engines/pseudo/translate.js | 541 +++ src/engines/pseudo/vague.js | 366 ++ src/engines/pseudo/vcgen.ts | 1398 +++++++ src/engines/pseudo/views-analysis.js | 803 ++++ src/engines/pseudo/views.js | 653 ++++ src/engines/shared/dom.js | 66 + src/engines/shared/share.js | 19 + src/engines/shared/text.js | 27 + src/engines/shared/viz.js | 714 ++++ src/exercises/_helpers.ts | 23 + src/exercises/_template.ts | 48 + src/exercises/asm-branchless.ts | 18 + src/exercises/asm-cmov.ts | 18 + src/exercises/asm-first-instructions.ts | 18 + src/exercises/asm-pack-bits.ts | 17 + src/exercises/asm-register-widths.ts | 17 + src/exercises/asm-two-pointers.ts | 17 + src/exercises/asm-vector.ts | 18 + src/exercises/best-position.ts | 19 + src/exercises/binary-search.ts | 23 + src/exercises/count-base-proved.ts | 24 + src/exercises/count-base.ts | 30 + src/exercises/demo.ts | 21 + src/exercises/edit-distance.ts | 19 + src/exercises/first-unknown.ts | 25 + src/exercises/five-bugs.ts | 18 + src/exercises/gc-fraction.ts | 20 + src/exercises/hamming.ts | 19 + src/exercises/index.ts | 63 + src/exercises/kmers.ts | 18 + src/exercises/match-off-by-one.ts | 18 + src/exercises/match-proved.ts | 24 + src/exercises/match-reset.ts | 19 + src/exercises/match.ts | 25 + src/exercises/revcomp.ts | 25 + src/exercises/tally.ts | 18 + src/exercises/translate-codons.ts | 19 + src/exercises/types.ts | 54 + src/library/generated.ts | 1803 +++++++++ src/library/registry.ts | 42 + src/library/types.ts | 72 + src/mojo/att.ts | 392 ++ src/mojo/check.ts | 61 + src/mojo/editing.ts | 25 + src/mojo/flow.ts | 470 +++ src/mojo/flowir.ts | 123 + src/mojo/generated/editor.ts | 26 + src/mojo/generated/listings.ts | 24 + src/mojo/generated/pin.ts | 15 + src/mojo/generated/programs.ts | 116 + src/mojo/generated/stdlib.ts | 652 ++++ src/mojo/generated/tokens.ts | 347 ++ src/mojo/generated/types.ts | 1772 +++++++++ src/mojo/interp.ts | 2231 +++++++++++ src/mojo/lex.ts | 128 + src/mojo/lifetimes.ts | 260 ++ src/mojo/parse.ts | 901 +++++ src/mojo/practices.ts | 679 ++++ src/mojo/properties.ts | 153 + src/mojo/reference.ts | 36 + src/mojo/resolve.ts | 233 ++ src/mojo/tokenize.ts | 165 + src/mojo/typecheck.ts | 524 +++ src/mojo/values.ts | 246 ++ src/mojo/warnings.ts | 73 + src/services/asm.ts | 188 + src/services/compile-backend.ts | 39 + src/services/godbolt.ts | 283 ++ src/services/index.ts | 20 + src/services/lingo.ts | 104 + src/services/matho.ts | 1241 +++++++ src/services/mojo.ts | 34 + src/services/pseudo-study.ts | 300 ++ src/services/pseudo.ts | 220 ++ src/services/speco.ts | 430 +++ src/sync/compare.ts | 83 + src/sync/header.ts | 8 + src/sync/hub.ts | 387 ++ src/sync/pseudo-to-lingo.ts | 545 +++ src/sync/pseudo-to-matho.ts | 625 ++++ src/sync/pseudo-to-mojo.ts | 1552 ++++++++ src/sync/types.ts | 38 + tests/asm-coverage.test.mjs | 69 + tests/asm-float.test.mjs | 256 ++ tests/asm-setup.test.mjs | 59 + tests/boundary.test.mjs | 59 + tests/build-args.test.mjs | 172 + tests/chain.test.mjs | 37 + tests/corpus.test.mjs | 40 + tests/curriculum.test.mjs | 119 + tests/dafny.test.mjs | 34 + tests/documents.test.mjs | 240 ++ tests/engine-solver.test.mjs | 219 ++ tests/exercises.test.mjs | 39 + tests/fuzz.test.mjs | 91 + tests/hardware.test.mjs | 87 + tests/helpers/sync-programs.mjs | 196 + tests/library-harness.test.mjs | 121 + tests/library.test.mjs | 102 + tests/listings.test.mjs | 101 + tests/matho.test.mjs | 186 + tests/mojo-check.test.mjs | 236 ++ tests/mojo-robustness.test.mjs | 182 + tests/mojo-run.test.mjs | 262 ++ tests/mojo.test.mjs | 291 ++ tests/numeric.test.mjs | 382 ++ tests/practices.test.mjs | 147 + tests/prover-limits.test.mjs | 255 ++ tests/prover-soundness.test.mjs | 187 + tests/pseudo-declare-init.test.mjs | 62 + tests/pseudo-explain.test.mjs | 53 + tests/pseudo-robustness.test.mjs | 319 ++ tests/reader-limits.test.mjs | 155 + tests/reference.test.mjs | 240 ++ tests/runview.test.mjs | 101 + tests/services.test.mjs | 102 + tests/snapshots/practices.json | 68 + tests/snapshots/sync-mojo/best-position.mojo | 19 + tests/snapshots/sync-mojo/binary-search.mojo | 25 + .../sync-mojo/count-base-proved.mojo | 18 + tests/snapshots/sync-mojo/count-base.mojo | 21 + tests/snapshots/sync-mojo/edit-distance.mojo | 31 + tests/snapshots/sync-mojo/first-unknown.mojo | 16 + tests/snapshots/sync-mojo/gc-fraction.mojo | 16 + tests/snapshots/sync-mojo/hamming.mojo | 17 + tests/snapshots/sync-mojo/kmers.mojo | 21 + .../snapshots/sync-mojo/match-off-by-one.mojo | 27 + tests/snapshots/sync-mojo/match-proved.mojo | 28 + tests/snapshots/sync-mojo/match-reset.mojo | 23 + tests/snapshots/sync-mojo/match.mojo | 27 + tests/snapshots/sync-mojo/revcomp.mojo | 25 + tests/snapshots/sync-mojo/tally.mojo | 17 + .../snapshots/sync-mojo/translate-codons.mojo | 34 + tests/spdx.test.mjs | 38 + tests/sync-coverage.test.mjs | 106 + tests/sync-hub.test.mjs | 244 ++ tests/sync-lingo.test.mjs | 114 + tests/sync-matho.test.mjs | 236 ++ tests/sync-mojo.test.mjs | 526 +++ tests/sync-speco.test.mjs | 152 + tests/tasks.test.mjs | 84 + tests/third-party.test.mjs | 86 + tsconfig.json | 47 + 265 files changed, 69224 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitattributes create mode 100644 .gitea/workflows/nightly.yml create mode 100644 .gitea/workflows/test.yml create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 AGENTS.md create mode 100644 CHANGELOG.md create mode 100644 CITATION.cff create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE.md create mode 100644 NOTICE.md create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 THIRD_PARTY.md create mode 100644 docs/asm-coverage.md create mode 100644 docs/ci.md create mode 100644 docs/engines.md create mode 100644 docs/exercises.md create mode 100644 docs/library.md create mode 100644 docs/limits.md create mode 100644 docs/mojo-corpus.md create mode 100644 docs/sync-coverage.md create mode 100644 docs/trust.md create mode 160000 library create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 pixi.lock create mode 100644 pixi.toml create mode 100644 scripts/asm-census.mjs create mode 100644 scripts/ci-fuzz.mjs create mode 100644 scripts/cli.mjs create mode 100644 scripts/dafny.py create mode 100644 scripts/engines-diff.mjs create mode 100644 scripts/engines-export.mjs create mode 100644 scripts/engines-files.mjs create mode 100644 scripts/library.mjs create mode 100644 scripts/library_godbolt.py create mode 100644 scripts/mojo/corpus.py create mode 100644 scripts/mojo/fuzz-programs.mjs create mode 100644 scripts/mojo/fuzz.mjs create mode 100644 scripts/mojo/listings.mjs create mode 100644 scripts/mojo/local.mjs create mode 100644 scripts/mojo/mojocorpus/__init__.py create mode 100644 scripts/mojo/mojocorpus/common.py create mode 100644 scripts/mojo/mojocorpus/emit_ts.py create mode 100644 scripts/mojo/mojocorpus/fetch.py create mode 100644 scripts/mojo/mojocorpus/godbolt.py create mode 100644 scripts/mojo/mojocorpus/harvest.py create mode 100644 scripts/mojo/mojocorpus/kde.py create mode 100644 scripts/mojo/mojocorpus/notices.py create mode 100644 scripts/mojo/mojocorpus/processors/__init__.py create mode 100644 scripts/mojo/mojocorpus/processors/diagnostics.py create mode 100644 scripts/mojo/mojocorpus/processors/editor.py create mode 100644 scripts/mojo/mojocorpus/processors/manual.py create mode 100644 scripts/mojo/mojocorpus/processors/messages.py create mode 100644 scripts/mojo/mojocorpus/processors/quest.py create mode 100644 scripts/mojo/mojocorpus/processors/release.py create mode 100644 scripts/mojo/mojocorpus/processors/skill.py create mode 100644 scripts/mojo/mojocorpus/processors/stdindex.py create mode 100644 scripts/mojo/mojocorpus/processors/stdtypes.py create mode 100644 scripts/mojo/mojocorpus/processors/syntax.py create mode 100644 scripts/mojo/mojocorpus/processors/tokens.py create mode 100644 scripts/mojo/mojocorpus/verify.py create mode 100644 scripts/mojo/new-program.mjs create mode 100644 scripts/mojo/tests/test_pipeline.py create mode 100644 scripts/new-exercise.mjs create mode 100644 scripts/package.py create mode 100644 scripts/play.mjs create mode 100644 scripts/prove-fuzz.mjs create mode 100644 scripts/tests/test_dafny.py create mode 100644 src/common/fuzz.ts create mode 100644 src/common/machine.ts create mode 100644 src/common/problems.ts create mode 100644 src/common/reference.ts create mode 100644 src/common/runview.ts create mode 100644 src/common/types.ts create mode 100644 src/common/workspace.ts create mode 100644 src/curriculum/advanced.ts create mode 100644 src/curriculum/concepts.ts create mode 100644 src/curriculum/core.ts create mode 100644 src/curriculum/foundations.ts create mode 100644 src/curriculum/index.ts create mode 100644 src/curriculum/science.ts create mode 100644 src/curriculum/types.ts create mode 100644 src/engines/asm86/cache.js create mode 100644 src/engines/asm86/dom.js create mode 100644 src/engines/asm86/encode.js create mode 100644 src/engines/asm86/hwviews.js create mode 100644 src/engines/asm86/interp.js create mode 100644 src/engines/asm86/machine.js create mode 100644 src/engines/asm86/ooo.js create mode 100644 src/engines/asm86/paging.js create mode 100644 src/engines/asm86/pipeline.js create mode 100644 src/engines/asm86/sandbox.js create mode 100644 src/engines/asm86/specs-hw.js create mode 100644 src/engines/asm86/specs.js create mode 100644 src/engines/asm86/uops.js create mode 100644 src/engines/asm86/views.js create mode 100644 src/engines/pseudo/analyze.js create mode 100644 src/engines/pseudo/ast.ts create mode 100644 src/engines/pseudo/check.ts create mode 100644 src/engines/pseudo/coach.js create mode 100644 src/engines/pseudo/dom.js create mode 100644 src/engines/pseudo/editor.js create mode 100644 src/engines/pseudo/english.js create mode 100644 src/engines/pseudo/exercise.js create mode 100644 src/engines/pseudo/highlight.js create mode 100644 src/engines/pseudo/interp.ts create mode 100644 src/engines/pseudo/lab.js create mode 100644 src/engines/pseudo/lang.ts create mode 100644 src/engines/pseudo/logic.ts create mode 100644 src/engines/pseudo/numeric.ts create mode 100644 src/engines/pseudo/parse.ts create mode 100644 src/engines/pseudo/plan.js create mode 100644 src/engines/pseudo/planner.js create mode 100644 src/engines/pseudo/sharpen.js create mode 100644 src/engines/pseudo/solver.ts create mode 100644 src/engines/pseudo/specs.js create mode 100644 src/engines/pseudo/study.js create mode 100644 src/engines/pseudo/testing.js create mode 100644 src/engines/pseudo/translate.js create mode 100644 src/engines/pseudo/vague.js create mode 100644 src/engines/pseudo/vcgen.ts create mode 100644 src/engines/pseudo/views-analysis.js create mode 100644 src/engines/pseudo/views.js create mode 100644 src/engines/shared/dom.js create mode 100644 src/engines/shared/share.js create mode 100644 src/engines/shared/text.js create mode 100644 src/engines/shared/viz.js create mode 100644 src/exercises/_helpers.ts create mode 100644 src/exercises/_template.ts create mode 100644 src/exercises/asm-branchless.ts create mode 100644 src/exercises/asm-cmov.ts create mode 100644 src/exercises/asm-first-instructions.ts create mode 100644 src/exercises/asm-pack-bits.ts create mode 100644 src/exercises/asm-register-widths.ts create mode 100644 src/exercises/asm-two-pointers.ts create mode 100644 src/exercises/asm-vector.ts create mode 100644 src/exercises/best-position.ts create mode 100644 src/exercises/binary-search.ts create mode 100644 src/exercises/count-base-proved.ts create mode 100644 src/exercises/count-base.ts create mode 100644 src/exercises/demo.ts create mode 100644 src/exercises/edit-distance.ts create mode 100644 src/exercises/first-unknown.ts create mode 100644 src/exercises/five-bugs.ts create mode 100644 src/exercises/gc-fraction.ts create mode 100644 src/exercises/hamming.ts create mode 100644 src/exercises/index.ts create mode 100644 src/exercises/kmers.ts create mode 100644 src/exercises/match-off-by-one.ts create mode 100644 src/exercises/match-proved.ts create mode 100644 src/exercises/match-reset.ts create mode 100644 src/exercises/match.ts create mode 100644 src/exercises/revcomp.ts create mode 100644 src/exercises/tally.ts create mode 100644 src/exercises/translate-codons.ts create mode 100644 src/exercises/types.ts create mode 100644 src/library/generated.ts create mode 100644 src/library/registry.ts create mode 100644 src/library/types.ts create mode 100644 src/mojo/att.ts create mode 100644 src/mojo/check.ts create mode 100644 src/mojo/editing.ts create mode 100644 src/mojo/flow.ts create mode 100644 src/mojo/flowir.ts create mode 100644 src/mojo/generated/editor.ts create mode 100644 src/mojo/generated/listings.ts create mode 100644 src/mojo/generated/pin.ts create mode 100644 src/mojo/generated/programs.ts create mode 100644 src/mojo/generated/stdlib.ts create mode 100644 src/mojo/generated/tokens.ts create mode 100644 src/mojo/generated/types.ts create mode 100644 src/mojo/interp.ts create mode 100644 src/mojo/lex.ts create mode 100644 src/mojo/lifetimes.ts create mode 100644 src/mojo/parse.ts create mode 100644 src/mojo/practices.ts create mode 100644 src/mojo/properties.ts create mode 100644 src/mojo/reference.ts create mode 100644 src/mojo/resolve.ts create mode 100644 src/mojo/tokenize.ts create mode 100644 src/mojo/typecheck.ts create mode 100644 src/mojo/values.ts create mode 100644 src/mojo/warnings.ts create mode 100644 src/services/asm.ts create mode 100644 src/services/compile-backend.ts create mode 100644 src/services/godbolt.ts create mode 100644 src/services/index.ts create mode 100644 src/services/lingo.ts create mode 100644 src/services/matho.ts create mode 100644 src/services/mojo.ts create mode 100644 src/services/pseudo-study.ts create mode 100644 src/services/pseudo.ts create mode 100644 src/services/speco.ts create mode 100644 src/sync/compare.ts create mode 100644 src/sync/header.ts create mode 100644 src/sync/hub.ts create mode 100644 src/sync/pseudo-to-lingo.ts create mode 100644 src/sync/pseudo-to-matho.ts create mode 100644 src/sync/pseudo-to-mojo.ts create mode 100644 src/sync/types.ts create mode 100644 tests/asm-coverage.test.mjs create mode 100644 tests/asm-float.test.mjs create mode 100644 tests/asm-setup.test.mjs create mode 100644 tests/boundary.test.mjs create mode 100644 tests/build-args.test.mjs create mode 100644 tests/chain.test.mjs create mode 100644 tests/corpus.test.mjs create mode 100644 tests/curriculum.test.mjs create mode 100644 tests/dafny.test.mjs create mode 100644 tests/documents.test.mjs create mode 100644 tests/engine-solver.test.mjs create mode 100644 tests/exercises.test.mjs create mode 100644 tests/fuzz.test.mjs create mode 100644 tests/hardware.test.mjs create mode 100644 tests/helpers/sync-programs.mjs create mode 100644 tests/library-harness.test.mjs create mode 100644 tests/library.test.mjs create mode 100644 tests/listings.test.mjs create mode 100644 tests/matho.test.mjs create mode 100644 tests/mojo-check.test.mjs create mode 100644 tests/mojo-robustness.test.mjs create mode 100644 tests/mojo-run.test.mjs create mode 100644 tests/mojo.test.mjs create mode 100644 tests/numeric.test.mjs create mode 100644 tests/practices.test.mjs create mode 100644 tests/prover-limits.test.mjs create mode 100644 tests/prover-soundness.test.mjs create mode 100644 tests/pseudo-declare-init.test.mjs create mode 100644 tests/pseudo-explain.test.mjs create mode 100644 tests/pseudo-robustness.test.mjs create mode 100644 tests/reader-limits.test.mjs create mode 100644 tests/reference.test.mjs create mode 100644 tests/runview.test.mjs create mode 100644 tests/services.test.mjs create mode 100644 tests/snapshots/practices.json create mode 100644 tests/snapshots/sync-mojo/best-position.mojo create mode 100644 tests/snapshots/sync-mojo/binary-search.mojo create mode 100644 tests/snapshots/sync-mojo/count-base-proved.mojo create mode 100644 tests/snapshots/sync-mojo/count-base.mojo create mode 100644 tests/snapshots/sync-mojo/edit-distance.mojo create mode 100644 tests/snapshots/sync-mojo/first-unknown.mojo create mode 100644 tests/snapshots/sync-mojo/gc-fraction.mojo create mode 100644 tests/snapshots/sync-mojo/hamming.mojo create mode 100644 tests/snapshots/sync-mojo/kmers.mojo create mode 100644 tests/snapshots/sync-mojo/match-off-by-one.mojo create mode 100644 tests/snapshots/sync-mojo/match-proved.mojo create mode 100644 tests/snapshots/sync-mojo/match-reset.mojo create mode 100644 tests/snapshots/sync-mojo/match.mojo create mode 100644 tests/snapshots/sync-mojo/revcomp.mojo create mode 100644 tests/snapshots/sync-mojo/tally.mojo create mode 100644 tests/snapshots/sync-mojo/translate-codons.mojo create mode 100644 tests/spdx.test.mjs create mode 100644 tests/sync-coverage.test.mjs create mode 100644 tests/sync-hub.test.mjs create mode 100644 tests/sync-lingo.test.mjs create mode 100644 tests/sync-matho.test.mjs create mode 100644 tests/sync-mojo.test.mjs create mode 100644 tests/sync-speco.test.mjs create mode 100644 tests/tasks.test.mjs create mode 100644 tests/third-party.test.mjs create mode 100644 tsconfig.json diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..0a4cb20 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,100 @@ +# EditorConfig: https://editorconfig.org +# In VS Code, install the "EditorConfig for VS Code" extension (EditorConfig.EditorConfig). + +root = true + +# Defaults for every file +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +indent_size = 4 +insert_final_newline = true +trim_trailing_whitespace = true + +# Python +[*.{py,pyi}] +indent_size = 4 +max_line_length = 88 + +# Mojo (`mojo format` is Black-style, 88 columns; the fire emoji extension is also valid) +[*.{mojo,🔥}] +indent_size = 4 +max_line_length = 88 + +# Rust (rustfmt defaults: 4 spaces, 100 columns) +[*.rs] +indent_size = 4 +max_line_length = 100 + +# Cargo manifests and rustfmt config +[{Cargo.toml,Cargo.lock,rustfmt.toml,.rustfmt.toml,clippy.toml}] +indent_size = 4 + +# Mojo project manifest +[{mojoproject.toml,pixi.toml,pixi.lock}] +indent_size = 4 + +# YAML +[*.{yml,yaml}] +indent_size = 2 + +# JSON and friends +[*.{json,jsonc,json5}] +indent_size = 2 + +# Web +[*.{js,jsx,mjs,cjs,ts,tsx,css,scss,less,html,vue,svelte}] +indent_size = 2 + +# Config formats +[*.{toml,ini,cfg}] +indent_size = 4 + +[*.xml] +indent_size = 2 + +# Markdown: trailing spaces are significant (hard line breaks) +[*.{md,markdown}] +indent_size = 2 +trim_trailing_whitespace = false + +# typst +[*.{typ}] +indent_size = 4 +trim_trailing_whitespace = false + +# Shell scripts +[*.{sh,bash,zsh}] +indent_size = 2 + +# Makefiles require tabs +[{Makefile,makefile,*.mk}] +indent_style = tab + +# Go uses tabs +[*.go] +indent_style = tab + +# Windows-only scripts need CRLF +[*.{bat,cmd,ps1}] +end_of_line = crlf + +# Dockerfiles +[{Dockerfile,Dockerfile.*,*.dockerfile}] +indent_size = 4 + +# Terraform +[*.{tf,tfvars,hcl}] +indent_size = 2 + +# Don't touch these +[LICENSE*] +insert_final_newline = false + +[*.{diff,patch}] +trim_trailing_whitespace = false + +[*.min.{js,css}] +insert_final_newline = unset +trim_trailing_whitespace = unset diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..997504b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# SCM syntax highlighting & preventing 3-way merges +pixi.lock merge=binary linguist-language=YAML linguist-generated=true -diff diff --git a/.gitea/workflows/nightly.yml b/.gitea/workflows/nightly.yml new file mode 100644 index 0000000..4ec3cf2 --- /dev/null +++ b/.gitea/workflows/nightly.yml @@ -0,0 +1,55 @@ +# pplus-core's scheduled run: the fuzzers for longer than the gate can afford, on seeds that move with the date, and +# mojo-status-strict (`pixi run nightly`). A failure opens nothing and sends nothing: the run goes red where anyone +# looking at the repository's Actions page sees it. docs/ci.md says how to reproduce a seed and what to do with it. +# +# Gitea Actions (host mode) as test.yml; on GitHub Actions the same file runs with `runs-on: ubuntu-latest`. +name: Nightly +on: + schedule: + - cron: "17 6 * * *" # 06:17 UTC, before the working day in America + workflow_dispatch: + +env: + CI: "true" + PIXI_VERSION: "0.83.0" + +jobs: + fuzz: + runs-on: quarto-deploy + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + # pixi: the host's when it is the pinned version, else the pinned release, fetched once into a cache directory + # the runner's user owns (no sudo), checked by SHA-256 + - name: pixi, pinned + shell: bash + run: | + set -euo pipefail + if command -v pixi >/dev/null && [ "$(pixi --version | cut -d' ' -f2)" = "$PIXI_VERSION" ]; then + echo "pixi $PIXI_VERSION is on the host: $(command -v pixi)"; exit 0 + fi + dir="${PPLUS_CI_CACHE:-$HOME/.cache/pplus-ci}/pixi-$PIXI_VERSION" + if [ ! -x "$dir/pixi" ]; then + case "$(uname -s)-$(uname -m)" in + Linux-x86_64) asset=pixi-x86_64-unknown-linux-musl.tar.gz; sum=064f371e692aaa3400ac581c89da2843c91437f20c2d9155395a5d0037f33551 ;; + Linux-aarch64) asset=pixi-aarch64-unknown-linux-musl.tar.gz; sum=d08403d8f048626ec93ba7b7c56e3b747a5f3a95c4e6e0c815ceb264f5f0b9ce ;; + *) echo "no pinned pixi for $(uname -sm)" >&2; exit 1 ;; + esac + tmp="$(mktemp -d)" + curl -fsSL --retry 3 -o "$tmp/$asset" "https://github.com/prefix-dev/pixi/releases/download/v$PIXI_VERSION/$asset" + echo "$sum $tmp/$asset" | sha256sum -c - + mkdir -p "$dir" && tar -xzf "$tmp/$asset" -C "$dir" pixi && rm -rf "$tmp" + fi + echo "$dir" >> "$GITHUB_PATH" + "$dir/pixi" --version + + - name: The pinned environment, exactly as pixi.lock has it + run: pixi install --locked + + # mojo-status-strict, then mojo-fuzz in both modes (four seeds, 100 programs each) and prove-fuzz (four seeds, 3,000 each); + # every seed is printed with the command that reproduces it + - name: pixi run nightly + run: pixi run nightly diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml new file mode 100644 index 0000000..970fe34 --- /dev/null +++ b/.gitea/workflows/test.yml @@ -0,0 +1,85 @@ +# pplus-core's gate on every push and pull request: `pixi run ci`, the same task a developer runs, in the +# environment pixi.lock pins (Node, Python, the Mojo compiler), so CI and a developer cannot drift. docs/ci.md says +# what it runs, how long it takes, and what to do when it fails. +# +# Gitea Actions, on a runner in host mode (no container: steps run on the host's shell; pplus-web's docs/deploy.md +# has the runner's setup). The same file runs on GitHub Actions with one change: `runs-on: ubuntu-latest`. +name: Test +on: + push: + pull_request: + workflow_dispatch: + +concurrency: + group: test-${{ github.ref }} + cancel-in-progress: true + +env: + # set by Gitea and GitHub Actions already; written here so the tests' "fail, not skip, under CI" never depends on it + CI: "true" + # the pixi that reads pixi.lock; when the host has another version, this one is fetched into the cache below + PIXI_VERSION: "0.83.0" + +jobs: + gate: + # a pull request runs only from a branch of this repository: on a host runner a fork's workflow would run its + # own code on the droplet (docs/deploy.md in pplus-web). Drop this line for a runner that is a throwaway VM. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: quarto-deploy + timeout-minutes: 60 + steps: + # library/ is pplus-library, a submodule pinned by commit: the programs, the corpus, the routines + - uses: actions/checkout@v4 + with: + submodules: recursive + + # pixi: the host's when it is the pinned version, else the pinned release, fetched once into a cache directory + # the runner's user owns (no sudo), checked by SHA-256 + - name: pixi, pinned + shell: bash + run: | + set -euo pipefail + if command -v pixi >/dev/null && [ "$(pixi --version | cut -d' ' -f2)" = "$PIXI_VERSION" ]; then + echo "pixi $PIXI_VERSION is on the host: $(command -v pixi)"; exit 0 + fi + dir="${PPLUS_CI_CACHE:-$HOME/.cache/pplus-ci}/pixi-$PIXI_VERSION" + if [ ! -x "$dir/pixi" ]; then + case "$(uname -s)-$(uname -m)" in + Linux-x86_64) asset=pixi-x86_64-unknown-linux-musl.tar.gz; sum=064f371e692aaa3400ac581c89da2843c91437f20c2d9155395a5d0037f33551 ;; + Linux-aarch64) asset=pixi-aarch64-unknown-linux-musl.tar.gz; sum=d08403d8f048626ec93ba7b7c56e3b747a5f3a95c4e6e0c815ceb264f5f0b9ce ;; + *) echo "no pinned pixi for $(uname -sm)" >&2; exit 1 ;; + esac + tmp="$(mktemp -d)" + curl -fsSL --retry 3 -o "$tmp/$asset" "https://github.com/prefix-dev/pixi/releases/download/v$PIXI_VERSION/$asset" + echo "$sum $tmp/$asset" | sha256sum -c - + mkdir -p "$dir" && tar -xzf "$tmp/$asset" -C "$dir" pixi && rm -rf "$tmp" + fi + echo "$dir" >> "$GITHUB_PATH" + "$dir/pixi" --version + + - name: The pinned environment, exactly as pixi.lock has it + run: pixi install --locked + + # .tools/ holds the pinned Dafny and mojo-local's answers (keyed by compiler, arguments and source); on a host + # runner it lives in the cache directory, so neither is fetched or built again on every push + - name: The pinned Dafny + shell: bash + run: | + set -euo pipefail + cache="${PPLUS_CI_CACHE:-$HOME/.cache/pplus-ci}/core-tools" + mkdir -p "$cache" + if [ -d .tools ] && [ ! -L .tools ]; then echo ".tools is a directory in this checkout already: using it"; else ln -sfn "$cache" .tools; fi + pixi run dafny-fetch + + # every test, mojo-status-strict, library-check, mojo-local and a short mojo-fuzz and prove-fuzz (pixi.toml) + - name: pixi run ci + run: pixi run ci + + # the gate rewrites what it records (the library's status.json and STATUS.md, the corpus's STATUS.md); a + # difference means a change was committed without what it regenerates + - name: Nothing the gate regenerates differs from what is committed + shell: bash + run: | + set -euo pipefail + git diff --stat --exit-code + git -C library diff --stat --exit-code diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..be9a15a --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +node_modules/ +dist/ +.DS_Store +patches/ +package/ +# the pinned Dafny (pixi run dafny-fetch, scripts/dafny.py) and mojo-local's answers; in CI a link to a cache +.tools +# pixi environments +.pixi/* +!.pixi/config.toml + +# Python +__pycache__/ +*.pyc diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..c6bf1fa --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "library"] + path = library + url = https://git.scient.ing/pplus/pplus-library.git + branch = main diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5aa1f7e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,61 @@ +# AGENTS.md + +Instructions for coding agents (and people) changing pplus-core. Programming+ takes a procedure from an idea to Mojo to the machine. Core decides what a program means in each of its notations: lingo (steps in plain English), matho (equations written in LaTeX), pseudo (pseudocode precise enough to run, prove and translate), speco (a specification to prove against) and Mojo (Modular's programming language, pinned to version 1.2.0). Read this first, then the [README](README.md) for what core is and where things are. When this file and a task's instructions disagree, the task wins; say so in your summary. + +## Build, test, and the gate + +```bash +pixi run test # build, then every test (node --test against dist/, the pipelines' Python tests included) +pixi run ci # what CI runs: every test, no Mojo program we wrote failing by godbolt's stored answers, library-check, mojo-local, a short fuzz (docs/ci.md) +``` + +Without pixi (a sandbox with only Node and Python): + +```bash +npx tsc -p tsconfig.json && node --test --test-concurrency=8 tests/*.test.mjs +python3 scripts/mojo/corpus.py status --strict +``` + +**Finish every change with the gate green.** The baseline passes completely; there is no known failure to work around. If a test fails, fix the cause or stop and report it. Do not skip, loosen, or delete a test to get green. + +The full suite takes under a minute (25 s on the machine this was last measured on). It runs eight files at a time: Node's default is one fewer than the processor count, and on a 24-core machine the 41 files at once (each 60 to 490 MB at its peak) passed a 3 GB memory cap on some runs and were OOM-killed on others; eight at a time peaked at 1.9 GB in the same 22 s. While iterating, build and run one file: `npx tsc -p tsconfig.json && node --test tests/sync-hub.test.mjs`. + +## Hard rules + +1. **Core has no DOM.** What a program means is decided without a browser. A library module never touches `document`, `window`, `localStorage` or `navigator`, never imports one of a course's page widgets, and never imports from outside `src/`; `tests/boundary.test.mjs` holds it to that. The widgets (listed in that test) live beside the engines only so a course's site and pplus-web take them from one place. +2. **Do not edit generated files by hand.** `src/mojo/generated/*` and `library/syntax/mojo.xml` come from the corpus pipeline (`pixi run mojo-emit`, `mojo-status`, `play`). `src/library/generated.ts` comes from `library/` (`pixi run library-emit`), and the library's `status.json`, `STATUS.md` and `golden/` from `pixi run library-check` and `library-verify-mojo`. `library/mojo/corpus/` comes from `mojo-harvest`. `library/mojo/golden/` holds godbolt's answers and comes only from `mojo-verify`. +3. **Do not edit a `.mojo` file's text unless the task needs it.** A stored godbolt answer is keyed by the whole source, header comments included, so any edit (even to a comment) makes the program unverified until it is asked again on the network. If you add or change one, say in your summary that it needs `pixi run mojo-verify-item '/'`. +4. **Never claim Mojo compiles, runs, or prints something without the real compiler's answer.** Mojo is pinned to 1.2.0. The checker may never report an error on a program the compiler accepts, and the interpreter must print exactly what godbolt printed or decline; `tests/mojo-check.test.mjs` and `tests/mojo-run.test.mjs` hold them to that from godbolt's stored answers, and `pixi run mojo-local` and `pixi run mojo-fuzz` hold them, and the Mojo the translator writes, to the pinned compiler on your machine. Run `mojo-local` after any change to `src/mojo/` or `src/sync/pseudo-to-mojo.ts`. Every Mojo construct you write into core must pass `checkMojo`, and every standard-library name must exist in `src/mojo/generated/stdlib.ts` or `types.ts`. +5. **Never let the prover call something proved that a run breaks.** "Proved" must be a proof. A change to `src/engines/pseudo/logic.ts`, `solver.ts` or `vcgen.ts` keeps `tests/prover-soundness.test.mjs` and `tests/engine-solver.test.mjs` green, and a fact the prover assumes must hold for every value a run can produce, or be guarded by an obligation. When you find a soundness bug, add the program that shows it to `library/pseudo/prover/` with a `// MUST-NOT-PROVE:` (or `// RUN-STOPS:`) header first, see the test fail, then fix the rule ([docs/engines.md](docs/engines.md#soundness)). +6. **No new dependencies.** TypeScript is the only npm package; there is no runtime dependency. The pipelines (the Mojo corpus, the Dafny pin) are Python's standard library only. Tools that are not npm packages are pinned: Mojo, Node and Python by pixi, Dafny by `scripts/dafny.py` (a version and a SHA-256 per platform). +7. **You have no network** in most sandboxes. Godbolt (`mojo-verify`), GitHub (`mojo-harvest`, `mojo-fetch`, `dafny-fetch`) need one. Never fake their results; leave those steps to the owner and list them. +8. **Core's exports are pplus-web's API.** pplus-web imports core's built modules as `#core/` and builds against the declarations in `dist/`. Renaming or removing an export, or changing what a function returns, breaks the site; when you do it, say so in the summary so web is updated in the same release. +9. **Data lives in pplus-library, code lives here.** Programs, corpora, pins, fixtures and stored answers belong in `library/` (its own AGENTS.md has its rules); scripts, engines and tests that read them stay in core. A change under `library/` is committed and pushed in pplus-library first, then committed here with the new submodule pointer (`git add library`) and whatever core regenerated from it. + +## Conventions + +- **Commands are pixi tasks.** Name them as `pixi run ` in docs, comments, and messages. A new command is a task in `pixi.toml` with a `description` and a row in the README's Commands table; `tests/tasks.test.mjs` fails otherwise, and also fails on any `pixi run ` that names no task. `package.json` keeps only `build`, `watch`, `check`, `test` (what CI uses). +- **Docs.** The README is the front door: keep it short. Put the engines, the prover and its soundness in [docs/engines.md](docs/engines.md), verdicts in [docs/trust.md](docs/trust.md), exercises in [docs/exercises.md](docs/exercises.md), and Mojo of our own and the pipeline in [docs/mojo-corpus.md](docs/mojo-corpus.md). Update the document a change belongs in, in the same commit. +- **Tests run against `dist/` with no DOM.** A test that needs a piece of work uses a `Workspace` (`src/common/workspace.ts`) with `workForProblem` (`src/common/problems.ts`), not pplus-web's session. +- **TypeScript.** Strict mode, ES modules, imports with the `.js` suffix (`from './workspace.js'`). Indent like the file you are editing (some files use two spaces, some four). A JavaScript module's exported function gets JSDoc for its optional parameters, so the declaration pplus-web builds against says so. +- **Stable ids.** Diagnostic ids (`matho-refused`, `reserved-name`, ...) and obligation kinds (`index`, `argument`, ...) are asserted by tests and shown by pplus-web. Change one only on purpose, and update every test and link that names it. +- **Writing.** User-facing copy is US English (color, behavior, optimized, labeled). Plain sentences; say what a thing does, not how good it is. + +## Adding things + +| To add | Do | +|---|---| +| A catalog problem | `pixi run new-exercise ` for one built around a Mojo program, or copy `src/exercises/_template.ts` ([docs/exercises.md](docs/exercises.md)) | +| A question about the compiler | `pixi run new-probe ""` | +| Realistic Mojo for the interpreter | `pixi run new-stress ""`, with the output you predict in `# expect-stdout:` lines | +| A pseudo regression program | a file in `library/pseudo/programs/` whose `// EXPECT:` lines are the output it must print | +| A prover soundness regression | a file in `library/pseudo/prover/` whose `// MUST-NOT-PROVE: @` names an obligation a run breaks, or `// RUN-STOPS: @` where a run must stop | +| A verified routine others can USE | in pplus-library (`library/`), under its AGENTS.md; then `pixi run library-check` and `library-emit` here ([docs/library.md](docs/library.md)) | +| A command | a task in `pixi.toml` and a row in the README's Commands table | +| A language | a `LanguageService` (`src/common/types.ts`) in `src/services/`, registered in `src/services/index.ts` | + +## Delivering work + +- One logical change per commit, with a message that says what changed and why: the problem first, then what each part of the change does. +- When the owner asks for patches, export them with `git format-patch -o patches/ ` (the `patches/` folder is ignored by git) and do not paste whole files into chat. +- End with a summary: what you changed, the gate's result, anything you could not verify (network steps), anything that changes core's exports, and anything you noticed but left alone. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9e06264 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,68 @@ +# Changelog + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +No version has been tagged yet. The library, pplus-library, is pinned by commit and has a changelog of its own. + +## [Unreleased] + +Mojo 1.2.0, held to the real compiler on the machine as well as on godbolt.org, and the fixes three audits found. + +### Changed + +- Everything Mojo is pinned to Mojo 1.2.0 (it was 1.1.0): the pin, the harvested corpus, godbolt's answers for every program, and the words the checker and the editor know. The corpus pipeline reads a release's notes from the nightly changelog when the tag has none, and `pixi run mojo-prune` removes stored answers nothing reads. +- `pixi run library-check` builds and runs every Mojo reference and the Mojo generated from each routine on the pinned compiler on the machine; every Mojo reference has the compiler's stored answer. +- The pseudo parser, checker and interpreter (`lang`, `numeric`, `parse`, `check`, `interp`) are strict TypeScript on a typed syntax tree (`ast.ts`), with no change in behavior: 0 differences over 8,451 texts against the JavaScript they replace. +- A routine's compiled references read their cases at run time; three checks that had agreed only because the compiler worked the answer out while building are now real. +- New limits, each a located diagnostic or a named stop and all listed in `docs/limits.md`: how deep and how long a Mojo, lingo, matho or pseudo text may be, how large a value a Mojo run may make, how much work the prover and the generated tests may do, and how much memory an x86-64 setup may name. +- The sources are in US English: `normalizeAnswer` (was `normaliseAnswer`), the lesson and concept id `memoization`, "not modeled" in messages. +- Mojo is built one stated way everywhere core builds it or asks godbolt to (the Compile button, the corpus, the library's references, `mojo-local`, `mojo-fuzz`, `mojo-run`): the arguments are `build.args` in the library's pin, before a program's own. `--target-cpu x86-64-v3` makes a listing the same whichever of godbolt's machines answers (one answered with AVX-512, which the model machine does not know); `--fp-mode contract=off` rounds every Float64 operation on its own, as both interpreters do, where Mojo's default fuses a multiply with an add. `BUILD_ARGS` carries them and `BUILD_LEVELS` is new. `mojo build` with no arguments targets the machine it runs on and may fuse (`docs/trust.md`). +- The Mojo syntax definition the pipeline writes states its license as Apache-2.0 (it said MIT), and `THIRD_PARTY.md` says that the engines and the lessons, which came from the course, are under this repository's copyright and license. + +### Added + +- `pixi run mojo-local` builds every Mojo program core holds or generates on the pinned local compiler and holds the checker, the interpreter and the translator to it; `pixi run mojo-fuzz` does the same with generated programs. +- `pixi run mojo-fuzz --statements` generates whole programs (loops, branches, collections, records, routines with BYREF), builds each with the pinned compiler and holds its output to the pseudo's line by line; `pixi run prove-fuzz` holds the prover to runs of generated programs. `pixi run ci` runs both on a fixed and a moving seed, and `pixi run nightly` runs them longer. +- The Mojo interpreter runs what the translator writes: typed list literals, `List[T](slice)`, `ord` and `chr`, std.math's exact functions (`gcd`, `lcm`, `comb`, `hypot`, `fma`, `nextafter`, `ulp`, `copysign`, `isinf` and the others), `atof` of a decimal, and the `match` statement. What it declines on purpose (a fractional power, the transcendental functions, random numbers) says why. +- CI workflows that run the gate a developer runs (`pixi run ci`: every test, strict corpus status, library-check, mojo-local and a short fuzz), and a scheduled one for the longer fuzz. +- `docs/limits.md`, `docs/ci.md`, and a test that holds the sources, scripts and tests to US English. +- Third-party inventories: `THIRD_PARTY.md` here and the library's, written from the pin, the lock and every harvested file, and tests that fail when they fall behind. +- The Apache License 2.0, a NOTICE, contributing rules, a security policy, a code of conduct, and a citation file. + +### Fixed + +- The translator writes Mojo the compiler accepts and that prints what the pseudo prints. +- The Mojo interpreter: a numeric literal beside a Float32 is a Float32; a float to a whole power is computed as the compiler computes it, and to any other power the interpreter declines; a list in brackets takes a List's element type. +- The prover: 18 rules that proved what a run breaks are closed, each with a regression program in the library. +- Pseudo: wrong answers (MOD past 2^53, CHOOSE, sums over whole numbers, literals past 2^53, sized types leaving their type, FLOAT32 literals rounded twice, comparisons with USE IEEE), crashes on malformed input, and missing limits (quantifier bodies and the Prove tab's fuzzing now have one), from an audit against exact arithmetic. +- Pseudo: a FUNCTION called inside an IF, WHILE or UNTIL condition ran twice, because the step's explanation evaluated it again; a second routine of a name was checked with the first one's record, and one named after a built-in could make the checker throw. +- The translator, from the statement fuzzer: about twenty-five shapes the compiler rejected or that printed something else (two names that become one, a name declared in a block and used after it, a variable some path leaves unset, DELETE on a set, slices, constants' types, CONTINUE in a REPEAT, an argument evaluated once per pass of a loop). Each is pinned in `tests/sync-mojo.test.mjs`. +- A function sent from a compiled listing to the x86-64 pane brought its read-only data as `bytes 0x.. 0x..`, five characters a byte: with the compiler's half-megabyte table of source locations the setup was 2.5 MB, more than a share link may hold. A long block is quoted text with `\xHH` escapes, which the setup reads back to the same bytes; a `#`, `;` or `//` inside quoted text is text. +- No text makes a reader or a run end in the engine's own error: 20 nested `ABS` no longer take the prover's memory, 30,000 nested brackets are one diagnostic, and anything a reader still throws becomes `engine-stop` on its pane. +- `scripts/mojo/listings.mjs` (run by `pixi run play`) wrote every function of a program into its exercise's x86-64 pane, not the one the exercise is about and what it calls as `docs/mojo-corpus.md` says: for `demo` 8,634 lines and a 2.9 MB `listings.ts`, with which the test suite passed a 3 GB memory cap. It writes the entry and its callees again, the stored listing's `build` is the arguments its answer was asked with (the pin's first), and a test fails when `listings.ts` is not what the tool writes or grows past 64 kB. +- A FLOAT made into a whole number must be one the lab holds exactly. +- What core said about fused multiply-adds: with its default `--fp-mode contract=fast`, Mojo 1.2.0 fuses a multiply with an add where the operands arrive at run time; core now builds with contraction off, and three library references that needed a tolerance for it agree exactly again. +- `pixi run mojo-local` compares a program given stdin with godbolt's stored answer too: godbolt does deliver stdin. +- Pseudo: brackets that each hold a long chain (three levels of 399 values, inside both the 200-level and the 400-value limit) built a tree about 1,200 levels tall, and the checker ended in the engine's own stack error. An expression's tree is now held to 400 levels where it is complete (`MAX_TREE`), one `too-deep` error. +- `docs/engines.md` is corrected against the code (which modules are TypeScript, which built-ins carry an `argument` obligation, what the reference test runs, what this repository changed in the engines), and the reference says the prover has 3 seconds for each routine, not for the program. + +## [0.2.0] - 2026-10-06 + +The engines' first home of their own, as `main` holds it. Never tagged. + +### Added + +- pplus-core as a project of its own: the languages, translators, Mojo checker and interpreter, prover, fuzzer and x86-64 machine model, with no DOM, used by pplus-web and the course as a pinned dependency. This repository owns the engines, and `pixi run engines-export` gives them to the course. +- A `Workspace` for one piece of work and what its sources mean; Compile & run goes through a `CompileBackend`; `docs/trust.md` says what each verdict means. +- Pseudo for numerical work: sized types (INT8 to UINT64, FLOAT32), exact built-ins, `USE IEEE`. +- `USE` and pplus-library as the submodule `library/`: routines verified once and used by name. The Mojo corpus and its stored answers moved into the library. + +### Changed + +- The prover's term language, solver and verification-condition generator are TypeScript, each with tests of its own. + +### Fixed + +- The prover is sound for what its gate holds: two rules fixed, missing obligations added, a soundness test over regression programs. +- A run stops on a FLOAT that is not finite, so proofs about FLOAT hold. +- "X is in the collection" is known inside a FOR EACH pass, not on the way out. +- A call's result prints in its own type: a FLOAT result 3.0 is not shown as 3. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..9b5d663 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,21 @@ +cff-version: 1.2.0 +message: "If you use Programming+ in published work, please cite it as below." +type: software +title: "pplus-core: the languages, translators, Mojo checker and interpreter, prover and machine model of Programming+" +version: 0.2.0 +authors: + - family-names: Maldonado + given-names: Alex M. + email: alex@scient.ing + orcid: "https://orcid.org/0000-0003-3280-062X" + - name: Scientific Computing Studio, LLC + email: us@scient.ing +url: "https://pplus.scient.ing" +repository-code: "https://git.scient.ing/pplus/pplus-core" +license: Apache-2.0 +keywords: + - programming education + - pseudocode + - Mojo + - program verification + - computer architecture diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..83647be --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,84 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at alex@scient.ing. All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of actions. + +**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at [https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..18d3fa2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,35 @@ +# Contributing + +Thank you for reading this far. +pplus-core is the part of Programming+ that has to be right, and it is maintained by one person; the rules below keep it honest about what it can take on. + +## Two rules first + +- **No outside code is merged until a contributor license agreement exists.** Until then, bug reports, failing programs, questions, and documentation fixes are welcome, and code changes are read and used as guidance but not merged. The agreement will be short and will ask you to confirm that you may license your change under Apache-2.0. +- **No contributions from students enrolled at the University of Pittsburgh.** The maintainer teaches there, and this project is separate from that job. Students may use Programming+ like anyone else. + +## Running the checks + +Programming+ uses [pixi](https://pixi.sh/latest/) for its environments; it brings the pinned Mojo, Node and Python. +From a clone with its submodule (`git clone --recurse-submodules`): + +```sh +pixi run test # build, then every test +pixi run ci # the gate: every test, no Mojo program of ours failing, every library routine checked, + # every program built by the pinned compiler, and a short fuzz (docs/ci.md) +``` + +Without pixi, with Node 20 or later: `npm install && npm test`. +[AGENTS.md](AGENTS.md) has the rules every change follows, whoever writes it: the gate, the generated files nobody edits by hand, and why a claim about Mojo needs the real compiler's answer. + +## Reporting a bug + +Say what you ran, what you expected, and what happened. +The best report is a program: a pseudo program and the output it should print, a Mojo program and what the compiler says about it, or a program the prover calls proved that a run breaks (that one is a soundness bug, the most serious kind; [docs/engines.md](docs/engines.md#soundness) says how they are pinned). +Where to file issues will be in the README once the repository is public. + +## Style + +US English, plain sentences. +A commit says the problem first, then what each part of the change does, and a fix comes with a test that fails without it. +Every change for a user has a line in `CHANGELOG.md`. diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..370fe53 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,167 @@ +# Apache License + +_Version 2.0, January 2004_ +_<>_ + +## Terms and Conditions for use, reproduction, and distribution + +### 1. Definitions + +“License” shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +“Licensor” shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +“Legal Entity” shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, “control” means **(i)** the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or **(ii)** ownership of fifty percent (50%) or more of the +outstanding shares, or **(iii)** beneficial ownership of such entity. + +“You” (or “Your”) shall mean an individual or Legal Entity exercising +permissions granted by this License. + +“Source” form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +“Object” form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +“Work” shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +“Derivative Works” shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +“Contribution” shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +“submitted” means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as “Not a Contribution.” + +“Contributor” shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +### 2. Grant of Copyright License + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +### 3. Grant of Patent License + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +### 4. Redistribution + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +* **(a)** You must give any other recipients of the Work or Derivative Works a copy of +this License; and +* **(b)** You must cause any modified files to carry prominent notices stating that You +changed the files; and +* **(c)** You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +* **(d)** If the Work includes a “NOTICE” text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. + +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +### 5. Submission of Contributions + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +### 6. Trademarks + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +### 7. Disclaimer of Warranty + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an “AS IS” BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +### 8. Limitation of Liability + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +### 9. Accepting Warranty or Additional Liability + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. diff --git a/NOTICE.md b/NOTICE.md new file mode 100644 index 0000000..822b626 --- /dev/null +++ b/NOTICE.md @@ -0,0 +1,16 @@ +# pplus-core + +Copyright 2026 Scientific Computing Studio, LLC + +This product includes software developed by Scienting ([scient.ing](https://scient.ing/)). + +Licensed under the Apache License, Version 2.0. +See the `LICENSE.md` file for the full license text. + +## Third-party material + +`THIRD_PARTY.md` lists what is not ours: the files under `src/mojo/generated/` and `library/syntax/mojo.xml` that are made from Modular's material, and the tools the build installs and does not redistribute. It also says where the engines under `src/engines/` and the lessons under `src/curriculum/` came from (the course site that runs the same engines); they are ours, under the same license. The submodule `library/` (pplus-library) has its own `LICENSE.md`, `NOTICE.md` and `THIRD_PARTY.md`. + +## Trademarks + +"Mojo" and "Modular" are trademarks of Modular Inc. Programming+ is not affiliated with, sponsored by, or endorsed by Modular Inc.; it uses the names only to say which language and compiler it works with. "Compiler Explorer" and "godbolt.org" name the service by Matt Godbolt that Programming+ asks for the compiler's answers; Programming+ is not affiliated with it either. diff --git a/README.md b/README.md new file mode 100644 index 0000000..45fddc5 --- /dev/null +++ b/README.md @@ -0,0 +1,152 @@ +# pplus-core + +Programming+ takes a procedure from an idea to Mojo to the machine. pplus-core is the part of it that decides what a program means: the notations and the translators between them, the Mojo checker and interpreter, the prover, the fuzzer, the x86-64 machine model, and the tools that hold all of them to the ground truth in pplus-library. It has no DOM and runs in Node as well as in a browser. pplus-web, the site, uses it as a pinned dependency, and `pixi run engines-export` copies the engines into a course's own site. + +- **Who it is for:** people who want to change or reuse the engines behind Programming+, or run them without a browser. The site, pplus-web, is where most people meet them. +- **Status:** in development, with no tagged release yet ([CHANGELOG.md](CHANGELOG.md)). It works from a clone today: the command line runs and the tests pass. +- **Try it:** [Quick start](#quick-start). **Run its checks:** `pixi run ci`, or the commands under Quick start without pixi. +- **Report a problem:** [SECURITY.md](SECURITY.md) says where to send a report and what counts as a security problem (a "proved" that a run breaks is one); [CONTRIBUTING.md](CONTRIBUTING.md) says what makes a good bug report. + +Programming+ works in five notations: + +- **lingo**: steps in plain English that a reader can run. +- **matho**: equations written in LaTeX. +- **pseudo**: pseudocode precise enough to run, prove and translate; every translation goes through it. +- **speco**: a specification to prove against: what a routine requires and ensures, and what each loop keeps. +- **Mojo**: Modular's programming language, pinned to version 1.2.0. Programming+ writes Mojo from pseudo, steps through it, and asks the real compiler about it. + +What core holds: + +- **Languages:** pseudo (parser, checker, interpreter, and the English reader for lingo), matho, speco, Mojo (a checker and an interpreter held to the real compiler's stored answers), x86-64 on a model machine with caches, a pipeline and an out-of-order core. +- **Translators:** pseudo to Mojo, lingo, matho and speco, with line maps and located refusals, and the hub that keeps the linked notations in step (`src/sync/`). +- **Verification:** the prover (`src/engines/pseudo/logic.ts`, `solver.ts`, `vcgen.ts`), meant to be sound and held to it by the soundness regressions in `library/pseudo/prover/` (an audit in November 2026 closed 18 rules that proved what a run breaks), a canary, and a run of every corpus program ([docs/engines.md](docs/engines.md#soundness)); the fuzzer (`src/common/fuzz.ts`); and a pinned Dafny (`scripts/dafny.py`). +- **A workspace:** `src/common/workspace.ts` is one piece of work and everything its sources mean. pplus-web's session extends it. +- **The ground truth:** pplus-library, the submodule `library/`, holds what the engines are held to; core also builds its library registry and the exercises' Mojo from it: + - verified routines (statistics, numerics, alignment, molecular simulation, sequences, special functions) that a program brings in with `USE`, each checked by every tool `pixi run library-check` has ([docs/library.md](docs/library.md)); + - the Mojo corpus (`library/mojo/`): the pin, material harvested from Modular's repositories at pinned commits, our own probes and programs, and godbolt's stored answer for every program we wrote and every harvested program that runs (the compiler's own diagnostic tests are asked only on request) ([docs/mojo-corpus.md](docs/mojo-corpus.md)); + - pseudo regression programs and the prover's soundness regressions (`library/pseudo/`). +- **A command line:** `pixi run cli check|run|prove|fuzz|translate ` (`translate` takes `--to mojo`, `lingo`, `matho` or `speco`). + +What core says about Mojo is pinned to **Mojo 1.2.0**. The checker and the interpreter are held to the real compiler's answers on the corpus: to godbolt's stored answers in the tests, and to the pinned compiler on a developer's machine with `pixi run mojo-local` and `mojo-fuzz`, which also build the Mojo the translator writes. On a program outside the corpus the interpreter can still run something the compiler rejects; [docs/trust.md](docs/trust.md) says what each verdict means and what it does not. + +## Quick start + +Clone with the library submodule (`git clone --recurse-submodules`, or `git submodule update --init` in an existing clone). With [pixi](https://pixi.sh) (it brings the pinned Mojo, Node and Python): + +```bash +pixi run dafny-fetch # once: the pinned Dafny, into .tools/ (network; until then the one test that runs it is skipped) +pixi run test # build, then every test +pixi run ci # the gate: every test, no Mojo program we wrote failing by godbolt's stored answers, library-check, mojo-local and a short fuzz +pixi run cli prove library/pseudo/programs/19-max-subarray.pseudo # each obligation and its status; the last line is "6 proved, 0 broken, 0 open" +``` + +The tests that need a tool that is missing (the fetched Dafny, `python3`, a C compiler `cc` for the library's C references) are skipped and say so; under CI they fail instead ([docs/ci.md](docs/ci.md)). + +Without pixi, with Node (`package.json` asks for 20 or later; the tests run on Node 26.10, which pixi pins) and `python3` for the pipelines' tests: + +```bash +npm install && npm test +python3 scripts/mojo/corpus.py status --strict +``` + +## Commands + +Every command is a pixi task (`pixi task list` shows them with their descriptions). `tests/tasks.test.mjs` fails if this table and `pixi.toml` disagree. + +| Task | What it does | +|---|---| +| **Build and test** | | +| `install` | Install TypeScript, the only npm dependency | +| `build` | Compile `src/` to `dist/`, with type declarations for pplus-web | +| `watch` | Rebuild `dist/` on every change | +| `check` | Type-check only | +| `test` | Build, then run every test, the pipelines' Python tests included | +| `cli` | `pplus check\|run\|prove\|fuzz\|translate `: core without a browser ([scripts/cli.mjs](scripts/cli.mjs)) | +| `prove-fuzz` | Fuzz the prover: random procedures with contracts are proved, then run; fail on a proved obligation a run breaks ([docs/engines.md](docs/engines.md#soundness)) | +| `engines-diff ` | Show what a course checkout has not taken yet from the engines ([docs/engines.md](docs/engines.md)) | +| `engines-export ` | Copy the engines into a course checkout, TypeScript modules as built ([docs/engines.md](docs/engines.md)) | +| **Dafny, pinned** ([scripts/dafny.py](scripts/dafny.py)) | | +| `dafny-fetch` | Download the pinned Dafny into `.tools/`, checked by SHA-256 (network, once) | +| `dafny` | Run the pinned Dafny with the arguments given, e.g. `pixi run dafny verify file.dfy` | +| **The library** ([docs/library.md](docs/library.md)) | | +| `library-check` | Check every library routine at every level; writes `library/status.json` and `STATUS.md` | +| `library-emit` | Regenerate `src/library/generated.ts`, the registry `USE` reads, from `library/` | +| `library-verify-mojo` | Ask the pinned Mojo compiler about every Mojo reference of the library (network) | +| `library-harness ` | Print the program a reference implementation is run as | +| `library-harness-input ` | Print the line of stdin a C or Mojo reference's program reads (its cases arrive at run time) | +| **Write content** | | +| `new-exercise ` | Start an exercise whose Mojo lives in `library/mojo/local/.mojo` ([docs/exercises.md](docs/exercises.md)) | +| `new-probe ` | Start `library/mojo/probes/.mojo`: one question about the compiler ([docs/mojo-corpus.md](docs/mojo-corpus.md)) | +| `new-stress ` | Start `library/mojo/stress/.mojo`: realistic Mojo that tests the interpreter | +| `play ` | For `library/mojo/local/.mojo`: run, verify on godbolt, build, list the assembly, test | +| `play-watch ` | The same, again on every save (tests off) | +| **The Mojo corpus** ([docs/mojo-corpus.md](docs/mojo-corpus.md)) | | +| `mojo-verify` | Ask godbolt about every program with no answer yet (network) | +| `mojo-verify-item ` | Ask again about one item, e.g. `'probes/life-*'` (network) | +| `mojo-verify-dry` | Show what `mojo-verify` would send; send nothing | +| `mojo-verify-diagnostics` | Also check the compiler's own diagnostic tests (opt-in; network) | +| `mojo-status` | Judge every item from the stored answers, offline; writes `library/mojo/golden/STATUS.md` | +| `mojo-status-strict` | The same, and fail if a program we wrote fails | +| `mojo-prune` | Remove stored answers nothing reads any more: another compiler's, and those of edited or removed programs | +| `mojo-harvest` | After editing `library/mojo/pin.json`: fetch Modular's sources, process, regenerate (network) | +| `mojo-harvest-local [dir]` | Harvest, with zips standing in for sources the pin leaves unresolved | +| `mojo-harvest-offline [dir]` | Harvest from zips only, no network | +| `mojo-fetch` | Re-resolve every ref in the pin and rewrite the lock (network) | +| `mojo-emit` | Regenerate `src/mojo/generated/` and `library/syntax/mojo.xml` from the pin and corpus, offline | +| `mojo-test` | The corpus pipeline's own tests | +| `mojo-run ` | Run one Mojo file on the pinned local compiler, built with the pin's build arguments and then its own | +| `mojo-local` | Build every Mojo program core holds or generates on the pinned local compiler; hold the checker, interpreter and translator to it | +| `mojo-fuzz` | Fuzz the translator, the checker and the interpreter against the pinned local compiler with generated programs | +| `mojo-probes-local` | Run every probe on the local compiler (a quick look) | +| `mojo-update` | After editing the pin: harvest, verify, status, test | +| **Ship** | | +| `ci` | What CI runs: every test, and no program of ours failing | +| `ci-fuzz` | The fuzzers' share of `ci`: `mojo-fuzz` and `prove-fuzz` on seed 1 and on a seed that moves, each printed ([docs/ci.md](docs/ci.md)) | +| `nightly` | What the scheduled CI runs: `mojo-status-strict`, and the fuzzers longer on four seeds that move with the date | +| `package` | Zip what git tracks, and nothing it ignores, into `package/` for sharing | + +## Where things are + +``` +AGENTS.md rules for changing core (read first) +src/ + common/ the workspace, the problem catalog, the fuzzer, the run view's data, the language + reference, the machine presets, the shared types + library/ the registry USE reads (generated from library/), and its types + engines/pseudo/ the pseudo language and the prover (logic, solver, vcgen in TypeScript), and a course's + page widgets for it (excluded from the build; see docs/engines.md) + engines/asm86/ the x86-64 model machine, and a course's widgets that draw it (views, hwviews) + engines/shared/ text helpers, and a course's DOM helpers + services/ one adapter per language behind LanguageService; the compiler request and backend + sync/ the translators and the hub that keeps the linked languages in step + mojo/ the Mojo checker, interpreter, lifetimes, practices, AT&T-to-Intel; generated/ from the corpus + exercises/ one file per catalog problem + curriculum/ a lesson corpus kept as an interpreter fixture +library/ pplus-library, a git submodule: the ground truth + modules/ verified routines that programs bring in with USE + mojo/ the Mojo corpus: the pin and lock, harvested material, our programs, godbolt's answers + pseudo/ programs/ (pseudo regression programs), prover/ (soundness regressions) + syntax/mojo.xml the Mojo words as a KDE syntax for Pandoc (generated; pplus-web copies it) +tests/ node --test against dist/, over the programs in library/ and the cases in each test +scripts/ the tasks above; scripts/mojo/ is the corpus pipeline; dafny.py the Dafny pin; cli.mjs +docs/ the documents below +``` + +## Read next + +| Document | For | +|---|---| +| [AGENTS.md](AGENTS.md) | Rules for anyone changing core, coding agents included: the gate, what not to edit, conventions | +| [docs/engines.md](docs/engines.md) | The engines, the prover and its soundness, and how a course's site takes them | +| [docs/trust.md](docs/trust.md) | Every verdict core produces: which tool earns it, what it means, and what it does not mean | +| [docs/limits.md](docs/limits.md) | Where each reader and each run stops on text from a share link, and what the student sees then | +| [docs/ci.md](docs/ci.md) | What CI and the nightly run do, and what to do when they fail | +| [docs/mojo-corpus.md](docs/mojo-corpus.md) | Writing Mojo of our own and moving to a new Mojo version | +| [docs/library.md](docs/library.md) | The library: USE, what each check of a routine means, and changing it | +| [docs/exercises.md](docs/exercises.md) | Adding a catalog problem | +| [docs/sync-coverage.md](docs/sync-coverage.md) | What each translator can say, written by the tests | +| [docs/asm-coverage.md](docs/asm-coverage.md) | Which instructions compiled Mojo uses, against what the machine models | + +## License + +Copyright 2026 Scientific Computing Studio, LLC, under the Apache License 2.0 ([LICENSE.md](LICENSE.md)); [NOTICE.md](NOTICE.md) has the notices and a disclaimer of affiliation with Modular, and [THIRD_PARTY.md](THIRD_PARTY.md) lists what is not ours and its licenses. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..5e8f42e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,16 @@ +# Security + +pplus-core reads programs and runs them inside its own interpreters: pseudo, Mojo and x86-64 are interpreted by JavaScript written here, with a step limit, and nothing a program says is ever handed to JavaScript's `eval` or to a shell. +The prover and the fuzzer work on the same trees. +A program that makes an engine crash (an exception escapes), hang past its step or time limits, use memory without bound, or reach anything outside the program's own state is a security issue, because pplus-web runs these engines on whatever a link carries. +So is a "proved" verdict for something a run breaks, which is a soundness bug: it would let a student trust a wrong program. + +Some tasks run real tools on your machine, and those run what you give them: `pixi run mojo-local`, `mojo-fuzz` and `library-check` build and run Mojo, C and Python programs from this repository and from the library, and `mojo-verify`, `mojo-verify-item`, `play` and `library-verify-mojo` send the corpus's programs and the library's Mojo references to godbolt.org. +Run them on a checkout you trust. + +Report a problem to Alex M. Maldonado at alex@scient.ing, with the program that shows it and what happened. +You will get an acknowledgment within a week. +We promise no response time beyond that; one person maintains Programming+. +Please do not open a public issue for a problem that could hurt someone else until a fix is out. + +Only the latest commit on `main` gets fixes. diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md new file mode 100644 index 0000000..a727b5f --- /dev/null +++ b/THIRD_PARTY.md @@ -0,0 +1,50 @@ +# Third-party material in pplus-core + +Everything in this repository is ours, under the Apache License 2.0 ([LICENSE.md](LICENSE.md)), except what this file lists. `tests/third-party.test.mjs` holds the versions and licenses below to `package-lock.json`, `pixi.lock` and `scripts/dafny.py`, and holds the list of generated files to `src/mojo/generated/`, so a change there fails the tests until this file says so. + +## The library + +`library/` is pplus-library, a repository of its own pinned as a submodule. Its [THIRD_PARTY.md](library/THIRD_PARTY.md), written by `pixi run mojo-emit`, lists every file harvested from Modular's repositories (`library/mojo/corpus/`), its source commit and the license header it carries, and the license file of each source at the pinned commit (`library/mojo/licenses/`). + +## Made here from Modular's material, and shipped in the app + +`pixi run mojo-emit` writes these from the corpus. Each file's first lines name the commit it came from. + +| File | Made from | What it is | +|---|---|---| +| `src/mojo/generated/tokens.ts` | modular/modular: `TokenKinds.def`, `lib/MojoParser`, the decorator reference, the stdlib prelude (Apache-2.0 WITH LLVM-exception) | Lists of the language's words: keywords, soft keywords, punctuation, decorators, prelude names, the words 1.x removed. Two messages, for `fn` and `read`, are the compiler's own wording, quoted from its source. | +| `src/mojo/generated/stdlib.ts` | modular/modular: `stdlib/std` (Apache-2.0 WITH LLVM-exception) | Which package exports each public standard-library name: names and module paths, no code. | +| `src/mojo/generated/types.ts` | modular/modular: `stdlib/std` (Apache-2.0 WITH LLVM-exception) | Each standard struct's and trait's name, the traits it declares, and the names of the methods a trait gives by default: names and relations, no code. | +| `src/mojo/generated/editor.ts` | modular/vscode-mojo: `language-configuration.json` (Apache-2.0) | The comment marker, bracket pairs and indentation patterns an editor uses for Mojo: a few configuration values. | +| `src/mojo/generated/pin.ts` | `library/mojo/pin.json` and the lock | The Mojo version, the godbolt compiler id and the source commits: ours. | +| `src/mojo/generated/programs.ts` | `library/mojo/local/` and `library/mojo/golden/` | Our own exercise programs, each with the verdict Compiler Explorer's answer gave it. | +| `src/mojo/generated/listings.ts` | `library/mojo/golden/` (written by `pixi run play`) | The Mojo compiler's assembly listing for each of our exercise programs, as Compiler Explorer returned it. | +| `library/syntax/mojo.xml` | the same words as `tokens.ts` | A KDE syntax definition for Pandoc listings. Its header attribute says `license="Apache-2.0"`, which `scripts/mojo/mojocorpus/kde.py` writes. | + +The Mojo checker (`src/mojo/`) reports some errors in the compiler's own words (for example "use of unknown declaration"), so that a student sees the message the compiler would print; `tests/mojo-check.test.mjs` holds them to the compiler's stored answers. These are short phrases, written into our code by hand. + +## Stored compiler answers + +`library/mojo/golden/` (in pplus-library) holds the answer [Compiler Explorer](https://godbolt.org) returned for each program it was asked about: what the Mojo compiler printed, its exit code and, for some, its assembly. The programs are ours (probes, exercises, stress programs, the library's references) and the harvested corpus. Compiler Explorer is BSD-2-Clause software run by Matt Godbolt and volunteers; its [privacy policy](https://github.com/compiler-explorer/compiler-explorer/blob/main/static/generated/privacy.pug) says it caches compilation results and deletes source code within minutes, and neither the site nor its [API documentation](https://github.com/compiler-explorer/compiler-explorer/blob/main/docs/API.md) states terms of use for its answers. + +## The engines and the lessons, which came from the course + +`src/engines/` (the pseudo, asm86 and shared modules, and the course's page widgets beside them) and `src/curriculum/` (a lesson corpus kept as an interpreter fixture) come from the course site that also runs these engines. [docs/engines.md](docs/engines.md) says the engines "started as a copy of the course's modules" and that this repository now owns them. They are listed here to say where they came from, not because they are someone else's: both are Copyright Scientific Computing Studio, LLC, under the Apache License 2.0, like the rest of the repository. + +## Installed to build and test, not redistributed + +pixi installs these into `.pixi/`, npm into `node_modules/`, and `scripts/dafny.py` into `.tools/`; git ignores all three. No compiler binary, runtime library or standard-library source of Mojo is in this repository or in its history. + +| Tool | Version | From | License | In what we ship | +|---|---|---|---|---| +| Mojo (`mojo`, `mojo-compiler`, `mojo-python`, `mblack`) | 1.2.0 | conda.modular.com, by pixi | LicenseRef-Modular-Proprietary (Modular's license, as `pixi.lock` records it) | Nothing. Tasks run the compiler on our programs. | +| Node.js | 26.10.0 | conda-forge, by pixi | MIT | Nothing. | +| Python | 3.14.7 | conda-forge, by pixi | Python-2.0 | Nothing. The pipelines use its standard library. | +| TypeScript | 6.0.3 | npm (`package-lock.json`) | Apache-2.0 | Nothing. `tsc` compiles `src/`; the output is our code. | +| Dafny | 4.11.0 | the GitHub release zip, by `scripts/dafny.py`, checked by SHA-256 | MIT (Dafny and the Z3 it carries, with .NET runtime files; Dafny's own NOTICES.txt lists the rest) | Nothing. Its verdicts are stored in `library/modules/*/*/golden/dafny.json`. | + +The other packages pixi installs come with these from conda-forge; `pixi.lock` names each with its license. + +## Names + +"Mojo" and "Modular" are trademarks of Modular Inc. Programming+ is not affiliated with or endorsed by Modular. "Compiler Explorer" and "godbolt.org" name the service by Matt Godbolt that the app asks. diff --git a/docs/asm-coverage.md b/docs/asm-coverage.md new file mode 100644 index 0000000..40cb6e4 --- /dev/null +++ b/docs/asm-coverage.md @@ -0,0 +1,51 @@ +# The model machine against compiled Mojo + +"Open in Hardware" puts a function from the real compiler's listing on the model machine (`src/engines/asm86`), translated from godbolt's AT&T syntax by `src/mojo/att.ts`. The machine implements a teaching subset of x86-64, so the question is how much of what the Mojo compiler actually emits it can take. This is the census: every function in every listing the corpus holds (`library/mojo/golden/*.json` answered with `asm`), sent exactly as the button sends it, and what the assembler said. `node scripts/asm-census.mjs` reruns it (add `--ops` for every mnemonic); `tests/asm-coverage.test.mjs` holds the numbers as a ratchet, so a new corpus item that brings a new instruction fails the build rather than a student's session. + +## Where it stands + +| | before | now | +|---|---|---| +| distinct instructions in the listings | 97 | 104 | +| of which outside the model's set | 38 | 6 (all 256-bit AVX2: `vextracti128`, `vinserti128`, `vpblendvb`, `vperm2i128`, `vpalignr`, `vptest`) | +| the user's own functions that assemble | 3 of 7 (`main` never did) | 17 of 17 | +| standard-library and runtime functions that assemble | — | 698 of 733 | + +The "before" column is the census of the listings the corpus held when this work began; the "now" column is today's census of today's listings, and `tests/documents.test.mjs` fails when it no longer matches `node scripts/asm-census.mjs`. Every listing is built for `--target-cpu x86-64-v3` (the pin's build arguments, [mojo-corpus.md](mojo-corpus.md#how-every-program-is-built)), so it holds AVX2 at most and never AVX-512, whichever of godbolt's machines answered; `tests/build-args.test.mjs` fails on a listing with a `zmm` or mask register. Asked again under those arguments, every listing the corpus held came back with the same instructions; `vptest` and four modeled ones (`adc`, `cmovle`, `jns`, `tzcnt`) came with `probes/godbolt-target-cpu`, String's `==` at the default optimization level. + +`main()` now steps up to its first call into `print` and stops there with "stopped: std_io_io_print: standard library, not on the model machine", which is the honest answer: the model has no standard library, and says exactly where the real program would enter it. + +## What was wrong, and what changed + +**The translator dropped size suffixes only for the mnemonics it knew.** AT&T writes `andnq`, `mulxq`, `shlxq`, `xaddq`, `cmpxchgq`, `btsq`, `bswapq`; the Intel pane reads `andn`, `mulx`, … The BMI and atomic mnemonics were missing from the suffix table, so the machine was asked for an instruction called `andnq`, which exists nowhere. They are in the table now, and `lock` (a prefix) is passed through as a prefix rather than treated as a mnemonic with an operand called `xaddq`. + +**The machine lacked the instructions the optimizer likes.** Added, each with a one-sentence gloss for the datapath view and the micro-op model told what it reads and writes: + +- BMI1/BMI2: `andn`, `mulx`, `shlx`, `shrx`, `sarx`, `bzhi`, `blsr`, `blsi`, `blsmsk`, `rorx`, `bextr` (`pdep`/`pext` are named as not modeled) +- the atomics behind reference counts: `xadd`, `cmpxchg`, and the `lock` prefix, accepted on the read-modify-writes the ISA allows it on +- bit tests that write: `bts`, `btr`, `btc`; `bswap`; `cwd`, `cwde`, `cbw` +- `ud2`: the trap the compiler plants on paths that must never run; reaching it stops the run with "the processor raised an invalid-opcode fault" +- the 128-bit AVX spellings of what the subset had: `vmovdqu`, `vmovdqa`, `vmovups`, `vmovaps`, `vmovd`, `vmovq`, `vpxor`, `vxorps`, `vpor`, `vpand`, `vpandn`, `vpaddb`, `vpsubb`, `vpcmpeqb`, `vpmovmskb`, `vzeroupper`; three-operand form (`vop dst, a, b`), sources untouched +- lanes: `vpinsr{b,w,d,q}`, `vpextr{b,w,d,q}`, `vpshufb`, `vpshufd`, `vpbroadcast{b,d}`, `vpmin/max{ub,ud}`, `vpcmpeq{w,d,q}`, `vpadd{w,d}`, `vpsubd`, `vpsubusb`, `vpsll/vpsrl{w,d,q}` + +**A call into the standard library refused the whole listing.** The assembler resolves every label up front, so one `call std::io::print` meant the function could not be stepped at all, even to the instruction before the call. The translator now appends a stub for every such symbol (`label:` then `ud2 ; …: standard library, not on the model machine`), so the function assembles, steps up to the call, and the stop says what it called. + +**Read-only strings were unreachable.** `lea rax, [rip + static_string_…]` named a label that was data, not code, and the machine has no data directives. The translator now finds the `.asciz` behind each such label, places the bytes in memory (`memory 0x6000 "…"`, octal and hex escapes decoded, from 0x6000 up), writes the address in the instruction's place, and hands the lines to the setup block, where the student can see them. + +## Scalar doubles, parity, shld (added) + +- `addsd subsd mulsd divsd sqrtsd movsd cvtsi2sd ucomisd comisd` and their `v` forms: IEEE binary64, round to nearest even, default NaN `0xFFF8000000000000`, a NaN operand is quieted and the first operand wins. MXCSR exception flags are not modeled. +- `ucomisd`/`comisd` set ZF/PF/CF as hardware does (unordered sets all three; OF, SF clear). +- PF is a real flag now: set by logic, add/sub and shifts, cleared by `popcnt`, untouched by rotates. `jp jnp jpe jpo`, `cmovp…` and `setp…` follow. +- `shld`/`shrd` on 32- and 64-bit operands; count masked to 5/6 bits; count 0 changes nothing, flags included. +- Latencies: add/mul 4, div 14, sqrt 18 (unpipelined on port 0), convert 5, compare 3. + +## What stays outside, and why + +- **256-bit AVX2 (`ymm`)**: the model keeps sixteen 128-bit `xmm` registers and no `ymm` halves; the six instructions above are used only in that width. A `ymm` operand is refused with that sentence. Modeling it means widening every vector register and every lane helper; worth doing only when a user function needs it, and none in the corpus does. +- **Symbols the listing addresses but does not define**: `KGEN_…` runtime globals, `…@GOTPCREL` function pointers, a `.zero` constant. The assembler names the symbol in its refusal. +- **Constant-pool blocks with a symbol in them** (jump tables): not placed; the instruction that loads one is refused by name. Numeric `.LCPI…` blocks (`.quad`, `.long`, `.double`, `.zero`, …) *are* placed, 16-byte aligned from 0x6000, when the code addresses them, and `.LCPI0_0+8(%rip)` works. + +## Keeping it honest + +The census runs against whatever the corpus holds: every stored answer that carries a listing (the probes and exercise programs whose header asks for `mode: asm` or `mode: both`). `pixi run mojo-verify` on a new such program adds its listing, and each addition either assembles or fails `tests/asm-coverage.test.mjs` with the instruction named. That test's ratchet (`LIBRARY_AT_LEAST`) is a floor under the table's library count; raise it when the machine learns more, never lower it. diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 0000000..d5cb9f3 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,48 @@ +# CI + +CI runs what a developer runs. Every workflow installs pixi, restores the environment `pixi.lock` pins (Node, Python, the Mojo compiler) with `pixi install --locked`, and runs a pixi task, so the gate on the runner and the gate on a laptop are the same command in the same environment. Five weeks once passed with `main` broken in two places and nobody told: CI then ran `npm ci && npm test` with whatever Node the host had, no pandoc, no Mojo compiler and no Dafny, and a missing tool skipped tests in silence. + +| Workflow | When | Runs | Time here | +|---|---|---|---| +| `.gitea/workflows/test.yml` | every push and pull request | `pixi run ci`, then checks that nothing the gate regenerates differs from what is committed | 4 minutes here, 2.5 with `mojo-local`'s answers cached | +| `.gitea/workflows/nightly.yml` | every day at 06:17 UTC, and by hand | `pixi run nightly` | 12 minutes here | +| pplus-library's `.gitea/workflows/test.yml` | every push and pull request to the library | core's default branch with that library commit in `library/`, then `pixi run ci` (below) | as core's | + +`pixi run ci` is `test` (every test), `mojo-status-strict` (no Mojo program of ours fails, from godbolt's stored answers), `library-check` (every library routine at every level, Dafny included), `mojo-local` (every Mojo program core holds or generates, built and run by the pinned compiler) and `ci-fuzz` (below). `pixi run nightly` is `mojo-status-strict` and the fuzzers for longer. + +## The runner + +The workflows are written for Gitea Actions on a runner in host mode (no container: steps run on the host's shell); pplus-web's `docs/deploy.md` has the setup. They use no `container:` and no `services:`, and each needs one change to run on GitHub Actions: `runs-on: ubuntu-latest` in place of the host's label. + +- **pixi.** `PIXI_VERSION` at the top of each workflow is the pixi that reads the lock. The first step uses the host's pixi when it is that version; otherwise it downloads that release from GitHub into `$PPLUS_CI_CACHE/pixi-/` (default `~/.cache/pplus-ci`), checks its SHA-256 (written in the workflow), and puts it on the PATH. No sudo; a host runner fetches it once. To move pixi: change `PIXI_VERSION` and both hashes in every workflow, in one commit, and check that `pixi install --locked` still accepts the lock. +- **The environment.** `pixi install --locked` fails if `pixi.toml` and `pixi.lock` disagree, so the runner never resolves anything new. pixi's package cache stays on the host, so a host runner downloads each package once. +- **`.tools/`.** The pinned Dafny (`pixi run dafny-fetch`, about 150 MB) and `mojo-local`'s answers live in `.tools/`; in CI `.tools` is a link to `$PPLUS_CI_CACHE/core-tools`, so neither is fetched or rebuilt on every push. `mojo-local` keys an answer by the compiler's version, the arguments, the input and the whole source, so a cached answer is the compiler's answer for exactly that program. Two jobs that share the cache at once could write the same answer file together; the runner takes one job at a time unless its `capacity` is raised. +- **`CI`.** Each workflow sets `CI=true` (Gitea and GitHub set it too). Under it a test that would skip for a missing tool fails instead, because a skip in CI is how a gap hides: python3 (`tests/mojo.test.mjs`, `tests/dafny.test.mjs`, `tests/library.test.mjs`), the fetched Dafny (`tests/dafny.test.mjs`), a git checkout to read the library pointer from (`tests/chain.test.mjs`), and in pplus-web pandoc for the Quarto tests. +- **What the host needs besides pixi:** git, curl, tar, `sha256sum`, bash, and Node for `actions/checkout` itself; a C compiler (`cc`) for the C references `library-check` builds. Without `cc` those checks are recorded as skipped, `library/status.json` then differs from the committed one, and the last step of `test.yml` fails and says so. + +## The fuzzers' seeds + +`pixi run ci-fuzz` runs `mojo-fuzz` (25 programs of expressions), `mojo-fuzz --statements` (25 whole programs) and `prove-fuzz` (300 programs) on two seeds each: seed 1, so a failure that comes back is the same failure, and a seed that moves, so every run looks somewhere new: the CI run's number (`GITHUB_RUN_NUMBER`, which Gitea sets too) or, outside CI, today's date in UTC as `YYYYMMDD`. `pixi run nightly` runs all three on four seeds, the date's number and the three numbers after it, with 100, 100 and 3,000 programs. The size matters: the statement mode's first run at 100 programs a seed found a translator fault (an argument evaluated once per pass of a loop) that many seeds of 40 had not. Every round prints its seed and the command that reproduces it: + +``` +== prove-fuzz seed 20261113 (reproduce: node scripts/prove-fuzz.mjs --seed 20261113 --programs 3000) +``` + +and the run ends with a summary, one line per round, `clean`, `FOUND` or `FAILED`. + +## When CI or the nightly run fails + +1. **Find the round.** The summary at the end of the log names it, with its command. +2. **Reproduce it here.** Run the same command as a pixi task, with the same seed and size: `pixi run prove-fuzz --seed 20261113 --programs 3000`, or `pixi run mojo-fuzz --seed 20261113 --programs 100`. The same seed makes the same programs on every machine. The prover has a time budget for each routine (`--budget-ms`, 1,500 ms), so a slower machine may prove less than the runner did, never more; if a prove-fuzz finding does not come back, raise the budget. +3. **A prove-fuzz finding** is a soundness bug: an obligation marked proved that a run breaks. The report names the program's number and prints the smallest program that still shows it, with the call and the run's seed. `pixi run prove-fuzz --seed S --only-program N --keep work/` writes it as a regression program with its `// CALL:` and `// MUST-NOT-PROVE:` header. Check it by hand (`pixi run cli prove ` says proved; running it with that call breaks it), put it in `library/pseudo/prover/`, watch `tests/prover-soundness.test.mjs` fail, then fix the rule (AGENTS.md, rule 5; [engines.md](engines.md#soundness)). `--without ` switches off one construct at a time to see which one the finding needs. +4. **A mojo-fuzz finding** is printed once per shape with the smallest expression that shows it; its header (`scripts/mojo/fuzz.mjs`) says what each kind means. A translator fix is pinned in `tests/sync-mojo.test.mjs`; a finding that is the compiler's own is listed there as such, with what was observed. +5. **mojo-status-strict** failing means a program we wrote fails against godbolt's stored answers: see [mojo-corpus.md](mojo-corpus.md). +6. **Anything else** in `pixi run ci` fails the same way on a developer's machine: run `pixi run ci` and read the first failure. + +A failure opens no issue and sends nothing: the run is red on the repository's Actions page, for whoever looks. Look after a push, and look at the nightly runs at least weekly. + +## The pointer to the library + +Core pins pplus-library as `library/` by commit. `tests/chain.test.mjs` fails when that commit is not in the library checkout, or the checkout is behind it, and says which command fixes it (`git submodule update --init`). The whole chain (web pins core, core pins library; each pointer pushed to its branch; the generated files current; Mojo pinned alike) is checked by pplus-web's `release-check` task, and moved in order by its `release-bump` task (pplus-web's `docs/deploy.md`). + +pplus-library has no tests of its own: its checks are core's. Its workflow checks out core's default branch (or the ref in the repository variable `CORE_REF`), puts the library commit being tested where core's submodule goes, moves core's pointer to it in that throwaway checkout and regenerates what core derives from the library (`library-emit`, `mojo-emit`), and runs `pixi run ci`. A green run shows that core's default branch, given this library and what core regenerates from it, passes core's whole gate. It does not show that core pins this commit (that is a later commit in core), that the site works with it, or anything about a core branch other than the one tested; and a red run can be core's own, so look at core's latest run first. diff --git a/docs/engines.md b/docs/engines.md new file mode 100644 index 0000000..6496e46 --- /dev/null +++ b/docs/engines.md @@ -0,0 +1,121 @@ +# The engines + +Programming+ runs the language engines that the course site also runs, so a program behaves the same in the app as on a course page. The course's widget stylesheets live in pplus-web (`styles/course/`), which compares them with the course's. **This repository owns the engines.** Changes are made and tested here, and the course copies them from here (`pixi run engines-export`). The engines started as a copy of the course's modules; the course no longer sends changes the other way. + +| Here | The course receives it as | What it is | +|---|---|---| +| `src/engines/pseudo/` | `static/js/pseudo/` | The pseudo language: parser, checker, interpreter, the English (lingo) reader, the prover (`logic.ts`, `solver.ts`, `vcgen.ts`), translation | +| `src/engines/asm86/` | `static/js/asm86/` | The x86-64 model machine, its caches, pipeline and out-of-order core, and the widgets that draw them | +| `src/engines/shared/` | `static/js/shared/` | DOM helpers, URL state, text distance, charts | + +The engines are wrapped behind one `LanguageService` interface (`src/common/types.ts`, adapters in `src/services/`). The course's page widgets (the pseudo views, the asm86 `views`/`hwviews`, the shared DOM helpers) sit beside the engines so the course and pplus-web take them from one place; the pseudo ones and a few others are excluded from core's build in `tsconfig.json`, and `tests/boundary.test.mjs` keeps core's library modules from importing any of them. A change in `src/engines/` is a change to the course's language, so it needs a test here and goes to the course by export. + +## The prover + +The prover is three TypeScript modules under strict mode. So are the parser, checker and interpreter it reads from: `ast.ts` states the syntax tree (discriminated unions `Expr` and `Stmt`), the checker's types (`PType`), symbols, run-time values and the trace, and `lang.ts`, `parse.ts`, `check.ts` and `interp.ts` are typed on it; `numeric.ts` (the exact arithmetic, which needs none of it) is TypeScript too. All five were ported with no change in behavior (each stage's results were compared over every pseudo text of the repository and 8,000 mutants of them). The course still receives each as `.js`, built into `dist/engines/pseudo/`. The rest of the pseudo engine (the English reader, analysis, coach, tests, translation, `study`, the course's example programs in `specs` and the vague-word finder in `vague`) and the page widgets are still JavaScript. + +- `logic.ts`: the term language (a discriminated union, `Term`), the smart constructors that simplify as they build, keys, substitution, linear forms and printing. +- `solver.ts`: the decision procedure, `prove(facts, goal)`. It answers `proved` or `unknown`, never "false". Equalities by union-find with congruence, linear arithmetic over whole numbers by Fourier-Motzkin with integer tightening, case splits, and FORALL used at the positions a case mentions. It is sound and incomplete: products of two unknowns, MOD by an unknown, and most facts about collections are opaque to it. +- `vcgen.ts`: the proof obligations, by symbolic execution, with loops cut at their invariants and simple invariants found by Houdini-style inference. + +"Proved" is about the notation's own number model. INTEGER is a whole number of any size and a FLOAT is a finite number whose rounding the prover does not see into. A run of the interpreter stops where a value leaves that model (a whole number past 2^53 − 1, a FLOAT that is not finite, a division by zero), so every run stays inside what a proof covers. The Mojo made from a program has other limits (a 64-bit `Int` that wraps, a `Float64` that becomes `inf` or `nan`), and a unit whose proof leans on the number model carries an `int-model` or `float-model` note saying so. + +### Numbers + +Pseudo is meant for numerical work, so its number model is stated exactly ([the reference](../src/common/reference.ts) shows every form with a runnable example): + +| Type | Holds | When a result does not fit | The prover | +|---|---|---|---| +| `INTEGER` | A whole number of any size; a run holds it exactly to 2^53 − 1 | The run stops past 2^53 − 1, also when the number comes from a FLOAT (`INT`, `ROUND`, `FLOOR`, `CEIL`), from SUM, on a FOR counter, and from INPUT or `TO_INTEGER` | Exact arithmetic of any size; an `int-model` note says the lab's and Mojo's limits are not part of the proof | +| `INT8` … `INT64`, `UINT8` … `UINT64` | Two's complement or unsigned, n bits (64-bit types held exactly to 2^53 − 1 in this lab) | The run stops (kind `overflow`); `WRAP(X, T)` wraps on purpose (up to 32 bits) | Exact arithmetic plus an `overflow` obligation on every result, store, parameter, RETURN, conversion and declared FOR counter: "fits INT8". After it, the range is a fact. A sized parameter and a sized variable cut by a loop keep their range | +| `FLOAT`, `FLOAT64` | IEEE 754 double | The run stops on infinity or NaN | `FLOAT+` and its kind are uninterpreted: the same operation on the same values gives the same result, nothing more | +| `FLOAT32` | IEEE 754 single: every result rounded to 24 bits (`Math.fround`, exact for +, −, ×, ÷ of singles) | As FLOAT | Its own uninterpreted functions (`F32+`, ...); a constant is rounded exactly; a value that becomes a FLOAT32 is `TOFLOAT32(v)`, about which nothing is assumed | + +Rules the checker holds: two different sized types never mix silently (`mixed-width`, `mixed-precision`; convert with `INT32(X)`, `FLOAT32(X)`); a numeric literal takes the type of the sized value beside it, as in Mojo, and a whole-number literal that does not fit is an error before any run (`literal-range`); INTEGER may go into a sized type (checked when it happens), never a different sized type. + +`USE IEEE` lets NaN and the infinities flow as IEEE 754 says (SQRT(−1.0) is NaN, 1.0 / 0.0 is infinity; whole-number division by zero still stops). Where IEEE 754 says what a built-in gives, that is what a run gives: `MIN` and `MAX` with a NaN among their values are NaN wherever it stands (754-2019 `minimum` and `maximum`), `ULP` of an infinity is infinity (as Python's `math.ulp`; 754 defines no ulp), `^` and `POW` follow 754's `pow` (1 to any power, NaN included, is 1, and so is −1 to an infinite power), and `SORTED` puts NaN last. The prover's reasoning about numbers does not hold for those values, so in such a program nothing is marked proved, and each unit carries an `ieee` note. Without it, NaN and the infinities do not enter a run from outside either: a call's argument (`run()` with `call`) that is or holds one for a FLOAT parameter stops the run before the call (kind `value`). + +What is exact or correctly rounded, and what is not: `FMA`, `ULP`, `NEXT_AFTER`, `EXACT`, `SQRT`, the conversions (`FLOAT32(X)`, `INT32(X)`, a decimal literal that becomes a FLOAT32), `MAX_VALUE`, `EPSILON`, the bit operations, `GCD`, `LCM`, `CHOOSE`, `DIV` and `MOD`, and `+`, `-`, `*`, `/` are: `src/engines/pseudo/numeric.ts` computes `FMA` with one rounding by exact binary arithmetic, `EXACT(X)` gives every decimal digit a FLOAT holds, and `ULP` and `NEXT_AFTER` step through the representation. The elementary functions (`EXP`, `LOG` and its kind, `EXPM1`, `LOG1P`, `CBRT`, `HYPOT`, the trigonometric and hyperbolic ones, `POW` and `^` with a FLOAT) are the JavaScript engine's: within one unit in the last place of the correctly rounded value in almost every case (`HYPOT` within two), not always equal to it (`EXPM1(1.0)` is 1.718281828459045, where the nearest FLOAT is 1.7182818284590453). The statistics (`SUM_OF`, `MEAN_OF`, `VARIANCE_OF`, `STDEV_OF`, `MEDIAN_OF`, `DOT`, `NORM_OF`) are the plain left-to-right sums a loop would compute, with that loop's rounding and its overflow on the way: over whole numbers every product and partial sum is checked as `*` and `+` check theirs (`DOT([94906267, 1], [94906267, -9007199254740991])` stops, though the total fits), over FLOATs each is rounded in turn; `MEDIAN_OF` sorts and takes the middle value or the mean of the two middle ones (`NORM_OF` of one value past about 1.34e154 stops, since its square passes the largest FLOAT, `SUM_OF([1e16, 1.0, -1e16])` is 0.0); a lesson on compensated summation writes its own. `IS_CLOSE` evaluates its formula in FLOAT arithmetic, as Python's `math.isclose` does. `DIV` and `MOD` are floored: the remainder has the divisor's sign (`-7 DIV 2` is -4, `-7 MOD 2` is 1). The arithmetic inside `WRAP(...)` is exact only to 2^53 - 1, so a product of two 32-bit values can stop there before it is wrapped. Functions JavaScript has no exact version of (erf, gamma) are not built-ins; they belong in the library as verified pseudo. These built-ins carry an `argument` obligation that their arguments are allowed: SQRT, the logarithms, ASIN and ACOS, LOG1P, CLAMP, the shifts, `FILL` and `RANDINT`. Others that stop the run for some arguments (`MIN`, `MAX` and `MEAN_OF` of an empty collection, `^`, `CHR`, `DOT` of two lengths) carry none. The built-ins added for numerical work stay usable as names (a variable `E`, a `FUNCTION GCD`; `SOFT_BUILTINS` in `lang.ts`): a FUNCTION or PROCEDURE of the same name replaces the built-in, and a variable `E` is read as `E` while `E()` still calls the built-in. + +`tests/numeric.test.mjs` pins each rule; the corpus programs `49-sized-integers` to `52-elementary` show them in use. + +### Limits of a run + +The reference's "Limits" and "What stops a run" tables (`src/common/reference.ts`) say the same to a student; `tests/reference.test.mjs` runs a program for every row of "What stops a run" (and fails on a row without one), and for five rows of "Limits": nesting, chains, a collection's size, steps and call depth. + +- **What the parser reads.** A line inside more than 200 blocks, or brackets, calls, unary operators or types nested more than 200 deep, is one `too-deep` error (`MAX_NESTING` in `parse.ts`); so is a chain of more than 400 operands (`1 + 1 + ...`, `MAX_CHAIN`). The two are counted apart and a chain leans one level per operand, so brackets that each hold a long chain would multiply them; an expression whose tree is taller than 400 levels is therefore `too-deep` as well (`MAX_TREE`, measured where each expression is complete). No later stage (checker, analysis, translators) walks a taller tree; `tests/pseudo-robustness.test.mjs` takes the tallest through every one of them. +- **What a run holds.** One collection holds at most 2,000,000 values (an ARRAY's declared size, `FILL`, a range walked by `FOR EACH` or listed for a quantifier), and a text at most 10,000,000 characters (`MAX_VALUES`, `MAX_TEXT` in `interp.ts`); past either the run stops (kind `value`) instead of allocating. +- **How long a run goes.** The step limit (20,000 in the app's runs, `maxSteps`) stops a loop that may never end (kind `limit`). The positions FORALL, EXISTS, SUM and sets written by rule visit are counted apart, up to 100 per step allowed, so one quantifier inside another cannot run without end. Calls go 400 deep (`maxDepth`, kind `depth`); a recursion whose call sits inside many nested blocks can exhaust the engine underneath before 400 calls (with ten blocks around the call, somewhere between 200 and 300 calls, varying from run to run), which is the same stop, at the line the run was on. +- **The Prove tab's budget.** The verifier's work is counted, not timed (`work` in `logic.ts`): one obligation's proof, one subroutine's paths and invariants, its proofs, and the whole program each have a budget, the same on every machine, and past one the answer is open ("too large to prove here") or the subroutine partial, never proved ([limits.md](limits.md#proving-and-testing)). A wall-clock budget stays behind them as a last resort: `provePseudo` gives the verifier 3 seconds per unit, then tries generated inputs on each routine within a counted budget of steps and also for up to 3 seconds (`checkProperties` with `maxWork` and `budgetMs`): a slow routine is tried on fewer inputs, not waited for. +- **Recording.** A recorded run (the stepper, the table of values) keeps a picture of the variables after every step, and a picture copies each variable the step changed. A large collection that changes at many steps is copied at each of them, so once the copies of a run come to more than 10,000,000 values (about 80 MB of numbers; `maxRecorded` in `interp.ts`), the trace ends with a step of kind `unrecorded` that says so, and the run goes on without recording: its output, result, stop and line counts are the whole run's. No program of the corpus, the exercises or the lessons comes near it (the largest copies about 42,000). `tests/pseudo-robustness.test.mjs` holds the cut with a small budget. + +- **Checker errors and stops added in the 2026-11 audits** (each in `tests/pseudo-robustness.test.mjs`, `tests/numeric.test.mjs` or `tests/reference.test.mjs`): `too-deep` (the limits above), `literal-range` for a whole-number literal past 2^53 − 1 and a FLOAT or FLOAT32 literal past its largest value, `recursive-type`, `array-of-array`, `array-length` (an ARRAY given a LIST or ARRAY of another length, when both are written as numbers), `bad-member` (a SET of anything but single values). Stops: the collection and text caps, the quantifier positions, an ARRAY given a value of another length at a store, a call or a RETURN, a FOR counter declared with a sized type leaving its type, a partial sum or product of `SUM_OF` or `DOT` past 2^53 − 1, `CHR` of a surrogate half, `TO_FLOAT` or `INPUT` of text that is not a decimal number, and NaN or an infinity passed to a FLOAT parameter from outside without `USE IEEE`. + +### Soundness + +A "proved" must be a proof. Two test files hold the prover to that, and a change to `logic.ts`, `solver.ts` or `vcgen.ts` must keep both green: + +- `tests/prover-soundness.test.mjs`: every regression program in `library/pseudo/prover/` names, in a `// MUST-NOT-PROVE: @` header, an obligation that a run breaks and that an unsound rule once proved; it may never be proved again, and the program must still break it. A program whose fix was to the language names where its run now stops, `// RUN-STOPS: @`. Across every exercise, corpus program, lesson solution and example, nothing marked proved may be broken by the fuzzer or by running the program, and `ASSERT 1 == 2` appended to a program that runs to its end may never be proved. +- `tests/engine-solver.test.mjs`: a table of goals the solver must prove and goals it must leave unknown, 400 random linear integer formulas checked against every small assignment, and obligations from small programs. + +Those hold what someone wrote down. `pixi run prove-fuzz` ([scripts/prove-fuzz.mjs](../scripts/prove-fuzz.mjs)) writes programs nobody did: procedures with contracts, loops with INVARIANTs (FOR, WHILE, FOR EACH, REPEAT), STEPs that are expressions, CASE, FLOAT, FLOAT32 and sized types, calls on the right of AND and OR, and FUNCTIONs that draw a random number or read INPUT. Each is proved, then run on random arguments; a proved obligation a run breaks is a finding, printed as the smallest program that still shows it (`--keep DIR` writes it with its `// MUST-NOT-PROVE:` header, ready for `library/pseudo/prover/`). CI runs it on every push and longer every night ([ci.md](ci.md)). + +When a soundness bug is found, add the program that shows it to `library/pseudo/prover/` first, watch the test fail, then fix the rule. The bugs found so far, each with its program: + +| Program | Rule | Fix | +|---|---|---| +| `houdini-cap.pseudo` | Invariant inference stopped after four rounds and assumed the candidates left, even when the last round still dropped some | Keep inferred invariants only at a fixed point | +| `rand-zero.pseudo`, `rand-negative.pseudo` | `RAND(X)` assumed `0 <= R < X` for every X, false for X ≤ 0, so the checker held a contradiction and proved anything | A fact that covers all three signs of X | +| `fill-negative.pseudo`, `randint-empty.pseudo` | `FILL` and `RANDINT` assumed facts about arguments the interpreter checks by stopping, without asking whether they hold | An `argument` obligation, and guarded facts | +| `float-overflow.pseudo` | The prover treats FLOAT as numbers (`Y == Y`), while a run went on with infinity and NaN | The interpreter stops on a FLOAT that is not finite | +| `foreach-empty.pseudo` | The way out of `FOR EACH` kept "X is in the collection", false after zero passes over an empty list | The fact holds only inside a pass | +| `int8-overflow.pseudo` | (a guard written with the sized types) Sized arithmetic treated as exact with no range would prove `X + 1` fits INT8 | Every sized result has an `overflow` obligation | +| `float32-store.pseudo` | (a guard) A whole number stored in a FLOAT32 keeping its exact value would prove `X == N` for N = 16777217 | A value becoming a FLOAT32 is rounded (constants) or `TOFLOAT32(v)` | +| `ieee-nan.pseudo` | (a guard) `X == X` for every number fails for NaN | Nothing is proved in a program that says `USE IEEE` | +| `havoc-float-sort.pseudo` | At the top of a loop a changed variable became an unknown of the sort its value had, so a FLOAT that started at `1` was a whole number on every pass | The unknown has the sort of the variable's declared type | +| `havoc-array-length.pseudo` | "An ARRAY keeps its dimensions through a loop" was assumed even when the body replaces the array whole (`A <- B`) | Assumed only when the body never assigns, declares or passes BYREF the bare name; and an ARRAY keeps its declared length: a store, an argument (BYREF too), a RETURN or a value APPENDed to a LIST OF ARRAY of another length is a checker error (`array-length`) when both lengths are plain numbers, and otherwise stops the run, where the prover asks for it (an `argument` obligation) and knows it afterwards; an ARRAY parameter's lengths are known inside the routine | +| `shortcircuit-range.pseudo`, `shortcircuit-ensure.pseudo`, `case-label-ensure.pseudo` | A fact learned while translating an expression (a sized result's range, a callee's ENSURE) was kept even where a short-circuit `AND`/`OR` or an earlier CASE label means the run never evaluates it | Such a fact is guarded by the conditions under which the expression is evaluated (`learn`) | +| `quantifier-ensure.pseudo` | The ENSURE of a call inside a quantifier's body was kept as a fact about the quantifier's own name, one free name shared by every quantifier | No ENSURE is stated for a call inside a quantifier's body | +| `impure-function.pseudo`, `input-function.pseudo`, `randn-twice.pseudo` | Every FUNCTION call, and `RANDN()`, was a function of its arguments, so two calls were equal | A function that reads INPUT or draws a random number (itself or through a call) gives a fresh unknown each call, with its ENSUREs; so does `RANDN()` | +| `rand-in-set-builder.pseudo` | A random draw in a quantifier's or set builder's body was one unknown for every value of its name | A body that draws makes the whole quantifier an unknown | +| `foreach-invariant-exit.pseudo`, `foreach-invariant-next.pseudo` | FOR EACH assumed its INVARIANT on the way out, where no run checked it, and proved "kept" for the value just visited while the run checked with the next | An INVARIANT that does not name the loop's variable is checked by the run once more when the loop ends (also for an empty collection) and is known afterwards; one that names it is proved for the value about to be visited and is not known afterwards | +| `repeat-invariant-exit.pseudo` | The run checks a REPEAT loop's INVARIANT after the last pass; "kept" covered only passes that repeat | The keep obligation covers the pass that ends the loop too | +| `step-not-literal.pseudo` | A FOR whose STEP is not a plain number was advanced by 1 | Its counter is an unknown after each pass, and inference offers nothing about it | +| `decreases-recursion.pseudo` | A recursive call's measure was compared with the parameters as they are at the call, not as they were on entry (what the run compares with) | Compared with the values on entry | +| `decreases-entry.pseudo` | The run checks `DECREASES >= 0` on every call; nothing asked for it | A `decreases-bound` obligation at the DECREASES line, from the REQUIREs | +| `sum-float-rounding.pseudo` | SUM over one more position was the exact sum, for FLOAT too | A FLOAT SUM grows by the uninterpreted `FLOAT+` | +| `sum-past-exact.pseudo`, `sum-not-finite.pseudo` | (a run that went on) SUM was not checked as `+` is: a whole number past 2^53 - 1 or an infinity went on, and two infinities made NaN | Every partial sum is checked; the run stops | +| `linear-doubles.pseudo`, `constant-doubles.pseudo`, `for-bound-doubles.pseudo` | Constants and coefficients were added and multiplied as doubles, so `9007199254740991 + 2 - 9007199254740991` was 1 | `linear()` works in exact rationals (`Q`, now in `logic.ts`), and constants fold only when both and the result are whole numbers a double holds exactly | +| `input-past-exact.pseudo` | (a run that went on) INPUT, `TO_INTEGER` and a FOR counter let a whole number past 2^53 - 1 into a run, where DIV and MOD are no longer exact | The run stops, as it does for `+` | +| `index-of-empty.pseudo` | `INDEX_OF` assumed a result below the length; an empty piece is found at 0 in an empty text | The fact allows 0 | +| `chr-surrogate.pseudo` | (a run that went on) `CHR` gave the halves of a surrogate pair, two CHARs that joined into one character | `CHR` stops for 0xD800 to 0xDFFF | +| `float-in-range.pseudo` | `X IN [lo, hi)` and membership of a set written by rule were `lo <= X < hi` for a FLOAT too; in a run only a whole number is in a range | Left open (an opaque term) unless X is a whole number | +| `set-literal-mixed.pseudo` | `{1, 2.5}` was a SET OF INTEGER holding 2.5 | A set literal widens to FLOAT, as a list literal does (`check.ts`) | +| `max-float32.pseudo`, `append-float32.pseudo` | A FLOAT32-typed value that was never rounded (MAX of a FLOAT32 and an INTEGER; APPEND of an INTEGER into a LIST OF FLOAT32) was taken for a FLOAT32 | MIN, MAX and CLAMP round a FLOAT32 answer, APPEND and INSERT store a value as the list's own type (and check a sized whole number), on both sides | +| `set-of-lists.pseudo` (removed) | A run's SET finds a LIST by identity; the prover compared contents | Membership of a collection in a SET was left open; then a SET's members were limited to the single values a HASHMAP key may be (checker error `bad-member`), so the program no longer checks and `tests/pseudo-robustness.test.mjs` holds the error instead | +| `sorted-2d.pseudo` | `SORTED` of a two-dimensional ARRAY has rows times columns values; the prover gave it the first length | Nothing is said about its length | +| `for-sized-counter.pseudo` | The run checks at the top of each pass that a FOR counter declared with a sized type (`DECLARE I : INT8`) still fits it, and stops there; no obligation asked, so a unit with everything proved still stopped | An `overflow` obligation at the FOR line, under the loop's guard, after which the range is a fact | + +Fixes with no program of their own, because the program that showed them is now rejected or no longer breaks (each has a test in `tests/numeric.test.mjs`): `CLAMP(I, 0.5, 2.5)` was typed INTEGER and is now the widest of its three types; `UPPER`/`LOWER` of a CHAR whose other case is two characters (`'ß'`) leaves it as it is, so a CHAR is always one character; `RAND(X)` no longer returns X itself for the smallest FLOATs; an ARRAY or HASHMAP given as an argument from outside is checked value by value; INPUT into a FLOAT takes a finite number only (unless `USE IEEE`). + +The harness reads a regression program's `// INPUT: ` headers (one per line of input) and `// CALL: `. A program may carry both a `MUST-NOT-PROVE` and a `RUN-STOPS` header when the run reports the failure on another line than the obligation's (a recursive DECREASES is reported at the DECREASES line, its obligation sits at the call). + +In the app, an obligation the prover marks proved that a generated input breaks at the same place is shown as broken, not proved. A failure that says which half of a contract it broke (an INVARIANT before the first pass or after one, a DECREASES below 0 or not smaller) counts against that obligation only. + +## What this repository changed + +Changes made here since the engines left the course, which the course receives with the next export: + +- **pseudo:** the text built-ins `SPLIT`, `JOIN`, `INDEX_OF`, `REPLACE` and the numerical built-ins (`FLOOR`, `CEIL`, `LOG10`, `SIN`, `COS`, `TAN`, `ATAN2`, `PI`, the statistics) in `lang`, `parse`, `check`, `interp`, `translate`, `vcgen`; `PARALLEL FOR` with its race detector; an indexed read `L[I]` recorded as a read of one position, not all of `L` (`interp`); one `reserved-name` error for a keyword used as a name (`parse`); `english.js` exporting `COMPARISONS` and `IN_PHRASES` so the reference can read them; `analyze.js` and `coach.js` treating a variable passed `BYREF` to a procedure that writes it first as an out-parameter, not a read; `coach.js` no longer warning "growing bound" on a queue drained by `WHILE LENGTH(Q) > 0`; the soundness fixes above, with the new `argument` obligation kind (`vcgen`, and `views-analysis.js` for the course's pages); a run that stops on a FLOAT that is not finite (`interp`); the sized types, FLOAT32, `USE IEEE` and the exact numerical built-ins (`numeric`, new, and `lang`, `parse`, `check`, `interp`, `vcgen`); `USE` of the library's routines by name (`parse`, `check`, `interp`); the limits of what the parser reads and a run holds, the recording cut, and the checker errors and stops of the 2026-11 audits (above); the prover's counted budgets (`work` in `logic`); a FUNCTION in an IF, WHILE or UNTIL condition run once per test (`interp`); and `logic`, `solver`, `vcgen`, `lang`, `numeric`, `parse`, `check` and `interp` rewritten in TypeScript with the same behavior, on the types in `ast.ts`. +- **asm86:** what compiled Mojo uses beyond the course's subset (`interp.js`, about 630 lines): the BMI bit instructions, atomic read-modify-writes with `lock`, bit tests that write, `bswap`, `ud2`, the 128-bit AVX spellings, scalar double precision, and the double-width shifts; a limit on the memory a setup names (a region of at most 1 MiB) and on what a run's writes copy (128 MiB), in `interp.js`; the parity flag (`interp`, `uops`, `views`); the floating-point units and their latencies (`uops`); a small change in `sandbox.js`; and in `hwviews.js` the whole core's arrows (`machineView`): only a lit edge carries an arrowhead, in its own color and on the end its data reached, each view has its own marker ids, and the result's edge into the registers starts at the port that produced it. + +One addition to the pseudo language lives outside the engine, in app code: `DECLARE : <- `, a declaration with its starting value on one line (Mojo's `var total: Int = 0`). `src/services/pseudo-study.ts` reads it by doing what the engine's `study()` does (parse, check, analyze, coach) with steps of its own: before the parse it stops reading past 2,000 lines or 100,000 characters (`too-long`), and between parse and check it puts in the starting values and brings in the routines a `USE` line names. For the starting value, the `<- value` is blanked out of the line, columns kept, so the engine parses a plain DECLARE, and the value is parsed on its own and put in as an ordinary assignment on the same line. Every app reader of pseudo (`studyOf`, and the lingo, matho and Mojo translators) goes through it; pseudo-to-mojo writes the pair as one `var` line. The two-line form reads as before. It can move into the engine's parser now that the engine lives here. + +## Giving the course the engines + +1. `pixi run engines-diff /static` shows what the course has not taken yet: for each module, the file the course would receive (`src/engines/` as written, `dist/engines/` for a TypeScript module) against the course's copy, whitespace ignored, and the files only one side has. +2. `pixi run engines-export /static` builds and writes those files into the course's `static/js/`. A TypeScript module arrives as plain ES-module JavaScript, so the course's pages import it unchanged. Source maps are not copied. +3. In the course repository, review the diff and run its pages before committing. +4. The course's stylesheets are pplus-web's to compare (`styles/course/README.md` there). + +A change someone makes in the course's copy is not picked up from there. Bring it here as a change with a test, then export. diff --git a/docs/exercises.md b/docs/exercises.md new file mode 100644 index 0000000..ffa8458 --- /dev/null +++ b/docs/exercises.md @@ -0,0 +1,38 @@ +# Adding an exercise + +Programming+ takes a procedure from an idea to Mojo to the machine. An exercise is one procedure on that road, written in any of its notations: lingo (steps in plain English), matho (equations written in LaTeX), pseudo (pseudocode precise enough to run, prove and translate), speco (a specification to prove against), Mojo (Modular's programming language, pinned to version 1.2.0) and x86-64. It is a catalog problem: one file in `src/exercises/`, listed in `src/exercises/index.ts`. The home page, the problem menu, share links, embeds, and the tests pick it up from there. `pixi run test` checks every exercise: its pseudo parses, its call runs, its answer matches `expect`, and its Mojo program (if it names one) exists and prints what it should. + +There are two ways to start one. + +## An exercise around a Mojo program + +```bash +pixi run new-exercise rolling-sum # library/mojo/local/rolling-sum.mojo, src/exercises/rolling-sum.ts, registered +pixi run play rolling-sum # after each edit: run, verify on godbolt, build, list the assembly, test +``` + +For a title or a home-page section other than the defaults, run the script directly: `node scripts/new-exercise.mjs rolling-sum --title "Rolling sum" --group "Loop forms"`. If `library/mojo/local/rolling-sum.mojo` already exists, it is kept as written and only the exercise is made. + +The exercise names its program (`mojoProgram: 'rolling-sum'`) instead of carrying the source. The Mojo pane shows the file without its header comments, and badges it "verified on Mojo 1.2.0" only when godbolt's answer for that exact source passed the file's own `# expect-stdout:` lines. Editing the file changes its key, so it reads as unverified until `play` (or `mojo-verify`) asks again. The header keys are in [mojo-corpus.md](mojo-corpus.md#mojo-of-our-own). + +With `# mode: both` and `# args: -O0 --debug-level=line-tables` in the header, godbolt also returns the listing, and `play` turns it into the exercise's x86-64 pane (see "The assembly" in [mojo-corpus.md](mojo-corpus.md#the-assembly)). + +## An exercise in lingo, pseudo, matho or speco + +Copy `src/exercises/_template.ts` to `.ts`, fill it in, import it in `src/exercises/index.ts`, and add it to the list (before `demo`, the playground, which stays last). Then `pixi run test`. + +## The fields + +| Field | Means | +|---|---| +| `id` | The URL (`#//p/`): lowercase words joined by hyphens | +| `title`, `group`, `blurb` | The name, the home-page section it files under (a new name makes a new section), and one or two sentences on what the student does | +| `meta` | Optional: `lecture`, `level` (`intro`, `core`, `advanced`), `tags`, `concepts`, `minutes`, `page` (where it is taught) | +| `sources` | Any of `lingo`, `pseudo`, `matho`, `speco`, `asm`. Pseudo is derived from lingo (or matho, written in LaTeX) when absent, and speco from the pseudo's inline contracts | +| `mojoProgram` | The name of a program in `library/mojo/local/`, shown in the Mojo pane | +| `call`, `expect` | The call to run, and what it should return; the tests compare them | +| `setup`, `machine` | For assembly: the setup block (registers and memory before the run) and the machine model preset | +| `lang` | The pane that opens first | +| `area` | The area it opens in: `bridge`, `dev` or `hw`. Absent: Hardware for `asm-*` ids, Developer for a problem that is only a Mojo program, Bridge otherwise | + +`src/exercises/types.ts` documents each field in full. diff --git a/docs/library.md b/docs/library.md new file mode 100644 index 0000000..15b7e3b --- /dev/null +++ b/docs/library.md @@ -0,0 +1,43 @@ +# The library + +Programming+ takes a procedure from an idea to Mojo to the machine. pplus-library (the submodule `library/`) is its ground truth. Its routines, which this document is about, are written once in pseudo (pseudocode precise enough to run, prove and translate); besides them it holds the Mojo corpus (`library/mojo/`, see [mojo-corpus.md](mojo-corpus.md)) and the pseudo regression and soundness programs (`library/pseudo/`). Each routine is checked by every tool `pixi run library-check` has (the table below), so a program can use them by name instead of rewriting and re-verifying them, and a learner can open any of them to see exactly what it does. Core reads it through a generated registry (`src/library/generated.ts`, `pixi run library-emit`), so the site needs no files at run time. + +## Using a routine + +``` +USE md // every routine of the module +USE align.MatchScore // one routine +``` + +USE lines go at the top of a program. From then on a routine is called like the program's own: + +- **The checker** types each call against the routine's parameters. A program may not define a routine of the same name (`library-clash`); an unknown module or routine is an error at the USE line, with the names that exist (`unknown-library`, `unknown-routine`). +- **A run** steps over a routine: one step where it is called, one where it hands back, and none of its own lines (they belong to another file). Its contracts are still checked; a broken REQUIRE is reported at the call, naming the routine. To step through a routine, open it from the site's library page, which opens it in the Bridge. +- **The prover** uses the routine's REQUIRE as an obligation at each call and its ENSURE as a fact afterwards, the way it treats the program's own routines. It does not prove the routine again: the library did. +- **The Mojo translator** writes each routine the program uses once, before the program's own code, dependencies first, labeled as coming from the library. + +A routine file (`routine.pseudo`) holds USE lines and exactly one FUNCTION or PROCEDURE. A routine that calls another brings it in with USE, so every function in the library is checked on its own. + +## What "verified" means for a routine + +`pixi run library-check` runs every check and records what it found in `library/status.json` and `library/STATUS.md`: + +| Check | Passes when | Recorded as | +|---|---|---| +| structure | `routine.pseudo` defines exactly the routine `routine.json` names, and nothing else | pass, fail | +| pseudo | every case runs; its result is recorded and is what the references are held to | pass, fail | +| prover | every obligation of the routine is proved | pass; open (some not proved, none broken: the routine's comment says which and why, and `tests/documents.test.mjs` fails when it does not mention it); fail (one broken) | +| fuzz | every input the fuzzer generates for the routine breaks no contract and stops no run (`STATUS.md` gives how many it generated) | pass, fail, skipped (the fuzzer cannot generate inputs for it) | +| mojo | the Mojo generated from the pseudo checks, and prints what the pseudo prints both in core's interpreter and built by the pinned compiler on this machine (to the last digits or nearly: [trust.md](trust.md#what-a-translation-says)) | pass (by both, or by the one that could run it), fail, skipped (the interpreter declines the construct and there is no pinned compiler here) | +| a reference, per runner | `cpython`, `cc -O0`, `cc -O2`, core's Mojo `interpreter`, the pinned Mojo `compiler` on this machine, or the pinned compiler on `godbolt` prints the pseudo's results on every case, exactly or within the tolerance `routine.json` states; a C or Mojo reference is given its cases at run time (below) | pass, fail, skipped (the tool is missing, or the `mojo` on the PATH is not the pinned version), unverified (no stored compiler answer for this exact program, input and build arguments) | +| proof | the pinned Dafny verifies the proof file, or its stored verdict matches this exact text | pass, fail, unverified | + +A reference implementation is the function alone; the tool writes the program it runs as (the function plus a main that prints each case), and `pixi run library-harness ` prints it. A Python reference's program calls it on each case's arguments. A Mojo or C reference's program holds no case: its main reads every case's arguments from one line of stdin, which `pixi run library-harness-input ` prints, so the compiler cannot work a call out while it builds (a call on literal arguments it would, with arithmetic and a math library of its own; built with contraction on, as Mojo is by default, with other rounding than the code it makes: `library/mojo/probes/float-contract-runtime`), and every runner is given the same program and the same line. Every Mojo program is built with the pin's arguments (`build.args` in `library/mojo/pin.json`: `--target-cpu x86-64-v3 --fp-mode contract=off`, [mojo-corpus.md](mojo-corpus.md#how-every-program-is-built)), on godbolt and on this machine, so a Mojo reference rounds each Float64 operation on its own, as the pseudo does. The line is the number of cases, each case's arguments, then `end`; a FLOAT is sent as two whole numbers m and e with the value m * 2^e, exactly, a STRING with `%`, space and line breaks escaped, a LIST as its length and then its items. `tests/library-harness.test.mjs` fails if a case's arguments are ever written into the program again. A Mojo reference's answer from the compiler is stored in the routine's `golden/mojo-.json`, keyed by the SHA-256 of that program, the stdin it was given and the build arguments, by `pixi run library-verify-mojo` (network). A Dafny verdict is stored in `golden/dafny.json`, keyed by the proof's text and the Dafny version. + +What each check does not say is in [trust.md](trust.md#library-routines). + +## Changing the library + +1. Change pplus-library under its [AGENTS.md](../library/AGENTS.md): one routine per file, cases, references with their notes and tolerances, a Dafny proof where one says something. +2. In core: `pixi run library-check` (no check may fail), then `pixi run library-emit`. +3. Commit pplus-library; commit core with the new `library` pointer and the regenerated registry. `tests/library.test.mjs` fails while the registry is out of date with the checkout. diff --git a/docs/limits.md b/docs/limits.md new file mode 100644 index 0000000..5410d5a --- /dev/null +++ b/docs/limits.md @@ -0,0 +1,84 @@ +# Limits + +A program can arrive in a share link, so every reader and every run in core has to answer any text with diagnostics, an output, a stop or a refusal: never with an exception from the JavaScript engine underneath, a run without end, or an allocation the size of the text's mistake. This page lists each limit that makes that so: where it is, its value, and what the student sees past it. Each is far beyond what a program written to be read or stepped through needs, and each is proven just past its value by a test. + +## Reading + +| What | Where | Value | Past it | +|---|---|---|---| +| Mojo nesting: brackets, call and index arguments, blocks, prefix operators, `**`, `if`/`else`, `:=`, lambdas | `MAX_NESTING`, `src/mojo/parse.ts` | 200 levels | The statement is not read: one note, "nested more than 200 levels deep; Programming+ cannot read this yet, and Compile & run asks the real compiler", where the limit is passed. The checks stay quiet and a run is declined, as for any code the parser cannot read | +| Mojo tree height: each operand of a chain such as `a + b + c`, each `elif` arm, each level above | `MAX_TREE`, `src/mojo/parse.ts` | 1,000 levels | The statement (the innermost one that is too tall) is not read: one note, "more than 1000 levels deep once read" | +| Mojo liveness and moved-value analyses (a loop's body is walked twice or more per enclosing loop) | `MAX_VISITS`, `src/mojo/lifetimes.ts`, `src/mojo/flow.ts` | 200,000 statements per function | The function is not analyzed: the interpreter declines to say when a value with a visible lifecycle dies, and the moved-value check says nothing more about it | +| Generated inputs for Mojo functions (`mojoProperties`) | `MAX_PARAMS`, `MAX_FUNCTIONS`, `src/mojo/properties.ts` | 64 parameters, the first 100 functions | The report says the function has too many parameters, or is past the first 100 | +| Pseudo nesting, one chain of an operator, and the tree both make together | `MAX_NESTING`, `MAX_CHAIN`, `MAX_TREE`, `src/engines/pseudo/parse.ts` | 200 levels, 400 operands, and an expression's tree 400 levels tall (brackets that each hold a long chain) | One `too-deep` error | +| One pseudocode program (the pseudo pane, and the pseudocode any other pane becomes) | `MAX_PSEUDO_LINES`, `MAX_PSEUDO_CHARS`, `src/services/pseudo-study.ts` | 2,000 lines, 100,000 characters | One `too-long` error on the line that passes it: "This program is more than 2,000 lines long" (or "longer than 100,000 characters"), "which is more than this tool reads: the lines from here on are not read, and the program is not run or proved." Blank lines past it do not count | +| lingo nesting: brackets, "not", "minus", quantifiers, steps inside steps | `MAX_NESTING`, `src/engines/pseudo/english.js` | 200 levels | One `too-deep` error: "This is nested more than 200 levels deep, which is more than this tool reads" | +| One lingo sentence | `MAX_SENTENCE`, `src/engines/pseudo/english.js` | 1,000 words and symbols | One `too-long` error; the rest of the sentence is not read | +| lingo routines and defined phrases (every value is tried against each one's first line) | `MAX_HEADS`, `src/engines/pseudo/english.js` | 200 | One `too-long` error; the ones past it are not read | +| One lingo program | `MAX_PSEUDO_LINES`, `src/engines/pseudo/english.js` | 2,000 lines | One `too-long` error; the lines past it are not read | +| The pseudocode lingo steps become | `MAX_PSEUDO_LINES`, `src/engines/pseudo/english.js` | 2,000 lines | One `too-long` error; it is not checked or run | +| matho nesting: brackets, groups, `\frac`, `\sqrt`, powers and subscripts in braces, functions of functions, signs | `MAX_NESTING`, `src/services/matho.ts` | 200 levels | `matho-too-deep` on that equation | +| One matho equation | `MAX_TOKENS`, `src/services/matho.ts` | 2,000 symbols | `matho-too-long` on that equation; it is not read | +| Equations in one matho pane | `MAX_EQUATIONS`, `src/services/matho.ts` | 500 | `matho-too-long`; none is read | +| The pseudocode matho equations become | `MAX_PSEUDO_LINES`, `src/services/matho.ts` | 2,000 lines | `matho-too-long`; it is not checked or run | +| `LENGTH(X)` written back as `\|X\|` when the speco is made from the pseudocode | `MAX_BARS`, `src/services/speco.ts` | 200 per claim | The rest stay `LENGTH(X)` | +| A reader, translator or the prover that throws anyway | `src/common/workspace.ts`, `src/sync/hub.ts` | | One `engine-stop` error on the pane ("Programming+ could not read this pane (...)"); in the hub, every other pane keeps its text, marked stale with the reason | + +## Running Mojo + +Past any of these the run stops as it does at its step limit: the trace ends with a stop at the line, and the result is `limited` (exit code -1), because how the compiled program would have ended is not known (`RUN_LIMITS`, `src/mojo/interp.ts`; a run may ask for lower limits, never higher). + +| What | Value | The stop says | +|---|---|---| +| Steps | 20,000 (the caller's `maxSteps`) | "Stopped after 20,000 steps." | +| Calls inside calls | 200 | "Stopped at 200 calls deep" | +| Characters in one String | 10,000,000 | "Stopped: this makes a String of N characters, more than Programming+ holds (10,000,000)." | +| Values in one List, Dict or Set | 2,000,000 | "Stopped: this makes a List of N values, more than Programming+ holds (2,000,000)." | +| Characters printed | 10,000,000 | "Stopped after printing 10,000,000 characters." | +| Values made or copied into collections, over the run (what it can come to hold) | 5,000,000 | "Stopped after making or copying 5,000,000 values in Lists, Dicts and Sets" | +| Characters and values built, copied or searched through, over the run (how long it can take; the whole length of a concatenation counts, because the trace keeps every step's String) | 100,000,000 | "Stopped after building, copying and searching through 100,000,000 characters and values" | +| Variables' values the trace records, one per variable per step | 1,000,000 | "Stopped after recording 1,000,000 values of variables" | +| Passes of one call of std.math's `gcd` or `comb` (`MATH_STEPS`), which the interpreter runs as Mojo 1.2.0's own loops: its `gcd` of an even number and a much larger odd one subtracts about 10^15 times, and the built program does not come back | 1,000,000 | "Stopped: gcd(6, 4503599627370597) takes more than 1,000,000 steps in Mojo's std.math ..., more than Programming+ follows in one call." | +| The engine's own stack or size limit, past all of the above | | "Stopped: the run went deeper ... than Programming+ can follow." or "Stopped: a value grew larger than Programming+ can hold." | + +A `for` loop lists only the items it can reach before the step limit. The trace shows a String past 1,000 characters and a collection past 20 items cut short; `print` writes them whole. What the engine underneath cannot compute is declined, as anything the interpreter does not model is ("Programming+ cannot run this yet: ..."): a shift by more than 64 bits, converting `inf` or `nan` to an integer, a float literal with more than 10,000 digits or an exponent past 10,000, and any other exception, which is a fault in Programming+ rather than an answer. + +## The model machine + +| What | Where | Value | Past it | +|---|---|---|---| +| Steps | `maxSteps`, `src/engines/asm86/interp.js` | 20,000 | The run is truncated | +| One memory region a setup maps | `MAX_REGION` | 1 MiB | The run does not start: "this machine maps regions of 0 to 1,048,576 bytes (1 MiB)" | +| Memory copied while writing (each step that writes a region keeps a copy for stepping back) | `MAX_COPIED` | 128 MiB | The run stops: "the steps so far wrote to memory regions totaling 128 MiB" | +| The stack | `stackSize` | 512 bytes | A page fault, explained as a stack overflow | + +## Proving and testing + +The prover's work is counted, not timed: each term it builds or visits, each case of a case split, each pair of constraints it combines, each fact it copies is a unit (`work` in `src/engines/pseudo/logic.ts`). A budget therefore runs out at the same point on every machine and in every test, and because the units are counted where memory is allocated, memory stays in proportion. Past a budget the answer is never "proved": an obligation is open with the reason "too large to prove here", or the subroutine is partial and says so (in the Prove tab a partial subroutine has nothing marked proved). Every program of the corpus takes far less than each budget, and proves the same as before they were added (`tests/prover-limits.test.mjs` proves each budget past its value). + +| What | Where | Value | Past it | +|---|---|---|---| +| Proving one obligation | `PROVE_WORK`, `src/engines/pseudo/solver.ts` | 2,000,000 units | The obligation is open: "too large to prove here" | +| Following one subroutine's paths and finding its loops' invariants | `UNIT_WORK`, `src/engines/pseudo/vcgen.ts` | 40,000,000 units | The subroutine is partial, with the note "This subroutine is too large to prove here: the checker stopped before the end of it, so nothing in it is marked proved. Every run still checks it." | +| Proving one subroutine's obligations | `PROOF_WORK`, `src/engines/pseudo/vcgen.ts` | 20,000,000 units | Each obligation not yet proved is open: "too large to prove here" | +| Everything above, for a whole program | `PROGRAM_WORK`, `src/engines/pseudo/vcgen.ts` | 60,000,000 units | The subroutines after it are partial, with the same note | +| How deep a value the checker follows is (`T <- T + A[T] + ...` line after line builds one operation inside another) | `MAX_DEPTH`, `src/engines/pseudo/vcgen.ts` | 500 operations (the corpus's deepest: 18) | The value becomes one the checker knows nothing about, with the note "The value on line N is built from more than 500 operations one inside another, more than the checker follows: from there on it treats such a value as one it knows nothing about, so what depends on it stays open." The engine's own stack or size limit, past everything, is the same as running out of work: the unit is partial, or the obligation open, too large to prove here | +| A term's key, which finds equal terms (a term that shares its parts, such as nested `ABS`, is far longer written out) | `MAX_KEY`, `src/engines/pseudo/logic.ts` | 2,000 characters | The term is equal only to itself: the prover concludes less about it, never more | +| A fact or goal written out with an obligation | `SHOW_PARTS`, `src/engines/pseudo/logic.ts` | 1,000 parts | The rest is written "…" | +| The facts and goals one subroutine's report writes out | `SHOWN_CHARS`, `src/engines/pseudo/vcgen.ts` | 1,000,000 characters | Each further obligation shows its goal alone, cut at 400 characters | +| The hypotheses looked through for the facts shown with an obligation | `SHOWN_FROM`, `src/engines/pseudo/vcgen.ts` | the nearest 400 | The facts shown come from those | +| The generated inputs that cross-check a proof (`checkProperties`), counted in steps and positions visited by FORALL, EXISTS, SUM and sets written by rule | `CROSS_CHECK_ROUTINE`, `CROSS_CHECK_PROGRAM`, `src/services/pseudo.ts` | 600,000 for one routine, 1,200,000 for a program | No more inputs are tried (the routines after it, none): an obligation they would have shown broken stays open. A run cut short by the budget is not a failure | +| The sample runs that check the equations a matho pane is given give the pseudo's values | `READ_BACK_VISITS`, `src/sync/pseudo-to-matho.ts` | 100,000 quantifier positions a run | The run stops there, on both sides alike, and the check compares where they stopped | +| The fuzzer's inputs for one routine (the Tests view, `Workspace.fuzz`), counted the same way | `WORK_PER_INPUT`, `src/common/fuzz.ts` | 3,000 for each input the budget asks for: 900,000 at the default 300 | No more inputs are run, and the report's `problem` says "Stopped after N steps of work, all one routine's inputs may take here (M): the inputs listed were run, and no more." A failure found before is kept, shrunk as far as the work allowed | +| The routines of one program the fuzzer tests | `FUZZ_ROUTINES`, `src/common/workspace.ts` | the first 100 | A report with no cases: "Not tested: generated inputs cover the first 100 routines of a program, and this is routine N." | +| The fuzzer's work for a whole program | `FUZZ_PROGRAM_INPUTS`, `src/common/workspace.ts` | 600 inputs' worth: 1,800,000 at the default budget | A report with no cases: "Not tested: the routines before this one took all the work the tests of one program may take (1,800,000 steps)." | + +The wall-clock budget the Prove tab gives each subroutine (3 seconds) stays behind these as a last resort. Measured on the machine this was written on: 20 nested `ABS` (or a matho pane `f(x) = ||||...x...||||` of 20 bars) took a 1 GB heap down before, and now ends in 0.1 seconds in a 128 MB heap; a subroutine at `UNIT_WORK` takes 0.7 to 2 seconds, one at `PROOF_WORK` about 1 second, and a program at `PROGRAM_WORK` about 4 seconds, each in a 128 MB heap. The most any program of the corpus takes is 234,000 units for one obligation, 145,000 for one subroutine's paths, 783,000 for its proofs and 886,000 for the whole program. `prove-fuzz` makes larger programs than anyone writes: the most in seeds 9137 and 22051 (800 programs each) are 768,000, 35,000,000 (a subroutine its 1.5-second wall-clock budget had already cut short), 5,100,000 and 38,000,000, and the proved obligations of both seeds are the same with the budgets as without. The cross-check is counted the same way: one input of a routine whose ENSURE nests 8 quantifiers visited 5,000,000 positions (7 seconds) before, and the whole cross-check of a routine now takes at most about 2 seconds; the most any routine of the corpus takes is 347,000 (`match-off-by-one`). The fuzzer runs a routine's inputs within its budget in about a second, a program's in about 2 (a program of 2,000 small routines, the first 100 fuzzed: 2 seconds); the most the corpus takes is 400,000 for one routine (kmers' `kmer_counts` in Mojo), 1,300 for one input, and every corpus program's report is the same as before. + +## What the compiler does + +The checker may never report an error on a program the compiler accepts, so the Mojo limits are a refusal to read, not an error. Asked through `mojo build` at the pin (Mojo 1.2.0), the compiler accepts 5,000 nested parentheses, 5,000 nested calls, 1,000 nested blocks, 5,000 `elif` arms and a 10,000-term sum, and crashes (exit 139, a stack overflow in the compiler) on 30,000 nested parentheses, a 30,000-term sum, or 30,000 minus signs. There is no depth it refuses with an error message. + +## The tests + +`tests/mojo-robustness.test.mjs` proves the Mojo limits and `tests/reader-limits.test.mjs` the others, each just past its value with inputs small enough to keep the test quick, and each shape just under it still reading and running. `tests/pseudo-robustness.test.mjs` has the pseudocode's. diff --git a/docs/mojo-corpus.md b/docs/mojo-corpus.md new file mode 100644 index 0000000..7666cde --- /dev/null +++ b/docs/mojo-corpus.md @@ -0,0 +1,179 @@ +# The Mojo corpus + +Programming+ teaches what Mojo does when it runs, so everything it says about Mojo has to be true of a real compiler at a known version. The corpus is how: material harvested from Modular's own repositories at pinned commits, plus small programs of our own, every one of them checked on the real compiler through Compiler Explorer (godbolt). The pipeline is Python, standard library only, in `scripts/mojo/` (`corpus.py` is its command line); Node is needed for one processor (Mojo Quest keeps its tickets in TypeScript). + +Every command below is a pixi task; the [README](../README.md#commands) lists them all. + +The corpus itself (the pin, the harvested material, our programs and godbolt's answers) lives in pplus-library, the submodule `library/`, under `library/mojo/`; the pipeline that builds and checks it (`scripts/mojo/`) and what it generates for the app (`src/mojo/generated/`) live here. A task that changes the corpus leaves changes in both repositories: commit and push `library/` first, then commit core with the new pointer (`git add library`) and the regenerated files. + +## Mojo of our own + +There are three kinds of Mojo program we write, and each has a folder, a task to start one, and a job: + +| Kind | Folder | Start one | Its job | Shipped in the app | +|---|---|---|---|---| +| probe | `library/mojo/probes/` | `pixi run new-probe ""` | Ask the pinned compiler one question (what does `//` do on a negative Int?). Godbolt's answer is the point; it is recorded, not judged. | no, but the checker and interpreter are held to the answers | +| exercise program | `library/mojo/local/` | `pixi run new-exercise ` | The Mojo of a catalog problem. It must pass its own `# expect-stdout:` lines. | yes, in `src/mojo/generated/programs.ts`, badged verified when it passed | +| stress program | `library/mojo/stress/` | `pixi run new-stress ""` | Realistic Mojo that stresses the interpreter. Its `# expect-stdout:` lines are the interpreter's prediction; godbolt confirms or refutes it. | no | + +The loop is the same for all three: + +```bash +pixi run new-probe int-floordiv-negative "does // floor toward negative infinity on Int?" +# edit library/mojo/probes/int-floordiv-negative.mojo +pixi run mojo-verify-item 'probes/int-floordiv-negative' # ask godbolt (network) +pixi run test # the checker and interpreter scoreboards read the new answer +``` + +For an exercise program, `pixi run play ` does the whole loop in one pass (below). + +### The header + +A program starts with comment lines that `corpus.py verify` reads. Only these keys are header lines; the program starts at the first other line, and the Mojo pane hides the header. + +```mojo +# probe: what does // do on negative Int? the question, or what the program shows (one line) +# stdin: 3\n4\n what input() reads, with \n escapes +# mode: run run | asm | both (asm and both also return the listing) +# args: -O0 compiler arguments, after the pin's build arguments (below) +# expect-stdout: -4 a line the program must print (repeatable); implies expect: pass +# expect: compile-error pass | fail | compile-error | observe (the default without expect-stdout) +# origin: where the program came from free text, for people +# status: a note on its state free text, for people +``` + +**A stored answer belongs to the exact text.** Godbolt's answer is keyed by the compiler, mode, arguments, stdin and the whole source, header included. Any edit, even to a comment, makes the program unverified until it is asked again. That is deliberate: a stale answer is never shown against new code. + +### How every program is built + +Mojo is built one stated way everywhere core builds a program or asks godbolt to: `corpus.py verify`, `library-verify-mojo`, the Compile button (`BUILD_ARGS` in `src/services/godbolt.ts`: the pin's arguments, then the level's), `library-check`, `mojo-local`, `mojo-fuzz`, `mojo-run` and `play`. The arguments are `build.args` in `library/mojo/pin.json`, each with its reason under `build.why`: + +- `--target-cpu x86-64-v3`. Without it `mojo build` targets the processor it runs on, and godbolt's machines differ: on 2026-11-12 every stored listing was AVX2 (`ymm`), and at 12:55 EST the same request for `local/demo` came back with AVX-512 (`zmm`, `kmovq`, `vpternlogd`), instructions the model machine does not know. x86-64-v3 is AVX2, FMA, BMI1 and BMI2, with no AVX-512, so a listing is the same whichever machine answers (`probes/godbolt-target-cpu`). A machine that is not x86-64 (an Apple silicon Mac) cannot build for it, so there the local tools leave it out. +- `--fp-mode contract=off`. Mojo's default, `contract=fast`, fuses a multiply with an add or subtract into one rounding wherever the operands arrive at run time: `probes/float-contract-runtime` (built with `contract=fast` on purpose) prints -0.22222222222222224 for `x * x - x` when `x` comes from stdin and -0.2222222222222222 when it is a literal the compiler works out while building. With contraction off every Float64 operation is rounded on its own, as IEEE 754 says and as the pseudo interpreter and core's Mojo interpreter compute (`probes/float-contract-runtime-off` prints -0.2222222222222222 twice). + +A program's own `# args:` come after these, and an option it names replaces the pin's (the contraction probes name `--fp-mode` themselves). Every stored answer records the arguments it was built with (`args`) and is keyed by them, so a change to `build.args` leaves every answer unverified until `pixi run mojo-verify` and `library-verify-mojo` ask again. `tests/build-args.test.mjs` fails when a builder writes a code-generation flag of its own or the Python and TypeScript halves combine the arguments differently. + +`mojo build` or `mojo run` on your own machine, without these arguments, targets your processor and fuses: its last digits and its listing can differ from what Programming+ shows. `pixi run mojo-run ` builds the way everything else does. + +### The fast loop for an exercise program + +```bash +pixi run play rolling-sum # one pass +pixi run play-watch rolling-sum # again on every save (tests off) +``` + +One pass is: run the file on the local `mojo` (pixi provides it); `corpus.py verify --item local/` (godbolt, network); `corpus.py status` (offline, rewrites `programs.ts`); `tsc`; `scripts/mojo/listings.mjs`; the tests. A step that cannot run (no local compiler, godbolt unreachable) is reported as skipped and the pass goes on, so offline you still get the build and the tests, with the program shown as unverified rather than guessed at. `node scripts/play.mjs --no-verify` or `--no-test` skips a step. + +### The assembly + +A program whose header says `# mode: both` (or `asm`) and `# args: -O0 --debug-level=line-tables` (after the pin's build arguments) makes godbolt return the compiler's listing with its answer. `scripts/mojo/listings.mjs` picks the first `def` that is not `main`, translates it and the user functions it calls to the x86-64 pane's syntax exactly as the **Send to x86-64 pane** button does (`src/mojo/att.ts`, `simulatorSetup` for the argument registers), tags each instruction `; mojo line N` with the line of the program the pane shows, and writes `src/mojo/generated/listings.ts`. An exercise that names the program in `mojoProgram` and leaves `sources.asm` empty gets that listing and its setup. No answer for the exact source, mode and arguments (the pin's and the program's) means no listing; a stale answer is never reused, because the key includes the source. Without `-O0` the optimizer may inline every function and the listing holds none of yours; the script says so. + +Header lines (`# probe:`, `# mode:`, …) are for `verify`; only those keys are stripped from what the Mojo pane shows. The first line after the header block is the program. + +## The compiler on your machine + +Godbolt's stored answers cover the programs someone wrote down, and they need the network. pixi also installs the pinned compiler, and two tasks hold core to it directly, offline: + +```bash +pixi run mojo-local # every Mojo program core holds or generates +pixi run mojo-fuzz --seed 7 --programs 50 # programs nobody wrote +pixi run mojo-fuzz --statements --seed 7 # whole programs of statements nobody wrote +``` + +`mojo-local` (`scripts/mojo/local.mjs`) builds and runs the probes, exercise programs and stress programs as written, the Mojo the translator writes for every exercise, every pseudo regression program and every library routine on its cases, and every Mojo reference of the library. For each it asks: does the checker stay silent on what the compiler accepts; does the interpreter print what the built program prints and end the same way, or decline; does the compiler accept generated Mojo, and does it print what the pseudo prints; does this compiler agree with godbolt's stored answer for the same text, arguments and stdin. Every "no" is printed, and the task fails. The differences that are right are listed in `EXPECTED` at the top of the script, each with its reason; a listed one that stops differing fails the pass too, so the list only shrinks. Answers are cached in `.tools/mojo-local/` by compiler version, arguments, stdin and source. + +`mojo-fuzz` (`scripts/mojo/fuzz.mjs`) writes typed pseudo expressions at random from the notation's operators and built-ins, one per OUTPUT line, drops the lines the pseudo itself stops on or refuses, translates the rest, builds them, and holds each printed line to the pseudo's. It reports a line the compiler rejects, a line where the built program stops, a line that prints another value, a line where the interpreter prints something else than the built program, and a checker error on an accepted program, once per shape with the smallest expression that shows it. The same seed gives the same programs. Run it with a few seeds after changing `src/sync/pseudo-to-mojo.ts` or `src/mojo/`; when it finds something, fix the rule and pin the spelling in `tests/sync-mojo.test.mjs`. + +With `--statements` it writes whole programs instead (20 to 60 lines, 100 per seed by default; the generator is `scripts/mojo/fuzz-programs.mjs`): records, constants, FUNCTIONs and PROCEDUREs with by-value and BYREF parameters of scalars and collections, some recursive with a bound, then the main lines, built from DECLARE of every scalar, sized, collection and record type, assignment to names, elements, fields and map entries, IF and CASE, FOR with positive and negative steps, FOR EACH over lists, arrays, text, ranges, maps and sets, WHILE, REPEAT, BREAK and CONTINUE, the collection built-ins, copies changed after they are made, and a tagged OUTPUT at many points, so the first line that differs says where. The generator is typed, so most programs check as written; the pseudo's checker and a run are the filter, and a line the checker refuses, a run stops on, or the translator says it has no Mojo for is left out and the rest tried again. A program whose output changes when SET members and map keys are visited in another order is dropped, since the notation never promised an order. The report says how many programs settled, which constructs they held, every finding once per shape with its smallest program, shrunk by leaving out statements one at a time while the built program still shows the same finding (`--shrink N` builds at most, 80 by default), and the pseudo engine's own faults apart. `--progress` prints each finding as it is found; `--dry` only generates and settles, to measure the generator. + +Two kinds of difference are counted and not reported as failures, because the compiler decides them ([trust.md](trust.md#what-a-translation-says)): numbers that differ only in their last digits (the compiled program's `std.math` is its own; its arithmetic is the pseudo's, since every program is built with contraction off), and, for lines that use `**`, `exp` or `log2`, what those lose in Mojo 1.2.0 (they are good to 9 to 13 digits). What the compiler does wrong on its own is listed in `UPSTREAM` in the fuzzer, with the probe that records it; read that list on every version bump. + +Neither task needs or replaces godbolt. A built program that traps is run with its core-dump limit set to 1, so a deliberate crash does not reach the desktop's crash handler. CI runs both in the pinned environment, which brings the compiler: `pixi run ci` runs `mojo-local` and a short `mojo-fuzz`, and the nightly run fuzzes longer ([ci.md](ci.md)). Run both yourself after changing the translator, the checker or the interpreter, and after a version bump. + +## Reading the status + +`pixi run mojo-status` judges every item from the stored answers, offline, and writes `library/mojo/golden/STATUS.md` (a table per corpus, every failure with its reason, and every probe's question and verdict) and `status.json` (`git -C library diff mojo/golden/status.json` shows exactly what changed). + +Each item says what the pinned compiler should do with it: `pass` (compiles, runs, exits 0, and prints what it should or contains the idiom it should), `fail` (that check must not hold: a starter), `compile-error` (must be rejected, with each expected error at its line), `observe` (a probe: the answer is the point), or `skip` (with a reason: newer syntax than the pin, no `main`, not reproducible on public godbolt). Verdicts are `pass`, `fail`, `observed`, `skipped`, and `unverified` (no golden yet). + +The dialect markers in the pin are a cheap first look: `newer` patterns are syntax that arrived after the pinned version (a manual sample using one is skipped), `older` patterns are syntax the pinned version rejects. They only set expectations; verify decides. + +**What has to pass.** `pixi run mojo-status-strict` (and CI) fails only when a program we wrote fails: a probe, an exercise program, or a stress program. Harvested items can fail for reasons that are facts about upstream (a manual sample written for a newer compiler, a test file with no `main`); they are listed in `STATUS.md` to read, not to fix here. `corpus.py status --strict --strict-corpus all` holds every corpus to it. + +## What lives where + +| Path | What it is | Who writes it | +|---|---|---| +| `library/mojo/pin.json` | The Mojo version, the godbolt compiler id, the arguments every program is built with (`build`), each source repository and the ref to use, and the dialect markers | You | +| `library/mojo/mojo.lock.json` | What each ref resolved to: commit, zip sha256, and the directory inside the zip | `fetch`/`harvest` | +| `library/mojo/corpus/` | Processed material, one directory per processor, each item with its provenance | `harvest` | +| `library/mojo/corpus/SOURCES.md` | Which commit and license everything came from, what each processor found, every warning | `harvest` | +| `library/mojo/licenses/` | Each source's license file, copied from the commit the pin's `licenseCheck` names | You | +| `library/THIRD_PARTY.md` | Every harvested file with its source and the license header it carries, and what each license check found | `emit` (and `harvest`, `verify`) | +| `library/mojo/probes/` | Our questions about the pinned compiler's behavior, one small program each | You (`new-probe`) | +| `library/mojo/local/` | Exercise programs, shipped in the app | You (`new-exercise`) | +| `library/mojo/stress/` | Realistic programs that test the interpreter, not shipped | You (`new-stress`) | +| `library/mojo/golden/.json` | Godbolt's answer for one program, keyed by compiler, mode, arguments (the pin's, then the program's), stdin and source | `verify`; `prune` removes the ones nothing reads | +| `library/mojo/golden/STATUS.md` | Every item's verdict against the goldens | `status` (also `harvest`, `verify`) | +| `src/mojo/generated/` | What the app reads at run time: `pin.ts` (version, compiler, build arguments), `tokens.ts` (the words, removed words and the compiler's messages for them), `editor.ts` (comment and indentation rules), `programs.ts` (`library/mojo/local/` programs with their verdicts), `stdlib.ts` and `types.ts` (which package exports each name, which types are copyable), and `listings.ts` (exercise listings, written by `scripts/mojo/listings.mjs` during `play`) | `harvest`/`emit`; `verify` and `status` refresh `programs.ts` | +| `library/syntax/mojo.xml` | The same words as a KDE syntax for Pandoc listings (pplus-web copies it into its Quarto extension with its quarto-css task) | `harvest`/`emit` | + +Commit all of it. The stored answers are what make `pixi run test` and `mojo-status` work offline. + +## Commands in detail + +The tasks wrap `corpus.py`, which takes more options than the tasks pass: + +- `harvest --local NAME=ZIP` uses a zip on disk for one source. `--local-dir DIR` matches every zip in a directory to a source by its marker files, but only for sources whose ref is still `UNRESOLVED`; add `--offline` to let zips stand in for pinned refs too (the lock then says `standsInFor`, and the next online run fetches the real ref). +- `harvest --processors quest,tokens` reruns some processors; `--keep` or `--workdir DIR` keeps the extracted sources for inspection. +- `verify --item 'probes/life-*' --force` asks again about some items (`pixi run mojo-verify-item` does this); `--dry-run` shows the request and the count and sends nothing; `--corpus diagnostics` adds the compiler's own tests (opt-in: about 140 files); `--limit N` asks about at most N. +- `status --strict` exits 1 if a program we wrote fails; `--strict-corpus` picks other corpora. +- `GITHUB_TOKEN` (any token with public read access) avoids GitHub's anonymous rate limit when resolving refs. + +## The processors + +| Processor | From | Produces | +|---|---|---| +| `quest` | mojo-quest | Each ticket's solution (must pass Mojo Quest's own check) and starter (must not) | +| `manual` | modular | The manual's tested code samples (must run and exit 0) and every `mojo` block that is a whole program followed by an `output` block (must print it) | +| `diagnostics` | modular | The compiler's `expected-error/warning/note` tests, each annotation resolved to its line and tagged with the pass that reports it (`parser`, `lifetimes`) | +| `tokens` | modular | Keywords (`TokenKinds.def`), soft keywords and removed words (the parser), decorators (the reference), prelude names (the stdlib) | +| `messages` | modular | The parser's and borrow checker's diagnostic wording and fix-its, read from their C++ | +| `release` | modular | The pinned release notes, section by section, and whether the snapshot is the pin, newer, or older | +| `syntax` | mojo-syntax | The TextMate grammar's tests as plain highlighting cases | +| `editor` | vscode-mojo | Comments, brackets, auto-closing pairs and indentation rules | +| `skill` | skills | Modular's table of removed syntax and its replacements | +| `stdindex` | modular | Which std package exports each public name (`sqrt` in `std.math`), for the compiler's import hint; a name exported from two places is left out; and which prelude functions raise (`input`, `atol`) | +| `stdtypes` | modular | Each std struct's declared traits (so which are implicitly copyable) and each trait's default methods (Copyable's `copy`, Equatable's `__eq__`), for the type checks | + +A processor whose input files are missing at the pinned ref is skipped with a warning, and its previous output is kept. + +## Moving to a new Mojo version + +1. Find the release. Mojo releases are tagged `mojo/vX.Y.Z` in `modular/modular` (the platform's own tags are `modular/vX.Y`). Find godbolt's compiler id for it, `mojo_X_Y_Z`, in `https://godbolt.org/api/compilers/mojo`, and check that pixi's channel has it (`pixi search -c https://conda.modular.com/max/ mojo`). +2. Edit `library/mojo/pin.json`: `mojo.version`, `godbolt.compiler`, and `sources.modular.ref`; and `mojo = "==X.Y.Z"` in `pixi.toml` (pplus-web's too), then `pixi install`. A test holds the three names in the pin to one release, and `pixi.toml` and the documents to the pin. Move the other refs to the commits that match: Mojo Quest's commit whose `src/lib/compile.ts` names the new compiler id; the vscode-mojo, mojo-syntax and skills commits released alongside it. +3. Review the dialect markers against the new release notes. Anything in the notes' **Removed** or **Language changes** sections belongs in `older`; anything in `newer` that this release now includes comes out. (`newer` matters only when harvesting a snapshot past a release; at a release tag it is empty, and a test says so.) +4. `pixi run mojo-harvest`, then read `library/mojo/corpus/SOURCES.md`: every warning is something to look at, and a `release` line saying `snapshot newer` means the ref is not the release tag. A release can be tagged before its notes leave the nightly changelog (1.2.0 was: there is no `v1.2.0.md` at `mojo/v1.2.0`); the `release` processor then reads the changelog as the release's notes and says `notesFrom nightly-changelog`. +5. `pixi run mojo-verify`. A new compiler id changes every key, so every item is asked again, one request each and two at a time (`pixi run mojo-verify-dry` prints how many). Run it once, not in a loop; the public server is shared. +6. `pixi run mojo-status`, then read `library/mojo/golden/STATUS.md`. `git -C library diff mojo/golden/status.json` shows exactly which verdicts and probe answers changed between versions. A changed probe answer is a change Programming+'s Mojo engine has to make; a failing Mojo Quest solution or manual sample usually means a language change the markers should know about. The compiler's wording changes too: the checker quotes it, and `tests/mojo-check.test.mjs` fails where a quoted message moved (1.2.0 names a `TString` by where its values live, not by their types). +7. Check each moved source's license at its new commit: read the repository's license files and the headers of what was harvested, copy the license file into `library/mojo/licenses/`, and update that source's `licenseCheck` in the pin (commit, date, file, sha256, what it says). `pixi run mojo-emit` rewrites `library/THIRD_PARTY.md`; `tests/third-party.test.mjs` fails until every source was checked at the commit the lock holds. +8. `pixi run mojo-local`, then `pixi run mojo-fuzz` with two or three seeds: the new compiler on every program core holds or generates. Then `pixi run library-verify-mojo` and `pixi run library-check` for the library's Mojo references. +9. `pixi run mojo-prune`: the old pin's answers are keyed by its compiler id, so nothing reads them any more (git keeps them). +10. `pixi run test`, then commit the pin, the lock, the corpus, the goldens and the library's records in pplus-library, and the new pointer, `pixi.toml`, `pixi.lock`, `src/mojo/generated/` and `src/library/generated.ts` here. + +## Licenses and provenance + +Every item records its repository, commit (or local zip hash), path, and license; `SOURCES.md` summarizes them, and `library/THIRD_PARTY.md` counts every harvested file by the license header it carries and lists the ones with none (the manual's snippets, cut from pages with no header, and Mojo Quest's starters). Modular's repository files carry Apache-2.0 WITH LLVM-exception headers, and the repository's own LICENSE covers the rest, the docs included. Mojo Quest's solutions carry the same headers, but the repository has no LICENSE file, and its starters come from a module with no header, so its programs are used as test fixtures here; confirm with Modular before showing them to students as exercises. mojo-syntax is MIT, vscode-mojo is Apache-2.0. Manual prose is not copied into the app; hover text and quick fixes will be written in our own words and link to mojolang.org. + +## When something goes wrong + +| Message | Meaning | +|---|---| +| `a network proxy refused api.github.com` | No route to GitHub from here. Use `--local-dir DIR --offline`, or run elsewhere. | +| `GitHub refused the request (403)` | Usually the anonymous rate limit. Set `GITHUB_TOKEN`. | +| `GitHub has no ref ...` | The tag or commit in the pin does not exist. | +| `the lock was made from a local zip` | The pin has not moved since a local harvest. Pass the same zip with `--local`, or set a ref. | +| `GitHub's zip ... hashes to ...` | The archive for a locked commit changed. Rare; accept it with `--allow-hash-change` after a look. | +| `no directory in the zip holds all of [...]` | The repository's layout changed. Update that source's `markers` in the pin. | +| `: skipped, ... has no ...` | That input is missing at this ref; the previous output is kept. | +| `the quest processor needs Node 22.6+` | Node strips the TypeScript types from Mojo Quest's ticket file. | diff --git a/docs/sync-coverage.md b/docs/sync-coverage.md new file mode 100644 index 0000000..bbd0d42 --- /dev/null +++ b/docs/sync-coverage.md @@ -0,0 +1,117 @@ +# Translation coverage + + + +Each translator in `src/sync/` is run over every exercise with pseudo (17), every program in `library/pseudo/programs/` (55), and 17 one-construct fixtures (`tests/helpers/sync-programs.mjs`). 3 of the 89 have pseudo that does not read, by design (they teach the notation's error messages), and are left out below. That leaves **86 programs**. + +| From pseudo to | Says it completely | Says it with a gap or a checker error | Cannot say it | +| --- | ---: | ---: | ---: | +| Mojo | 83 | 3 | 0 | +| lingo | 31 | 0 | 55 | +| matho | 3 | 0 | 83 | + +Mojo is a program-for-program translation that always produces something: a line with no faithful spelling becomes a visible `raise Error("not translated: pseudo line N")` and a warning. Lingo and matho must read back as the same program, so what they cannot say exactly they refuse, with the construct and its line, and the pane keeps its last good text. + +## Mojo: lines that become a visible gap + +These programs translate, with the listed lines left as a `raise Error`. + +| What it has no form for | Programs | Which | +| --- | ---: | --- | +| Programming+ could not translate this line into Mojo (the built-in EXACT, which Mojo's standard library has no function for). The generated code stops here with an error instead of guessing. | 1 | 50-float32.pseudo | + +## Mojo: programs the translator refuses + +Pseudo the course itself rejects is not translated. + +Nothing. + +## lingo: what English has no sentence for + +English is read by `readSteps`, whose grammar has no sentence for these. + +| What it has no form for | Programs | Which | +| --- | ---: | --- | +| steps outside a routine (every English step belongs to a routine) | 53 | 01-arithmetic.pseudo, 02-temperature.pseudo, 03-compound-interest.pseudo, 04-gcd-recursion.pseudo, and 49 more | +| a BYREF parameter | 4 | 06-fibonacci-memo.pseudo, 10-insertion-sort.pseudo, 26-swap-byref.pseudo, 55-statements.pseudo | +| a fixed-length ARRAY | 4 | ex:edit-distance, 05-primes-sieve.pseudo, 11-matrix-multiply.pseudo, 15-game-of-life.pseudo | +| a HASHMAP whose values are not counts | 3 | ex:translate-codons, 13-stack-brackets.pseudo, 39-constants-codons.pseudo | +| a change to this part of a value | 2 | ex:edit-distance, 16-running-stats.pseudo | +| a DEFINE set | 2 | ex:translate-codons, 21-sets.pseudo | +| a loop step that is not a whole number written out | 2 | 05-primes-sieve.pseudo, 39-constants-codons.pseudo | +| a record | 2 | 12-records.pseudo, 16-running-stats.pseudo | +| a record type | 2 | 12-records.pseudo, 16-running-stats.pseudo | +| REMOVE(...) | 2 | 13-stack-brackets.pseudo, 14-queue-bfs.pseudo | +| the built-in ROWS | 2 | 11-matrix-multiply.pseudo, 15-game-of-life.pseudo | +| this kind of position | 2 | ex:edit-distance, 15-game-of-life.pseudo | +| a CONSTANT | 1 | 39-constants-codons.pseudo | +| a Field value | 1 | 12-records.pseudo | +| PARALLEL FOR | 1 | 43-parallel-map.pseudo | +| REPEAT ... UNTIL, which English reads as a loop that tests first | 1 | 46-projectile.pseudo | +| the built-in CHOOSE | 1 | 47-binomial-and-ph.pseudo | +| the built-in CHR | 1 | 08-caesar.pseudo | +| the built-in COS | 1 | 46-projectile.pseudo | +| the built-in FILL | 1 | 43-parallel-map.pseudo | +| the built-in LOG10 | 1 | 47-binomial-and-ph.pseudo | +| the built-in SIN | 1 | 46-projectile.pseudo | +| the built-in SPLIT | 1 | 40-text-builtins.pseudo | +| the built-in TO_INTEGER | 1 | 22-binary-to-decimal.pseudo | +| the built-in TO_STRING | 1 | 22-binary-to-decimal.pseudo | + +## matho: what is not an equation + +An equation lowers to one `FUNCTION` that returns a value; a program with output, text, a loop that does more than add up, or steps outside a function is not one. + +| What it has no form for | Programs | Which | +| --- | ---: | --- | +| steps outside a function (an equation is a function) | 49 | 01-arithmetic.pseudo, 02-temperature.pseudo, 03-compound-interest.pseudo, 04-gcd-recursion.pseudo, and 45 more | +| a function that answers LIST (equations answer numbers) | 5 | ex:match, ex:match-proved, ex:match-off-by-one, ex:match-reset, and 1 more | +| a function that answers STRING (equations answer numbers) | 5 | ex:revcomp, fixture:string-build, fixture:slice, fixture:case, and 1 more | +| a function that answers BOOL (equations answer numbers) | 3 | ex:binary-search, fixture:early-return, fixture:forall | +| a loop that is not adding up or multiplying one value (FOR) | 3 | ex:best-position, fixture:loop-down, fixture:break-continue | +| the input T, which is not a number or a list of numbers | 3 | ex:first-unknown, ex:gc-fraction, fixture:set | +| a DEFINE set | 2 | ex:translate-codons, 21-sets.pseudo | +| a function that answers HASHMAP (equations answer numbers) | 2 | ex:tally, ex:kmers | +| a record type | 2 | 12-records.pseudo, 16-running-stats.pseudo | +| the input A, which is not a number or a list of numbers | 2 | ex:hamming, ex:edit-distance | +| the input B, which is not a number or a list of numbers | 2 | ex:count-base, ex:count-base-proved | +| a CONSTANT | 1 | 39-constants-codons.pseudo | +| a loop with a STEP | 1 | fixture:loop-step | +| a position that is not a loop counter plus or minus a number | 1 | fixture:min-max | +| a PROCEDURE | 1 | fixture:procedure | +| a WHILE loop | 1 | fixture:div-mod | + +## matho: the equation sets + +Each set in `tests/matho.test.mjs` is lowered to pseudo by the matho reader, written back as equations by `pseudoToMatho`, and checked: 26 of 28 round-trip. + +| Equation set | Round-trips | Why not | +| --- | :---: | --- | +| mean-and-variance | yes | | +| mean | yes | | +| sum-from-one | yes | | +| sum-no-bounds | yes | | +| binomial | yes | | +| height | yes | | +| kinetic | yes | | +| gravity | yes | | +| omega | yes | | +| normal-density | yes | | +| ph | yes | | +| log2 | yes | | +| ln-and-log | yes | | +| factorial | yes | | +| choose | no | the loop counter K2 (matho writes it as a single letter) | +| cosine | yes | | +| distance | yes | | +| matrix-row | no | a two-dimensional position that is not two loop counters | +| nested-sums | yes | | +| relu | yes | | +| piecewise | yes | | +| calls-an-earlier-equation | yes | | +| sum-over-a-set | yes | | +| floor | yes | | +| ceiling | yes | | +| absolute | yes | | +| max-min | yes | | +| mod | yes | | diff --git a/docs/trust.md b/docs/trust.md new file mode 100644 index 0000000..2547ccf --- /dev/null +++ b/docs/trust.md @@ -0,0 +1,79 @@ +# What each badge and verdict means + +Programming+ takes a procedure from an idea to Mojo to the machine. The idea is written in lingo (steps in plain English), matho (equations written in LaTeX), pseudo (pseudocode precise enough to run, prove and translate) or speco (a specification to prove against); then comes Mojo (Modular's programming language, pinned to version 1.2.0), then x86-64 on a model machine. pplus-web shows the badges; the verdicts behind them come from pplus-core (the prover, the fuzzer, the translators, the compiler's stored and live answers, the model machine). This page is kept here, beside what produces them. + +Programming+ is built for checking work that someone or something else produced, so every claim it shows names what answered it, and none claims more than that. This page lists each one: who earns it, what it means, and what it does not mean. When a new badge or verdict is added, it gets a row here in the same commit. The texts below are pplus-web's (`src/ui/shell.ts`, `src/ui/provenance.ts`, `src/ui/home.ts`, `src/ui/library.ts`, `src/ui/views/`); pplus-web's `tests/trust.test.mjs` fails when a badge the site draws has no row here. + +## Where code came from + +| Shown | Earned by | Means | Does not mean | +|---|---|---|---| +| **Generated from pseudo** | The Bridge's Mojo tab, always (`mojoProvenance` in pplus-web's `src/core/provenance.ts`); Mojo copied from it with **Edit in Developer** | The translator wrote this Mojo from the pseudo, literally, with a header comment saying so | That anyone has read it, or that it is the Mojo a person would write | +| **Generated from pseudo, with N best-practice rewrites** | The Best practices tab's rewrite of generated Mojo | The best-practice rules rewrote N places of the generated Mojo | That the rewrite was compiled. Copy it into the Developer area and Compile & run to ask | +| **Started from generated Mojo** | Generated Mojo edited in the Developer area (`mojoOrigin: 'generated-edited'`) | A person has changed generated Mojo | That the person reviewed the parts they did not change | +| (badge cleared) | **I've reviewed this**, in the Developer area only | The person says they reviewed it; the header comment goes too | That anything checked the review. Compiling and editing never clear the badge | + +## What a compiler said + +| Shown | Earned by | Means | Does not mean | +|---|---|---|---| +| **Compiled on godbolt** (ran, exit N / compiled, did not run / did not compile / timed out) | An answer from the compile backend (`src/services/compile-backend.ts`, godbolt.org with Mojo 1.2.0 by default) for this exact source, input and build level, made in this browser | The real compiler answered as shown | That the program is right. The badge is keyed by a hash, so any edit, other input or other level removes it, and a share link never carries it | +| **Edited since compile** | An edit after a compile | The compiler's answer is for an earlier text | Anything about the current text | +| The Compile & run tab's own badge: **compiling**, **ran, exit 0**, **exited with N**, **compiled, did not run**, **did not compile**, **timed out**, **no answer** | The last request from this browser to the compile backend, and whether it is still waiting | What the backend answered for the source it was sent, built the one way Programming+ builds Mojo (`--target-cpu x86-64-v3 --fp-mode contract=off`, then `-O0` and line tables at the default level: [mojo-corpus.md](mojo-corpus.md#how-every-program-is-built)), so the listing is the same whichever of godbolt's machines answers; when the pane has changed since, the tab says so | That the current pane compiles, or anything when there was no answer (a network failure, for one). That `mojo build` on your own machine gives the same listing or the same last digits: with no arguments it targets your processor and fuses `a * b + c` into one rounding | +| **verified on Mojo 1.2.0** | An exercise's own program in `library/mojo/local/`, shown unchanged, when godbolt's stored answer for that exact source passed the file's own expectation: its `# expect-stdout:` lines, or `# expect: fail` for a starter whose bug the student is to find (`pixi run mojo-verify`) | The program did what its file says it should, on the real compiler | That the program is right on other inputs, or that a starter is right at all | +| **not verified** | An exercise's own program, shown unchanged, with no passing stored answer for that exact source | Not yet asked, or the answer did not pass; the badge's tooltip says which | That it fails | +| **your edits** | An exercise's own program after the student changed it | The text is no longer the one that was verified | Anything about the new text; Compile & run asks about it | +| **Mojo 1.2.0** (no other word) | A Mojo pane whose text is neither an exercise's own program nor generated | The pinned version the checker, the interpreter and Compile & run use | Anything about this program | +| The home page's **verified** chip on a problem | The problem's own Mojo program has the **verified on Mojo 1.2.0** badge | As that badge | As that badge | +| The home page's **your edits saved** chip | A saved version of the problem in this browser | Opening it brings back the saved text | Anything about that text | +| A Mojo checker error, warning or note | Programming+'s own checker (`src/mojo/`) | The compiler would report this. The checker may never report an error on a program the compiler accepts: `tests/mojo-check.test.mjs` holds it to that on every program the corpus holds with a stored answer, and `pixi run mojo-local` and `mojo-fuzz` on the programs they build | That a program without checker errors compiles. Only the compiler says that | +| A **best practices** note | The optional helper (`src/mojo/practices.ts`), off by default | A more idiomatic way to write a line, whose rewrite keeps the output the same where it offers one | An error. Notes never look like the compiler | +| Mojo output in the Run view and stepper | Programming+'s Mojo interpreter (`src/mojo/interp.ts`) | For a program the compiler builds, what it prints, or the interpreter declines and says why. One listed exception: `probes/struct-field-loop-copy`, which Mojo 1.2.0 builds into a program that crashes (it frees a struct's List field twice); the interpreter runs it as written, and the test holds that difference by name. `tests/mojo-run.test.mjs` holds it to that on every program godbolt ran, from the stored answers; `pixi run mojo-local` and `mojo-fuzz` hold it to the pinned compiler on a developer's machine | That the program compiles. The interpreter can run a program the compiler rejects (`print(Int(3.9))` prints `3` here; Mojo 1.2.0 refuses to build it), so Compile & run is the word on whether it compiles. On a program outside the corpus, agreement is the rule the tests hold it to, not a proof. Its FLOAT arithmetic is the compiler's as Programming+ builds it: every build passes `--fp-mode contract=off` (`library/mojo/pin.json`), so `a * b + c` is rounded twice, as the interpreter rounds it (`probes/float-contract-runtime-off`). Three things it does not promise. The last digits of a program built another way: with Mojo's default, `--fp-mode contract=fast`, the built program fuses a multiply with an add or subtract into one rounding where the operands arrive at run time (`probes/float-contract-runtime` is built that way on purpose, and `pixi run mojo-local` lists it as the interpreter's one difference of this kind), so `mojo build` on your own machine can differ in a last digit. A float raised to a power that is not a whole number, and std.math's transcendental functions (`sin`, `cos`, `exp`, `log`, `cbrt` and the others), which Mojo computes with routines of its own or the C library's, so the interpreter declines them (a whole power it computes as Mojo does, by squaring; `fma`, `hypot`, `nextafter`, `ulp`, `copysign`, `sqrt`, `gcd`, `lcm` and `comb` it computes exactly as Mojo does, each held to a probe), as it declines random numbers, whose values come from Mojo's own generator. And input: the interpreter hands `input()` one line per call, as a terminal does; in Mojo 1.1.0 and 1.2.0 each `input()` opens its own buffered reader, so when the input is a pipe (Compile & run, a script) the first call takes every line and the second raises `EOF` | + +## What a proof says + +The Prove tab reports every obligation in three statuses (`src/engines/pseudo/vcgen.ts`, `src/services/pseudo.ts`). + +| Shown | Earned by | Means | Does not mean | +|---|---|---|---| +| **Proved** | The built-in prover (`solver.ts`), on every path it examined, in a unit it examined completely | The claim follows from what is known, in the notation's number model: INTEGER as whole numbers of any size, the sized types (INT8 … UINT64) exactly, with an `overflow` obligation wherever a value could leave its range, FLOAT and FLOAT32 as finite numbers whose rounding is not seen into. The prover is meant to be sound: `tests/prover-soundness.test.mjs` holds it to every known hole (the regression programs in `library/pseudo/prover/`), to a canary, and to every corpus program run and fuzzed; an audit in November 2026 closed 18 rules that proved what a run breaks. In a program that says `USE IEEE` nothing is marked proved | That the Mojo made from the program has the same property. A 64-bit `Int` wraps and a `Float64` becomes `inf` or `nan`; the `int-model` and `float-model` notes say when a proof leans on the number model. That the prover has no hole left: a "proved" that a run breaks is a soundness bug ([engines.md](engines.md#soundness)) | +| **Broken** | A generated input (the fuzzer, seeded) or a run that breaks the claim at that place | There is an input for which the claim is false, and **Step through** shows it | Anything about other claims. A run outranks the prover: a proved obligation that an input breaks is shown as broken (and would be a soundness bug) | +| **Open** | Neither of the above | The checker could not finish the argument, usually because a loop needs an INVARIANT | That the claim is false | +| **partial: not every path was examined**, on a unit | The prover left some paths of the unit unexamined (more paths than its limit, or past its time limit) | Nothing in the unit is marked proved: what it would have proved shows as open | That anything in it is wrong | +| An invariant "the checker found on its own" | Inference in `vcgen.ts`, kept only at a fixed point | It holds on entry and survives every pass, and the proof used it | That it is the invariant a reader needs | + +## What testing says + +| Shown | Earned by | Means | Does not mean | +|---|---|---|---| +| A routine's cases in the Tests tab, under **no failures**, **N ways of failing** or **not tested** | The fuzzer (`src/common/fuzz.ts`), with the seed and input count shown; **not tested** when it cannot generate inputs for the routine, with the reason | Every input listed was run and checked against each claim and limit | That untried inputs pass. "Held on 300 inputs" is evidence, not proof | +| A failure's shrunk input | Shrinking in the fuzzer | The smallest input it found that fails the same way | The smallest input that fails, or the only one | + +## What a translation says + +| Shown | Earned by | Means | Does not mean | +|---|---|---|---| +| ⇄ in step (lingo, matho, speco) | The translator, which reads its own output back and holds it to the pseudo (`tests/sync-lingo.test.mjs`, `sync-matho.test.mjs`, `sync-speco.test.mjs`) | The two panes describe the same program, as far as the round trip checks | A proof that they are equivalent on every input | +| ⚠ not in step | A language that cannot say the program, or a pane that does not read | The pane keeps its last text and says why | That the pane is wrong | +| Generated Mojo agreeing with the pseudo | `tests/sync-mojo.test.mjs`, over every exercise and corpus program, in core's interpreter; and `pixi run mojo-local` and `mojo-fuzz`, on the pinned compiler itself, over those programs, the library's routines, and generated ones | On those programs, the Mojo compiles and prints what the pseudo prints: in the tests by core's interpreter, and by the compiler only on a machine with the pinned Mojo (`mojo-local` and `mojo-fuzz`) | That the translator is verified. Where a line has no faithful Mojo, the translation stops with `not translated` rather than guess. Numbers agree to their last digits or nearly: the compiler's `std.math` is its own (in Mojo 1.2.0, `**` to a fractional power is good to about 13 digits and to a whole power, done by repeated multiplication, to the last digit or two; `exp` and `cosh` are good to about 13 digits near 1 and about 10 at 100 (`probes/float-cosh-accuracy`); `log2` to about 9; a whole exponent near 1e10 stops the built program, and `gcd` of an even number and a much larger odd one does not come back), while its arithmetic is the pseudo's, each FLOAT operation rounded on its own, since every program is built with contraction off; the translation carries a note where it writes `**` or `log2`. Mojo writes some FLOATs as other text (6.02e+23 for 6.02e23), so a FLOAT made into text is translated where it is printed and is `not translated` where the program uses the text itself | + +## What the machine says + +| Shown | Earned by | Means | Does not mean | +|---|---|---|---| +| Registers, caches, pipeline and counters in the Hardware area | The model machine (`src/engines/asm86/`), on the preset machine named in Setup | What that model does with these instructions | A measurement of a real processor. Cycle counts are a teaching model; compare a run with itself across edits | +| "stopped: … standard library, not on the model machine" | A call into a function the model does not have | The real program would enter the library here | That the library call does anything in particular | + +## Library routines + +A routine of pplus-library (`USE md`) carries the record of `pixi run library-check` ([library.md](library.md)). + +| Shown | Earned by | Means | Does not mean | +|---|---|---|---| +| **Cases pass** | Running the pseudo on the cases in `routine.json` | The routine gives those results; they are what every reference is held to | That the results are right for other inputs | +| **Prover: pass** | Every obligation of the routine proved, as for any pseudo | Its contracts hold for every input that meets its REQUIRE, in the notation's number model | Anything about the value of FLOAT arithmetic beyond what the contracts say | +| **Prover: open** | Some obligation not proved, none broken | The routine's comment says which and why (usually FLOAT rounding, which is opaque to the prover; `tests/documents.test.mjs` fails when the comment does not mention it) | That it is wrong | +| **A reference agrees** | Python, C at -O0 and -O2, core's Mojo interpreter, the pinned Mojo compiler on the machine that ran the check (`compiler`), or the pinned compiler on godbolt, run on the cases. A C or Mojo reference's program holds no case: it reads every case's arguments from stdin at run time, each FLOAT bit for bit (`tests/library-harness.test.mjs`), so the compiler cannot work them out while it builds | That implementation, at that level, prints the pseudo's results exactly, or within the stated tolerance, computed by the code the compiler made for inputs it did not know. A Mojo reference is built with the pin's arguments, contraction off among them, so that code rounds each FLOAT operation on its own, as the pseudo does; Mojo's own math library (`exp`, `log2`, `cosh`, `**`) is not the pseudo's, and a reference that calls it states a tolerance and why | That it agrees on other inputs, or built with other flags: with Mojo's defaults the code fuses a multiply and an add into one rounding where it can (`probes/float-contract-runtime`), and a last digit can differ | +| **Mojo: unverified** | No stored answer from the pinned compiler for that exact program and input | Not yet asked (`pixi run library-verify-mojo`) | That it fails | +| **Proof: pass** | The pinned Dafny verified the proof file (or its stored verdict for this exact text) | The method is correct over exact numbers, as the proof states it | That the FLOAT version computes the same digits: rounding is outside the proof | + diff --git a/library b/library new file mode 160000 index 0000000..568474d --- /dev/null +++ b/library @@ -0,0 +1 @@ +Subproject commit 568474d52f1dc48227449c2eaee412a11154e272 diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..e8ff43d --- /dev/null +++ b/package-lock.json @@ -0,0 +1,33 @@ +{ + "name": "pplus-core", + "version": "0.2.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "pplus-core", + "version": "0.2.0", + "license": "Apache-2.0", + "devDependencies": { + "typescript": "^6.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..55643fd --- /dev/null +++ b/package.json @@ -0,0 +1,20 @@ +{ + "name": "pplus-core", + "version": "0.2.0", + "private": true, + "license": "Apache-2.0", + "type": "module", + "description": "Programming+ core: the languages, translators, Mojo checker and interpreter, prover, fuzzer and machine model, with the stored answers of the real tools. Day-to-day commands are pixi tasks (pixi.toml); these scripts are what CI and a machine without pixi need.", + "scripts": { + "build": "tsc -p tsconfig.json", + "watch": "tsc -p tsconfig.json --watch", + "check": "tsc -p tsconfig.json --noEmit", + "test": "tsc -p tsconfig.json && node --test --test-concurrency=8 tests/*.test.mjs" + }, + "devDependencies": { + "typescript": "^6.0.0" + }, + "engines": { + "node": ">=20" + } +} diff --git a/pixi.lock b/pixi.lock new file mode 100644 index 0000000..02b86e5 --- /dev/null +++ b/pixi.lock @@ -0,0 +1,1395 @@ +version: 7 +platforms: +- name: linux-64 + virtual-packages: + - __unix=0=0 + - __linux=4.18 + - __glibc=2.28 + - __archspec=0=x86_64 +- name: osx-arm64 + virtual-packages: + - __unix=0=0 + - __osx=13.0 + - __archspec=0=m1 +environments: + default: + channels: + - url: https://conda.modular.com/max/ + - url: https://conda.anaconda.org/conda-forge/ + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_10.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.8-hebe6cf0_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.3-py310h44b86e0_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.3-h7cc23a3_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/krb5-1.22.2-hbc21106_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.46.1-default_hbd61a6d_102.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libabseil-20260526.0-cxx17_h0dc7533_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.2.0-h39a168f_4.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.2.0-ha411449_4.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.2.0-h018ffa1_4.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h373387f_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h280c20c_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.8.5-hd2095e1_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.7.0-h81df57d_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-16.2.0-ha9f2e26_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-16.2.0-he0feb66_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.3-hb03c661_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb03c661_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.68.1-h74cf4be_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libpython-3.14.7-hdc7f604_106_cp314.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.22-hebe6cf0_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.53.4-h13e7031_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-16.2.0-h934c35e_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.42.4-hcfc3c73_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuv-1.52.1-h280c20c_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.2-h25fd6f3_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.6-hdb14827_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/nodejs-26.10.0-ha5cdc1e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.4-h781a0a9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.14.7-hcd007b5_106_cp314.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyzmq-27.2.0-py312h8a5ba0d_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-hd6e31c0_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h1df4ec4_4.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tornado-6.5.10-py314h89acca1_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.5-h901266b_12.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_7.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.5.0-pyh5ded981_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.14.7-py314hd8ed1ab_106.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-9.0.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyhc90fa1f_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.1.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.3-pyhc364b38_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pathspec-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.12.2-pyh5ded981_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.14.7-h4df99d1_106.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-9_cp314.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.4.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.16.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda + - conda: https://conda.modular.com/max/linux-64/mojo-1.2.0-release.conda + - conda: https://conda.modular.com/max/linux-64/mojo-compiler-1.2.0-release.conda + - conda: https://conda.modular.com/max/noarch/mblack-26.7.0-release.conda + - conda: https://conda.modular.com/max/noarch/mojo-python-1.2.0-release.conda + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.5.0-pyh5ded981_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.14.7-py314hd8ed1ab_106.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-9.0.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyhc90fa1f_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.1.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.3-pyhc364b38_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pathspec-1.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.12.2-pyh5ded981_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.14.7-h4df99d1_106.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-9_cp314.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.4.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.16.1-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h4e30115_10.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.8-h74c22ad_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-py310h579977c_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/krb5-1.22.2-h34f8a20_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libabseil-20260526.0-cxx17_h4c27e2a_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlicommon-1.2.0-h1dcdb26_4.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlidec-1.2.0-h5295a6a_4.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlienc-1.2.0-h2ddc9cb_4.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-23.1.2-h55c6f16_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libedit-3.1.20250104-pl5321h26f1114_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libev-4.33-h1a92334_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.8.5-h6111c0a_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.7.0-h47dc5ef_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.3-h8088a28_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libmpdec-4.0.0-h84a0fba_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libnghttp2-1.68.1-h435687b_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libpython-3.14.7-h4311e70_106_cp314.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsodium-1.0.22-h74c22ad_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.53.4-hca69786_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libuv-1.52.1-h1a92334_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.6-he64c551_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/nodejs-26.10.0-h8d74fab_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.4-h55eecbc_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.14.7-h0ae1c2c_106_cp314.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyzmq-27.2.0-py312hdfb5542_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h8b90a29_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-hbeba79b_4.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tornado-6.5.10-py314hcd3153d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zeromq-4.3.5-h6eb0046_12.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda + - conda: https://conda.modular.com/max/noarch/mblack-26.7.0-release.conda + - conda: https://conda.modular.com/max/noarch/mojo-python-1.2.0-release.conda + - conda: https://conda.modular.com/max/osx-arm64/mojo-1.2.0-release.conda + - conda: https://conda.modular.com/max/osx-arm64/mojo-compiler-1.2.0-release.conda +packages: +- conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda + build_number: 20 + sha256: 1dd3fffd892081df9726d7eb7e0dea6198962ba775bd88842135a4ddb4deb3c9 + md5: a9f577daf3de00bca7c3c76c0ecbd1de + depends: + - __glibc >=2.17,<3.0.a0 + - libgomp >=7.5.0 + constrains: + - openmp_impl <0.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + strong: + - _openmp_mutex >=4.5 + size: 28948 + timestamp: 1770939786096 +- conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_10.conda + sha256: 1a0d382c515ebf55f8ee1f38c8b81bc95af5c2acc42ad53b66bc5df932032f96 + md5: e675fabcf81499adc7edf58124fb1e01 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: bzip2-1.0.6 + license_family: BSD + run_exports: + weak: + - bzip2 >=1.0.8,<2.0a0 + size: 257808 + timestamp: 1785906269155 +- conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.8-hebe6cf0_2.conda + sha256: 9c37cc233238adf366ea75b0e845662a5be07ceadf62e458183516369340274b + md5: 3ad2b403be8fc5612b9159e2ce621f69 + depends: + - libgcc >=15 + - __glibc >=2.17,<3.0.a0 + license: MIT + license_family: MIT + run_exports: + weak: + - c-ares >=1.34.8,<2.0a0 + size: 228700 + timestamp: 1787169971173 +- conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.3-py310h44b86e0_2.conda + sha256: 9f07834f0c546ab14d885ce0366285f61f44e326c0edd1fc63b8294e113ae432 + md5: 72a381cbad04f24b1c2a43ef707f45b4 + depends: + - __glibc >=2.17,<3.0.a0 + - libstdcxx >=14 + - libgcc >=14 + license: MIT + license_family: MIT + run_exports: + weak: + - icu >=78.3,<79.0a0 + size: 14459115 + timestamp: 1786545741408 +- conda: https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.3-h7cc23a3_1.conda + sha256: dd053c96dcb0dcfd59422aefea9d2fe937a190167f34fba7893ec1e10a7e8963 + md5: ba55d1b89fd7775e67de8291029b4059 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=15 + license: LGPL-2.1-or-later + run_exports: + weak: + - keyutils >=1.6.3,<2.0a0 + size: 135295 + timestamp: 1786739238128 +- conda: https://conda.anaconda.org/conda-forge/linux-64/krb5-1.22.2-hbc21106_2.conda + sha256: 2a5c38c85e63df84c4e69ee71439841ce570d259ae3060627bb9a49a938d66f4 + md5: 53318d715316929a574f83591308b1f8 + depends: + - __glibc >=2.17,<3.0.a0 + - keyutils >=1.6.3,<2.0a0 + - libedit >=3.1.20250104,<3.2.0a0 + - libedit >=3.1.20250104,<4.0a0 + - libgcc >=15 + - libstdcxx >=15 + - openssl >=3.5.7,<4.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - krb5 >=1.22.2,<1.23.0a0 + size: 1394333 + timestamp: 1786762112514 +- conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.46.1-default_hbd61a6d_102.conda + sha256: 27d83f1188cd19bcb7754a078b3fa7f4cfb8527f8eb2fde54dd01fc529d1adec + md5: 449500f2c089da11c40f5c21312e3e07 + depends: + - __glibc >=2.17,<3.0.a0 + - zstd >=1.5.7,<1.6.0a0 + constrains: + - binutils_impl_linux-64 2.46.1 + license: GPL-3.0-only + license_family: GPL + run_exports: {} + size: 745303 + timestamp: 1784214507189 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libabseil-20260526.0-cxx17_h0dc7533_2.conda + sha256: 7bb3d495411b7059e85225aae500d84e7846a4bb02ebd77e4450200b20c180f0 + md5: 6983bfe8e09992014b9cf393769c7a84 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libstdcxx >=14 + constrains: + - abseil-cpp =20260526.0 + - libabseil-static =20260526.0=cxx17* + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - libabseil >=20260526.0,<20260527.0a0 + - libabseil =*=cxx17* + size: 1436888 + timestamp: 1787217011773 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.2.0-h39a168f_4.conda + sha256: c4e9854c585f3ee1785f651287ecdf8c28f30a2ea1b159a513005f8a0f6453a3 + md5: df210655e30a05e3b069c91b7aaddd2d + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=15 + license: MIT + license_family: MIT + run_exports: + weak: + - libbrotlicommon >=1.2.0,<1.3.0a0 + size: 80561 + timestamp: 1788480364653 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.2.0-ha411449_4.conda + sha256: d5ed24da3e583a92227fe77280978b7933f07a39a2b60156c5ad95e5c76733fa + md5: 0cfd601eb03cca403dcc248844787486 + depends: + - __glibc >=2.17,<3.0.a0 + - libbrotlicommon 1.2.0 h39a168f_4 + - libgcc >=15 + license: MIT + license_family: MIT + run_exports: + weak: + - libbrotlidec >=1.2.0,<1.3.0a0 + size: 34739 + timestamp: 1788480374261 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.2.0-h018ffa1_4.conda + sha256: b9f3cb510337655485c2e79ce5f7dd468991b19ad29723ffa1175a0aed1b69ae + md5: 4136f6e4ef4835cc7df39a707cbd511c + depends: + - __glibc >=2.17,<3.0.a0 + - libbrotlicommon 1.2.0 h39a168f_4 + - libgcc >=15 + license: MIT + license_family: MIT + run_exports: + weak: + - libbrotlienc >=1.2.0,<1.3.0a0 + size: 298134 + timestamp: 1788480383223 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h373387f_1.conda + sha256: 6473eb8caf2aae830f37caa93db9b26dddf7ac84b63229e8bf7fc0e5c3ab95b0 + md5: 50708d3b951d0f8e2d7f2df5b5edc040 + depends: + - ncurses + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + - ncurses >=6.6,<7.0a0 + license: BSD-2-Clause + license_family: BSD + run_exports: + weak: + - libedit >=3.1.20250104,<3.2.0a0 + size: 135098 + timestamp: 1786616658086 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h280c20c_3.conda + sha256: e4418f68d01a6307c4431b585c71b584f40189877364e542ee87deb777f5e85b + md5: 7bc31538d6e3fb349e897353969237ec + depends: + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + license: BSD-2-Clause OR GPL-2.0-or-later + license_family: GPL + run_exports: + weak: + - libev >=4.33,<4.34.0a0 + size: 43220 + timestamp: 1785917328200 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.8.5-hd2095e1_0.conda + sha256: 6de73033a7c5f712c9c5a54124e4581e025691029835e5bceaf10a333a7e7bc0 + md5: 72cfd2b95b0540247220b42f58a148d7 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=15 + constrains: + - expat 2.8.5.* + license: MIT + license_family: MIT + run_exports: {} + size: 77893 + timestamp: 1790346247149 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.7.0-h81df57d_1.conda + sha256: c8c7583ef063bc3c430f1d48298e5ad24f796a35c22ed5b14183325825a61106 + md5: 6525a0b06aa4fd390795f0740636a9dd + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=15 + license: MIT + license_family: MIT + run_exports: + weak: + - libffi >=3.7.0,<3.8.0a0 + size: 68004 + timestamp: 1787753412298 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-16.2.0-ha9f2e26_7.conda + sha256: e031634c3a928f9594eba868bc46ef5230d6571a364a0c63215b06c4563bfcea + md5: 735629b400b70eeab21a2ba8f5b1c2ce + depends: + - __glibc >=2.17,<3.0.a0 + - _openmp_mutex >=4.5 + constrains: + - libgcc-ng ==16.2.0=*_7 + - libgomp 16.2.0 he0feb66_7 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + run_exports: {} + size: 1058444 + timestamp: 1790303109576 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-16.2.0-he0feb66_7.conda + sha256: 92a0bb350c7be6a4eedcb4e111c60aefd176018e6b1c009a6e025dbe9bbcd758 + md5: d904c7ddd95e026254b4b80808ee9e12 + depends: + - __glibc >=2.17,<3.0.a0 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + run_exports: + strong: + - _openmp_mutex >=4.5 + size: 640566 + timestamp: 1790303046034 +- conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.3-hb03c661_1.conda + sha256: 9787df8c22a59c9a70d3e5a10db9ad663485e75e9ccc3f09bd092cb7b95e0dab + md5: 1390b7c5ac0b1d8e447bc5efa6d3c8c2 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + constrains: + - xz 5.8.3.* + license: 0BSD + run_exports: + weak: + - liblzma >=5.8.3,<6.0a0 + size: 112995 + timestamp: 1786348617826 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb03c661_2.conda + sha256: 46820b4a835e175940ae20bec00fdddaff804cda27a1408ab1b95e77a2196437 + md5: fcfed1dc5053eb1901b66e7b1fc32588 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: BSD-2-Clause + license_family: BSD + run_exports: {} + size: 92759 + timestamp: 1786650399772 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.68.1-h74cf4be_1.conda + sha256: 11a2f54a6f391e25738bce0023e6ef499600147bbcf21c1fa69d2e251b03cc6a + md5: 75d211f04ac87506f1f43420712a69fe + depends: + - __glibc >=2.17,<3.0.a0 + - c-ares >=1.34.8,<2.0a0 + - libev >=4.33,<4.34.0a0 + - libev >=4.33,<5.0a0 + - libgcc >=15 + - libstdcxx >=15 + - libzlib >=1.3.2,<2.0a0 + - openssl >=3.5.7,<4.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - libnghttp2 >=1.68.1,<2.0a0 + size: 649974 + timestamp: 1787177490105 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libpython-3.14.7-hdc7f604_106_cp314.conda + build_number: 106 + sha256: 5f879892bd439c3d94f4a97b06c214b9a19c4b92f74d84f0305c6ca1b2b239b2 + md5: 28af2158bb8dbb62e55b0af3b44f9d4a + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=15 + - libstdcxx >=15 + license: Python-2.0 + run_exports: {} + size: 10520530 + timestamp: 1788383918299 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.22-hebe6cf0_3.conda + sha256: 2f73c05a1ad07e5a2cb43f0b3759cb04505f0c8bef30b43c9db0caa50218bfd8 + md5: 2d14868d78e969226e4ba07444c0dd25 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=15 + license: ISC + run_exports: + weak: + - libsodium >=1.0.22,<1.0.23.0a0 + size: 270970 + timestamp: 1789425325397 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.53.4-h13e7031_1.conda + sha256: f20d70da54e5b31dd4a51fb1efeaafafd5ab6dd8d7ac9d1438eaa2526ac4ed3d + md5: e72bbec309c2b0f37823ee7d4fabfcd3 + depends: + - __glibc >=2.17,<3.0.a0 + - icu >=78.3,<79.0a0 + - libgcc >=15 + - libzlib >=1.3.2,<2.0a0 + license: blessing + run_exports: + weak: + - libsqlite >=3.53.4,<4.0a0 + size: 974348 + timestamp: 1787051145557 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-16.2.0-h934c35e_7.conda + sha256: fa7018298629fc429971fffd6ecba3d27a7b02e4e890d6a93ea01ffebe8cb745 + md5: 9b6707e0caca917fa6ea9ee473f37a5d + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc 16.2.0 ha9f2e26_7 + constrains: + - libstdcxx-ng ==16.2.0=*_7 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + run_exports: {} + size: 6615436 + timestamp: 1790303132935 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.42.4-hcfc3c73_0.conda + sha256: f34b8d1de9a2bf62476d8608057d9b8a918d97444236dc53a47128f2fae79041 + md5: 54a948337e66706380ff24d6b5fde63f + depends: + - libgcc >=15 + - __glibc >=2.17,<3.0.a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - libuuid >=2.42.4,<3.0a0 + size: 34442 + timestamp: 1790121449161 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libuv-1.52.1-h280c20c_1.conda + sha256: 67761d0206f84140047a367eaf9befe03a7e157a29ee25aee0047b40801cc6b6 + md5: 01c4ed87826af55996768af6dbc936b4 + depends: + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + license: MIT + license_family: MIT + run_exports: + weak: + - libuv >=1.52.1,<2.0a0 + size: 420040 + timestamp: 1785914567661 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.2-h25fd6f3_3.conda + sha256: eb8a0db0aa570124f7d2a93d7c7f596e3390df5e047818d873baad32985fc736 + md5: 0de0122d9570a8ab637c6b73db268389 + depends: + - __glibc >=2.17,<3.0.a0 + constrains: + - zlib 1.3.2 *_3 + license: Zlib + license_family: Other + run_exports: + weak: + - libzlib >=1.3.2,<2.0a0 + size: 63713 + timestamp: 1785362952714 +- conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.6-hdb14827_1.conda + sha256: 5d46557214ed184381dafe835b7c94a474a1c3b307a08a250b1ea4779b44ffb3 + md5: ee6c0cd80a60961a1f48aa3e0b91f986 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: X11 AND BSD-3-Clause + run_exports: + weak: + - ncurses >=6.6,<7.0a0 + size: 911196 + timestamp: 1786355078102 +- conda: https://conda.anaconda.org/conda-forge/linux-64/nodejs-26.10.0-ha5cdc1e_0.conda + sha256: ea1ae98c7352198d6d6a4027f4a60d2b634e05d5fd6f0b97e72fc6c16712b2c7 + md5: 46a3d9861f49b845fbd54897ff07fe4f + depends: + - libgcc >=15 + - __glibc >=2.28,<3.0.a0 + - libstdcxx >=15 + - zstd >=1.5.7,<1.6.0a0 + - libzlib >=1.3.2,<2.0a0 + - libuv >=1.52.1,<2.0a0 + - openssl >=3.5.8,<4.0a0 + - libnghttp2 >=1.68.1,<2.0a0 + - libabseil >=20260526.0,<20260527.0a0 + - libabseil * cxx17* + - icu >=78.3,<79.0a0 + - c-ares >=1.34.8,<2.0a0 + - libsqlite >=3.53.4,<4.0a0 + - libbrotlicommon >=1.2.0,<1.3.0a0 + - libbrotlienc >=1.2.0,<1.3.0a0 + - libbrotlidec >=1.2.0,<1.3.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - nodejs >=26.10.0,<27.0a0 + size: 20474429 + timestamp: 1790238478566 +- conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.4-h781a0a9_0.conda + sha256: 4747b2d6a8336f52343bceb8a1ebf41a9e6e665b9d2e3f989972de53a310599e + md5: 16e3034a330cc625f2c685edec60cf4e + depends: + - __glibc >=2.17,<3.0.a0 + - ca-certificates + - libgcc >=15 + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - openssl >=3.6.4,<4.0a0 + size: 3202955 + timestamp: 1787698780103 +- conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.14.7-hcd007b5_106_cp314.conda + build_number: 106 + sha256: 4cd05407d6b07d00fd5b6cc78bd2f60ae5e21f777eb26ead82be2e3751c610a1 + md5: 56ee91e118243e46bfc0c41e4030d354 + depends: + - __glibc >=2.17,<3.0.a0 + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 + - libexpat >=2.8.1,<3.0a0 + - libffi >=3.7.0,<3.8.0a0 + - libgcc >=15 + - liblzma >=5.8.3,<6.0a0 + - libmpdec >=4.0.0,<5.0a0 + - libpython 3.14.7 hdc7f604_106_cp314 + - libsqlite >=3.53.4,<4.0a0 + - libuuid >=2.42.3,<3.0a0 + - libzlib >=1.3.2,<2.0a0 + - ncurses >=6.6,<7.0a0 + - openssl >=3.5.8,<4.0a0 + - python_abi 3.14.* *_cp314 + - readline >=8.3,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - zstd >=1.5.7,<1.6.0a0 + license: Python-2.0 + run_exports: + weak: + - python_abi 3.14.* *_cp314 + noarch: + - python + size: 26484459 + timestamp: 1788383955577 + python_site_packages_path: lib/python3.14/site-packages +- conda: https://conda.anaconda.org/conda-forge/linux-64/pyzmq-27.2.0-py312h8a5ba0d_2.conda + noarch: python + sha256: d1323678d9403504d05cdf6a65bbd648f8667a89d9d5d4146ff96bae0e5311a7 + md5: 2186325469ffe83c7b8807d638a478d9 + depends: + - python + - __glibc >=2.17,<3.0.a0 + - libgcc >=15 + - libstdcxx >=15 + - zeromq >=4.3.5,<4.4.0a0 + - _python_abi3_support 1.* + - cpython >=3.12 + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 216556 + timestamp: 1789730121925 +- conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-hd6e31c0_1.conda + sha256: 01b3fe073a66e321970d09e04b388708f8cbdca5cdfbfcb7c9eeb470ad10383d + md5: 69c01c781e7c8190bbdaf79e3848c5ca + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=15 + - ncurses >=6.6,<7.0a0 + license: GPL-3.0-only + license_family: GPL + run_exports: + weak: + - readline >=8.3,<9.0a0 + size: 348899 + timestamp: 1787033801506 +- conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h1df4ec4_4.conda + build_number: 104 + sha256: a1a241d172c1ccab067ba245206dd048bc3c2d1b84504b53c9468e99adfc16a1 + md5: 676a2f9b1c4fcf57b70aa5c65f6c60d0 + depends: + - libgcc >=15 + - __glibc >=2.17,<3.0.a0 + - libzlib >=1.3.2,<2.0a0 + constrains: + - xorg-libx11 >=1.8.13,<2.0a0 + license: TCL + run_exports: + weak: + - tk >=8.6.13,<8.7.0a0 + size: 3566806 + timestamp: 1787272857910 +- conda: https://conda.anaconda.org/conda-forge/linux-64/tornado-6.5.10-py314h89acca1_0.conda + sha256: c205ceaa1536a8729bdad405ca711e94d2196aba30771029793557f250294598 + md5: d2bd0918d9c10df4fe2355e33fe7bc63 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=15 + - python >=3.14,<3.15.0a0 + - python_abi 3.14.* *_cp314 + license: Apache-2.0 + license_family: Apache + run_exports: {} + size: 950218 + timestamp: 1789911064461 +- conda: https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.5-h901266b_12.conda + sha256: 37dc9a5ff564fdce07c5ad002ee572057cf41dedda274d7abc154e77094d981a + md5: 292a05f1436b98e17055090b5ef07746 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=15 + - libstdcxx >=15 + - libsodium >=1.0.22,<1.0.23.0a0 + - krb5 >=1.22.2,<1.23.0a0 + license: MPL-2.0 + license_family: MOZILLA + run_exports: + weak: + - zeromq >=4.3.5,<4.4.0a0 + size: 321839 + timestamp: 1789076548604 +- conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_7.conda + sha256: 47d682b9f6d6ec9eb1a6e6c3e75ea6273e899e78fb7fc59f81d39745009fbc60 + md5: aa459086047c0e5e27023ab19f8cb86a + depends: + - __glibc >=2.17,<3.0.a0 + - libzlib >=1.3.2,<2.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - zstd >=1.5.7,<1.6.0a0 + size: 601301 + timestamp: 1786599621503 +- conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_3.conda + sha256: 2a7204314663eeda5dec482a956f0e2eaf289bd5b9953eaaaad0e81aa64638f2 + md5: 3845f3d75991bae0fb90884662f4327c + depends: + - cpython + - python-gil + license: MIT + license_family: MIT + run_exports: {} + size: 8144 + timestamp: 1784221492234 +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + sha256: 0a0544cf95f64394fe4959286f5c71f5444ad58feb0602e53becb27448d24da6 + md5: 0f51e2391ade309db462a55611263e9c + depends: + - __unix + license: ISC + run_exports: {} + size: 131780 + timestamp: 1784754889428 +- conda: https://conda.anaconda.org/conda-forge/noarch/click-8.5.0-pyh5ded981_0.conda + sha256: 9afb0c2c089330321a219c7ee5a25313bd574d4024be606cbb33e7a5735df662 + md5: dea5b13a211bbf99876deb980b408a66 + depends: + - python >=3.11 + - python + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 112341 + timestamp: 1788802215348 +- conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.14.7-py314hd8ed1ab_106.conda + noarch: generic + sha256: fd701880cf6f696420aa5b4b530dbdf0c83a16bdb50aeb760b3159eec5c416bf + md5: 3b52b830ac00367685c08f98f50963bf + depends: + - python >=3.14,<3.15.0a0 + - python_abi * *_cp314 + license: Python-2.0 + run_exports: {} + size: 51289 + timestamp: 1788383537525 +- conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-9.0.1-pyhcf101f3_0.conda + sha256: 95a074a7d3f58b3494c068d4789c364dcb591c56b70ca6b12f149a540f9aeea4 + md5: 77ec68e0aa61c3fff9ecbf840f93d64e + depends: + - python >=3.10 + - zipp >=3.20 + - python + license: Apache-2.0 + license_family: APACHE + run_exports: {} + size: 34920 + timestamp: 1787943971257 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda + sha256: 19d8bd5bb2fde910ec59e081eeb59529491995ce0d653a5209366611023a0b3a + md5: 4ebae00eae9705b0c3d6d1018a81d047 + depends: + - importlib-metadata >=4.8.3 + - jupyter_core >=4.12,!=5.0.* + - python >=3.9 + - python-dateutil >=2.8.2 + - pyzmq >=23.0 + - tornado >=6.2 + - traitlets >=5.3 + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 106342 + timestamp: 1733441040958 +- conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyhc90fa1f_0.conda + sha256: 1d34b80e5bfcd5323f104dbf99a2aafc0e5d823019d626d0dce5d3d356a2a52a + md5: b38fe4e78ee75def7e599843ef4c1ab0 + depends: + - __unix + - python + - platformdirs >=2.5 + - python >=3.10 + - traitlets >=5.3 + - python + constrains: + - pywin32 >=300 + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 65503 + timestamp: 1760643864586 +- conda: https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.1.0-pyha770c72_0.conda + sha256: 6ed158e4e5dd8f6a10ad9e525631e35cee8557718f83de7a4e3966b1f772c4b1 + md5: e9c622e0d00fa24a6292279af3ab6d06 + depends: + - python >=3.9 + license: MIT + license_family: MIT + run_exports: {} + size: 11766 + timestamp: 1745776666688 +- conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.3-pyhc364b38_0.conda + sha256: c432626b16768b8dab228bfb706f7060c2d462a21c516d240f68f2f902b5a044 + md5: 936687ed80f295a1f5dbcf8bd34c252c + depends: + - python >=3.9 + - python + license: Apache-2.0 + license_family: APACHE + run_exports: {} + size: 116363 + timestamp: 1785888127370 +- conda: https://conda.anaconda.org/conda-forge/noarch/pathspec-1.1.1-pyhd8ed1ab_0.conda + sha256: 6eaee417d33f298db79bc7185ab1208604c0e6cf51dade34cd513c6f9db9c6f3 + md5: 11adc78451c998c0fd162584abfa3559 + depends: + - python >=3.10 + license: MPL-2.0 + license_family: MOZILLA + run_exports: {} + size: 56559 + timestamp: 1777271601895 +- conda: https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.12.2-pyh5ded981_0.conda + sha256: f23f022bbbf809cd248940eca853d1132f05752e0404984cde7b014fd6384c97 + md5: 4b76957bed5e02e067a348eda60c6f94 + depends: + - python >=3.11 + - python + license: MIT + license_family: MIT + run_exports: {} + size: 33868 + timestamp: 1790738680369 +- conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda + sha256: d6a17ece93bbd5139e02d2bd7dbfa80bee1a4261dced63f65f679121686bf664 + md5: 5b8d21249ff20967101ffa321cab24e8 + depends: + - python >=3.9 + - six >=1.5 + - python + license: Apache-2.0 + license_family: APACHE + run_exports: {} + size: 233310 + timestamp: 1751104122689 +- conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.14.7-h4df99d1_106.conda + sha256: 801ef80bc9f9394d9b3b5cae5a90c33cbbc33004b3e93195b93af818aad8888b + md5: b672ecb44198ade19ef249bf99f4b1ff + depends: + - cpython 3.14.7.* + - python_abi * *_cp314 + license: Python-2.0 + run_exports: {} + size: 51276 + timestamp: 1788383547342 +- conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-9_cp314.conda + build_number: 9 + sha256: e7f8e965bbfb98e08feb2365cacdad8bb218fa5b5a0a2752f747287f425f213c + md5: 350d89b50d7de805abf2e63e29dee451 + constrains: + - python 3.14.* *_cp314 + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 6791 + timestamp: 1788302824440 +- conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + sha256: 458227f759d5e3fcec5d9b7acce54e10c9e1f4f4b7ec978f3bfd54ce4ee9853d + md5: 3339e3b65d58accf4ca4fb8748ab16b3 + depends: + - python >=3.9 + - python + license: MIT + license_family: MIT + run_exports: {} + size: 18455 + timestamp: 1753199211006 +- conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.4.1-pyhcf101f3_0.conda + sha256: 91cafdb64268e43e0e10d30bd1bef5af392e69f00edd34dfaf909f69ab2da6bd + md5: b5325cf06a000c5b14970462ff5e4d58 + depends: + - python >=3.10 + - python + license: MIT + license_family: MIT + run_exports: {} + size: 21561 + timestamp: 1774492402955 +- conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.16.1-pyhcf101f3_0.conda + sha256: 03dba5917f944c6684ab44c81daacac1624cd148e4b2cae215dcec594a210c48 + md5: a79bf97561232a31447b6246c2153ab5 + depends: + - python >=3.10 + - python + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 116935 + timestamp: 1785761789772 +- conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda + sha256: b928c30ddcb0e3f544c6eade8352737e6e610e263276b90232db6a578ef899d8 + md5: fcb489df604d100968b737f2cb6076c6 + license: LicenseRef-Public-Domain + run_exports: {} + size: 118849 + timestamp: 1784250406640 +- conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda + sha256: 210bd31c22bb88f5e2a167df24c95bb5f152b2ada7502f9b8c49d1f5366db423 + md5: ba3dcdc8584155c97c648ae9c044b7a3 + depends: + - python >=3.10 + - python + license: MIT + license_family: MIT + run_exports: {} + size: 24190 + timestamp: 1779159948016 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h4e30115_10.conda + sha256: 8ec22f0ba25cbfc2e64d70cf29459eccd7ffdf6436f6a6ff15bbfef799f7d4f6 + md5: b50612e7d190b8061ab4e7dc119cf4d5 + depends: + - __osx >=11.0 + license: bzip2-1.0.6 + license_family: BSD + run_exports: + weak: + - bzip2 >=1.0.8,<2.0a0 + size: 124965 + timestamp: 1785906749812 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.8-h74c22ad_2.conda + sha256: 82bdd5a3fcada6afef6255a9bf41172f07b362f36e04a354dc2abc0a29bfb756 + md5: 4cc01a374215de38387d45e19c8c5c9c + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - c-ares >=1.34.8,<2.0a0 + size: 197819 + timestamp: 1787169996553 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-py310h579977c_2.conda + sha256: 6cdb5dee54c72e56ab189fb3ad33cb28533553d42590e7e831160248f4416a43 + md5: a5efc0b42bb8b42e97d0a29ae3e3c187 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - icu >=78.3,<79.0a0 + size: 14070242 + timestamp: 1786545847761 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/krb5-1.22.2-h34f8a20_2.conda + sha256: aaea1d42b07769920db2af7471ece9399d2613448863da64ad8272998db5db34 + md5: 15235dd10450d67bc25ccafd5b46d2bc + depends: + - __osx >=11.0 + - libcxx >=21 + - libedit >=3.1.20250104,<3.2.0a0 + - libedit >=3.1.20250104,<4.0a0 + - openssl >=3.5.7,<4.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - krb5 >=1.22.2,<1.23.0a0 + size: 1165740 + timestamp: 1786762145768 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libabseil-20260526.0-cxx17_h4c27e2a_2.conda + sha256: d9c62dae9d8f5bebadf72fcf369c00cc353183cb2521f9fffbf4c2f70b58bef9 + md5: 2aa5e7dc7b5d218908effd2a8c70a8a8 + depends: + - __osx >=11.0 + - libcxx >=19 + constrains: + - abseil-cpp =20260526.0 + - libabseil-static =20260526.0=cxx17* + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - libabseil >=20260526.0,<20260527.0a0 + - libabseil =*=cxx17* + size: 1277537 + timestamp: 1787217005681 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlicommon-1.2.0-h1dcdb26_4.conda + sha256: 8a6e4789a6049dabb41c6fcaf6719232245484d81dfeb199b5abdb95c84c9fc2 + md5: 8f3981871d167a6cd323e636e1929dd4 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - libbrotlicommon >=1.2.0,<1.3.0a0 + size: 79648 + timestamp: 1788480342707 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlidec-1.2.0-h5295a6a_4.conda + sha256: 4e1f5989561de4546209b42c935795a41fa4265e0b670861480f1adffec623a8 + md5: da537a1643ef3e13bdccf16cca206f2c + depends: + - __osx >=11.0 + - libbrotlicommon 1.2.0 h1dcdb26_4 + license: MIT + license_family: MIT + run_exports: + weak: + - libbrotlidec >=1.2.0,<1.3.0a0 + size: 29864 + timestamp: 1788480352084 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libbrotlienc-1.2.0-h2ddc9cb_4.conda + sha256: 4d2da97825b221054a39f492e4dffe361e1931c8381d0367f1e0598a50d02f34 + md5: 39a25d500b191c16996239c4afa104f1 + depends: + - __osx >=11.0 + - libbrotlicommon 1.2.0 h1dcdb26_4 + license: MIT + license_family: MIT + run_exports: + weak: + - libbrotlienc >=1.2.0,<1.3.0a0 + size: 295505 + timestamp: 1788480359341 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-23.1.2-h55c6f16_0.conda + sha256: 087a610bacee7901d259ef72c6f060d795ab26831bd7541ba75ced786a5a63f9 + md5: 01af3f03d850cbab9ae9b86d726a9921 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + run_exports: {} + size: 579372 + timestamp: 1790163402114 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libedit-3.1.20250104-pl5321h26f1114_1.conda + sha256: 257c926f19e32bdb981fc674c76966049b6fc73f706bae58e9fed8757ad1da70 + md5: 843ef89082f368cb889305084d3b483c + depends: + - ncurses + - __osx >=11.0 + - ncurses >=6.6,<7.0a0 + license: BSD-2-Clause + license_family: BSD + run_exports: + weak: + - libedit >=3.1.20250104,<3.2.0a0 + size: 107742 + timestamp: 1786616721640 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libev-4.33-h1a92334_3.conda + sha256: c0100064506ae8abb432c5a506d474f10af2cf48c33d62bc221fb28b6d6ff6ac + md5: 19e86c8a6a47e92bb2e70ca12e758c5c + depends: + - __osx >=11.0 + license: BSD-2-Clause OR GPL-2.0-or-later + license_family: GPL + run_exports: + weak: + - libev >=4.33,<4.34.0a0 + size: 39991 + timestamp: 1785917376956 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.8.5-h6111c0a_0.conda + sha256: 50aa761b83fc7a116ca88fe0cb4cefce5abababb10a926bf8afbafd5c298d102 + md5: 5383c550b785bdaafcc4eaf638071a7c + depends: + - __osx >=11.0 + constrains: + - expat 2.8.5.* + license: MIT + license_family: MIT + run_exports: {} + size: 69565 + timestamp: 1790370026495 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.7.0-h47dc5ef_1.conda + sha256: 2783389a7b9dda04c62cc54c6bbeb03dd3cef24b3fc642715d02a3fa19464a9a + md5: 216bbcc23c11e9695b3db4a8771d76eb + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - libffi >=3.7.0,<3.8.0a0 + size: 43637 + timestamp: 1787753403688 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.3-h8088a28_1.conda + sha256: 23d0630046a3e8b164d8f80f2b74ed2605af2e7050ab9913018056402fae4311 + md5: 8ab10323068b107661a4b9a4af84f3b5 + depends: + - __osx >=11.0 + constrains: + - xz 5.8.3.* + license: 0BSD + run_exports: + weak: + - liblzma >=5.8.3,<6.0a0 + size: 91720 + timestamp: 1786348695846 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libmpdec-4.0.0-h84a0fba_2.conda + sha256: 04cc136c5a956a73aa14e0a160b5822b0b29714e67b299fa1b1fd16dbcba5366 + md5: ff33a4dbd93abc8a798cc4e0e7c8136d + depends: + - __osx >=11.0 + license: BSD-2-Clause + license_family: BSD + run_exports: {} + size: 73289 + timestamp: 1786651074391 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libnghttp2-1.68.1-h435687b_1.conda + sha256: 445440d8cccccb6585f71d1eb7c949d64a2af2ced5481d924621424ff3b6a398 + md5: ac9902dcbe0417f50cf95ab2bde062fa + depends: + - __osx >=11.0 + - c-ares >=1.34.8,<2.0a0 + - libcxx >=21 + - libev >=4.33,<4.34.0a0 + - libev >=4.33,<5.0a0 + - libzlib >=1.3.2,<2.0a0 + - openssl >=3.5.7,<4.0a0 + license: MIT + license_family: MIT + run_exports: + weak: + - libnghttp2 >=1.68.1,<2.0a0 + size: 567024 + timestamp: 1787177422467 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libpython-3.14.7-h4311e70_106_cp314.conda + build_number: 106 + sha256: b239028180e1ba2f7daff4053c89e41f7c0f1a3c21bb82f6dcb09384d123ec1f + md5: 4d7a303343eff82e8b92d52c5fd3991a + depends: + - __osx >=11.0 + - libcxx >=20 + license: Python-2.0 + run_exports: {} + size: 1875368 + timestamp: 1788383432953 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsodium-1.0.22-h74c22ad_3.conda + sha256: 0fb001d9b28dc19096388beadaed3af98100f4260707ecb9c177fae68b0e9ece + md5: b237804d151d3d05d4d7fd9052c5664b + depends: + - __osx >=11.0 + license: ISC + run_exports: + weak: + - libsodium >=1.0.22,<1.0.23.0a0 + size: 249403 + timestamp: 1789425371505 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.53.4-hca69786_1.conda + sha256: 839b31d4830e896b4d315b551d27f2bb08026bc946df04bcc360d8627c3ba2cd + md5: fde96d40ebe9a9cb34e4122380189ddb + depends: + - __osx >=11.0 + - icu >=78.3,<79.0a0 + - libzlib >=1.3.2,<2.0a0 + license: blessing + run_exports: + weak: + - libsqlite >=3.53.4,<4.0a0 + size: 942754 + timestamp: 1787051243846 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libuv-1.52.1-h1a92334_1.conda + sha256: 4f47de9de1990efd998edbbd6793f89c8f02ffde987ae8120b1c006acefd2a04 + md5: de09bd0f175611e94f21b28f8c708e80 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + run_exports: + weak: + - libuv >=1.52.1,<2.0a0 + size: 122729 + timestamp: 1785914645797 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + sha256: a18fa5d5bac452401459f966cf0d872224e8080c4ff93c77e168d43ab42ef9d7 + md5: f39288f0ea63ae962e1a2e4f355a0d75 + depends: + - __osx >=11.0 + constrains: + - zlib 1.3.2 *_3 + license: Zlib + license_family: Other + run_exports: + weak: + - libzlib >=1.3.2,<2.0a0 + size: 47822 + timestamp: 1785277049190 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.6-he64c551_1.conda + sha256: 7024a48c8c0d0114ed4ab53c76bf9275d50e91ba7cea367a9aead638d3c29c68 + md5: 3dfa0d0316dc246cd44937a557de4501 + depends: + - __osx >=11.0 + license: X11 AND BSD-3-Clause + run_exports: + weak: + - ncurses >=6.6,<7.0a0 + size: 804298 + timestamp: 1786355189145 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/nodejs-26.10.0-h8d74fab_0.conda + sha256: e08552a430a3683f1cbc3bc8934449e301cd3ad6cc38fe34627de0301893ab44 + md5: 2f4e4682e0a68bd0e15a7964d6ceb73b + depends: + - __osx >=12.0 + - libcxx >=21 + - c-ares >=1.34.8,<2.0a0 + - libbrotlicommon >=1.2.0,<1.3.0a0 + - libbrotlienc >=1.2.0,<1.3.0a0 + - libbrotlidec >=1.2.0,<1.3.0a0 + - libsqlite >=3.53.4,<4.0a0 + - openssl >=3.5.8,<4.0a0 + - zstd >=1.5.7,<1.6.0a0 + - icu >=78.3,<79.0a0 + - libnghttp2 >=1.68.1,<2.0a0 + - libzlib >=1.3.2,<2.0a0 + - libuv >=1.52.1,<2.0a0 + - libabseil >=20260526.0,<20260527.0a0 + - libabseil * cxx17* + license: MIT + license_family: MIT + run_exports: + weak: + - nodejs >=26.10.0,<27.0a0 + size: 18529792 + timestamp: 1790238394580 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.4-h55eecbc_0.conda + sha256: f23239eacd75c4c50705e68fae1aa3292da473e6a3a4abe2330f1e6afa680704 + md5: ae71ab40048c19a389a7dcccb86c2481 + depends: + - __osx >=11.0 + - ca-certificates + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - openssl >=3.6.4,<4.0a0 + size: 3110142 + timestamp: 1787698648639 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.14.7-h0ae1c2c_106_cp314.conda + build_number: 106 + sha256: 5ffc8349ca089ec5ac39ad36a095323ad177ebe05cbfc690c1e135560cfd68b7 + md5: 5fd37c78250b4f57c2bcf9ad2e54beb1 + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.8.1,<3.0a0 + - libffi >=3.7.0,<3.8.0a0 + - liblzma >=5.8.3,<6.0a0 + - libmpdec >=4.0.0,<5.0a0 + - libpython 3.14.7 h4311e70_106_cp314 + - libsqlite >=3.53.4,<4.0a0 + - libzlib >=1.3.2,<2.0a0 + - ncurses >=6.6,<7.0a0 + - openssl >=3.5.8,<4.0a0 + - python_abi 3.14.* *_cp314 + - readline >=8.3,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - zstd >=1.5.7,<1.6.0a0 + license: Python-2.0 + run_exports: + weak: + - python_abi 3.14.* *_cp314 + noarch: + - python + size: 12223441 + timestamp: 1788383470498 + python_site_packages_path: lib/python3.14/site-packages +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyzmq-27.2.0-py312hdfb5542_2.conda + noarch: python + sha256: 42941c85f5f6435ccd84e9aed37de8fd9b3b40ab1c14ae2542ffb4c5e1c84e4e + md5: 8f718ff26a7f93e460445351c20929ef + depends: + - python + - libcxx >=21 + - __osx >=11.0 + - zeromq >=4.3.5,<4.4.0a0 + - _python_abi3_support 1.* + - cpython >=3.12 + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 197695 + timestamp: 1789730185188 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h8b90a29_1.conda + sha256: d6782b430e6dce4fe8c7ac118cd988d5a193eb4a7f60741edfaede58016b6110 + md5: 9347fd56a002e6b58946831d48fe62f6 + depends: + - __osx >=11.0 + - ncurses >=6.6,<7.0a0 + license: GPL-3.0-only + license_family: GPL + run_exports: + weak: + - readline >=8.3,<9.0a0 + size: 314395 + timestamp: 1787033878899 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-hbeba79b_4.conda + sha256: 857d89087ae7f2c328cf256728affcb7343031a148b4af847334d248a9cf564c + md5: 90a01bf394d1c594e0eb9258f967d828 + depends: + - __osx >=11.0 + - libzlib >=1.3.2,<2.0a0 + license: TCL + run_exports: + weak: + - tk >=8.6.13,<8.7.0a0 + size: 3342183 + timestamp: 1787272852357 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/tornado-6.5.10-py314hcd3153d_0.conda + sha256: b18706fdd0022f0467912956a24d2e28180ba37d14555be5e382bea51869a6ae + md5: 9079ddcc60b0cf5848f665fa0e7088f9 + depends: + - __osx >=11.0 + - python >=3.14,<3.15.0a0 + - python_abi 3.14.* *_cp314 + license: Apache-2.0 + license_family: Apache + run_exports: {} + size: 952431 + timestamp: 1789911042938 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/zeromq-4.3.5-h6eb0046_12.conda + sha256: 4fe0cd73a686cc4466ca005166a5aa351dc665aa26166b68752d90930f214382 + md5: 89102eac6e230f7636c40a1411223484 + depends: + - libcxx >=21 + - __osx >=11.0 + - libsodium >=1.0.22,<1.0.23.0a0 + - krb5 >=1.22.2,<1.23.0a0 + license: MPL-2.0 + license_family: MOZILLA + run_exports: + weak: + - zeromq >=4.3.5,<4.4.0a0 + size: 244219 + timestamp: 1789076600961 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda + sha256: da867f5092eb0cb746d353694f0098031fd9817a4ce7d5743121209ae0f406ca + md5: 4ec2684c73812cc2c3d78379384a39cc + depends: + - __osx >=11.0 + - libzlib >=1.3.2,<2.0a0 + license: BSD-3-Clause + license_family: BSD + run_exports: + weak: + - zstd >=1.5.7,<1.6.0a0 + size: 433687 + timestamp: 1786599629846 +- conda: https://conda.modular.com/max/linux-64/mojo-1.2.0-release.conda + sha256: 18485673c7089a187107633a0b9086c5fcb2f0fbe5daf67af44490271e6a2e57 + md5: 24f2a26d0e2cf92bb7ba4712a030a39d + depends: + - python >=3.10,<3.15 + - mojo-compiler ==1.2.0 + - mblack ==26.7.0 + - jupyter_client >=8.6.2,<8.7 + license: LicenseRef-Modular-Proprietary + run_exports: {} + size: 113694280 + timestamp: 1792188456301 +- conda: https://conda.modular.com/max/linux-64/mojo-compiler-1.2.0-release.conda + sha256: a746d7db173a9882e7073ae6d905acfe383a754faecb4a53710b7b90db377e6b + md5: 7029fcd79f3dc5573e9c73eae92b37e0 + depends: + - mojo-python ==1.2.0 + license: LicenseRef-Modular-Proprietary + run_exports: {} + size: 71258238 + timestamp: 1792188428581 +- conda: https://conda.modular.com/max/noarch/mblack-26.7.0-release.conda + noarch: python + sha256: 4f2478dfce255de9825609cc199b84920e78bdc631624bb96f132ddd0abe766c + md5: 5759078a3ebbe08e3339a60ae876a02f + depends: + - python >=3.10,<3.15 + - click >=8.0.0 + - mypy_extensions >=0.4.3 + - packaging >=22.0 + - pathspec >=1.1.0 + - platformdirs >=2 + - tomli >=1.1.0 + license: LicenseRef-Modular-Proprietary + run_exports: {} + size: 141260 + timestamp: 1792101242288 +- conda: https://conda.modular.com/max/noarch/mojo-python-1.2.0-release.conda + noarch: python + sha256: 52905cdbe15595db926615438dd3bc8a74a5277e55c4e0821ccf45ee3719cdb7 + md5: f5bca66e7717194164db37455fd5ed2d + depends: + - python >=3.10,<3.15 + license: LicenseRef-Modular-Proprietary + run_exports: {} + size: 26506 + timestamp: 1792101239805 +- conda: https://conda.modular.com/max/osx-arm64/mojo-1.2.0-release.conda + sha256: 15a11e832acae6cb52b32541d35d4feab608f265c6f8fb4b2d3569d7eeebd55c + md5: fb8048363ef43fd17dd38e4837041d7c + depends: + - python >=3.10,<3.15 + - mojo-compiler ==1.2.0 + - mblack ==26.7.0 + - jupyter_client >=8.6.2,<8.7 + license: LicenseRef-Modular-Proprietary + run_exports: {} + size: 104053018 + timestamp: 1792188836245 +- conda: https://conda.modular.com/max/osx-arm64/mojo-compiler-1.2.0-release.conda + sha256: 1f5811bc7ff86dc8a684ddab6d9612a966529935fa184e204990323e573228de + md5: 512e3e5f027823618cee6b513d92dc39 + depends: + - mojo-python ==1.2.0 + license: LicenseRef-Modular-Proprietary + run_exports: {} + size: 63449264 + timestamp: 1792188854801 diff --git a/pixi.toml b/pixi.toml new file mode 100644 index 0000000..bd419a5 --- /dev/null +++ b/pixi.toml @@ -0,0 +1,78 @@ +# pplus-core — every command a person runs, in one place. +# +# pixi run [args] list them all: `pixi task list` +# +# The README's "Commands" table lists every task here; tests/tasks.test.mjs +# keeps the two in step. Without pixi, `npm install && npm test` builds and +# tests core with whatever node and python3 are on the PATH; CI runs +# `pixi run ci` in the pinned environment below (docs/ci.md). + +[workspace] +authors = ["Alex Maldonado "] +channels = ["https://conda.modular.com/max/", "conda-forge"] +name = "pplus-core" +platforms = ["linux-64", "osx-arm64"] +license = "Apache-2.0" +license-file = "LICENSE.md" +version = "0.2.0" + +[dependencies] +mojo = "==1.2.0" # the compiler the corpus is pinned to (library/mojo/pin.json) +nodejs = ">=26.10.0,<26.11" # the build and tests; the quest processor needs >= 22.6 +python = ">=3.14.7,<3.15" # the corpus pipeline and the Dafny pin (standard library only) + +[tasks] +# --- build and test -------------------------------------------------------- +install = { cmd = "npm install", inputs = ["package.json"], outputs = ["node_modules/.package-lock.json"], description = "Install TypeScript (the only npm dependency)" } +build = { cmd = "npx tsc -p tsconfig.json", depends-on = ["install"], description = "Compile src/ to dist/, with type declarations for pplus-web" } +watch = { cmd = "npx tsc -p tsconfig.json --watch", depends-on = ["install"], description = "Rebuild dist/ on every change" } +check = { cmd = "npx tsc -p tsconfig.json --noEmit", depends-on = ["install"], description = "Type-check only" } +test = { cmd = "node --test --test-concurrency=8 tests/*.test.mjs", depends-on = ["build"], description = "Build, then run every test (the corpus pipeline's and the Dafny pin's Python tests included)" } +cli = { cmd = "node scripts/cli.mjs", depends-on = ["build"], description = "pplus check|run|prove|fuzz|translate : core without a browser, e.g. pixi run cli prove library/pseudo/programs/27-quantifiers.pseudo" } +prove-fuzz = { cmd = "node scripts/prove-fuzz.mjs", depends-on = ["build"], description = "Fuzz the prover: random procedures with contracts are proved, then run; a proved obligation a run breaks is a finding, e.g. pixi run prove-fuzz --seed 7 --programs 500 (docs/ci.md)" } +engines-diff = { cmd = "node scripts/engines-diff.mjs {{ static }}", args = ["static"], depends-on = ["build"], description = "Show what a course checkout's static/ directory has not taken yet from src/engines/ (docs/engines.md)" } +engines-export = { cmd = "node scripts/engines-export.mjs {{ static }}", args = ["static"], depends-on = ["build"], description = "Copy the engines into a course checkout's static/js/, TypeScript modules as built (docs/engines.md)" } + +# --- Dafny, pinned (scripts/dafny.py) --------------------------------------- +dafny-fetch = { cmd = "python scripts/dafny.py fetch", description = "Download the pinned Dafny into .tools/, checked by SHA-256 (network, once)" } +dafny = { cmd = "python scripts/dafny.py", description = "Run the pinned Dafny with the arguments given, e.g. pixi run dafny verify file.dfy (fetches it first if missing)" } + +# --- the library (library/, the pplus-library submodule; docs/library.md) --- +library-check = { cmd = "node scripts/library.mjs check", depends-on = ["build"], description = "Check every library routine at every level (cases, prover, fuzzer, generated Mojo, every reference, the Dafny proof); writes library/status.json and STATUS.md" } +library-emit = { cmd = "node scripts/library.mjs emit", depends-on = ["build"], description = "Regenerate src/library/generated.ts, the registry USE reads, from library/" } +library-verify-mojo = { cmd = "node scripts/library.mjs mojo-sources | python scripts/library_godbolt.py", depends-on = ["build"], description = "Ask the pinned Mojo compiler about every Mojo reference of the library and store the answers (network)" } +library-harness = { cmd = "node scripts/library.mjs harness {{ id }} {{ lang }} {{ level }}", args = ["id", "lang", "level"], depends-on = ["build"], description = "Print the program a reference implementation is run as, e.g. pixi run library-harness md.LennardJones c multiply" } +library-harness-input = { cmd = "node scripts/library.mjs harness {{ id }} {{ lang }} {{ level }} --input", args = ["id", "lang", "level"], depends-on = ["build"], description = "Print the line of stdin a C or Mojo reference's program reads: every case's arguments, e.g. pixi run library-harness-input md.LennardJones mojo multiply" } + +# --- write content --------------------------------------------------------- +new-exercise = { cmd = "node scripts/new-exercise.mjs {{ id }}", args = ["id"], description = "Start an exercise: library/mojo/local/.mojo and src/exercises/.ts, registered in index.ts" } +new-probe = { cmd = "node scripts/mojo/new-program.mjs probe {{ name }} \"{{ question }}\"", args = ["name", "question"], description = "Start library/mojo/probes/.mojo: one question about the pinned compiler" } +new-stress = { cmd = "node scripts/mojo/new-program.mjs stress {{ name }} \"{{ about }}\"", args = ["name", "about"], description = "Start library/mojo/stress/.mojo: realistic Mojo that tests the interpreter" } +play = { cmd = "node scripts/play.mjs {{ name }}", args = ["name"], depends-on = ["install"], description = "For library/mojo/local/.mojo: run, verify on godbolt, build, list the assembly, test" } +play-watch = { cmd = "node scripts/play.mjs {{ name }} --watch", args = ["name"], depends-on = ["install"], description = "The same, again on every save (tests off)" } + +# --- the Mojo corpus (docs/mojo-corpus.md) --------------------------------- +mojo-verify = { cmd = "python scripts/mojo/corpus.py verify", description = "Ask godbolt about every program with no answer yet (network)" } +mojo-verify-item = { cmd = "python scripts/mojo/corpus.py verify --force --item '{{ id }}'", args = ["id"], description = "Ask again about one item; a trailing * matches a prefix, e.g. 'probes/life-*' (network)" } +mojo-verify-dry = { cmd = "python scripts/mojo/corpus.py verify --dry-run", description = "Show what mojo-verify would send; send nothing" } +mojo-verify-diagnostics = { cmd = "python scripts/mojo/corpus.py verify --corpus diagnostics", description = "Also check the compiler's own diagnostic tests (opt-in, about 140 files; network)" } +mojo-status = { cmd = "python scripts/mojo/corpus.py status", description = "Judge every item from the stored answers, offline; writes library/mojo/golden/STATUS.md" } +mojo-status-strict = { cmd = "python scripts/mojo/corpus.py status --strict", description = "The same, and fail if a program we wrote (probes, local, stress) fails" } +mojo-prune = { cmd = "python scripts/mojo/corpus.py prune", description = "Remove stored answers nothing reads any more: those of a compiler other than the pin, and those of programs since edited or removed (git keeps them)" } +mojo-harvest = { cmd = "python scripts/mojo/corpus.py harvest", description = "After editing library/mojo/pin.json: fetch Modular's sources, process them, regenerate src/mojo/generated/ and syntax/ (network)" } +mojo-harvest-local = { cmd = "python scripts/mojo/corpus.py harvest --local-dir {{ dir }}", args = [{ arg = "dir", default = "zips" }], description = "Harvest, using zips in for sources the pin leaves UNRESOLVED" } +mojo-harvest-offline = { cmd = "python scripts/mojo/corpus.py harvest --local-dir {{ dir }} --offline", args = [{ arg = "dir", default = "zips" }], description = "Harvest from zips in only, no network" } +mojo-fetch = { cmd = "python scripts/mojo/corpus.py fetch --update", description = "Re-resolve every ref in the pin and rewrite library/mojo/mojo.lock.json (network)" } +mojo-emit = { cmd = "python scripts/mojo/corpus.py emit", description = "Regenerate src/mojo/generated/ and library/syntax/mojo.xml from the pin and the corpus, offline" } +mojo-test = { cmd = "python -m unittest discover -s scripts/mojo/tests", description = "The corpus pipeline's own tests (also run by test)" } +mojo-run = { cmd = "python scripts/mojo/corpus.py mojo-run {{ file }}", args = ["file"], description = "Run one Mojo file on the pinned local compiler, built with the pin's build arguments and then the file's own, e.g. library/mojo/probes/int-floordiv.mojo" } +mojo-local = { cmd = "node scripts/mojo/local.mjs", depends-on = ["build"], description = "Build every Mojo program core holds or generates on the pinned local compiler and hold the checker, the interpreter and the translator to it (no network)" } +mojo-fuzz = { cmd = "node scripts/mojo/fuzz.mjs", depends-on = ["build"], description = "Fuzz the pseudo-to-Mojo translator, the checker and the interpreter against the pinned local compiler with generated programs, e.g. pixi run mojo-fuzz --seed 7 --programs 50 (expressions) or pixi run mojo-fuzz --statements --seed 7 (whole programs)" } +mojo-probes-local = { cmd = "for f in library/mojo/probes/*.mojo; do echo \"== $f\"; python scripts/mojo/corpus.py mojo-run \"$f\" < /dev/null; echo \"exit $?\"; done", description = "Run every probe on the local compiler (a quick look; godbolt's answers stay the reference)" } +mojo-update = { depends-on = ["mojo-harvest", "mojo-verify", "mojo-status", "test"], description = "After editing library/mojo/pin.json: harvest, verify, status, test" } + +# --- ship ------------------------------------------------------------------ +ci = { depends-on = ["test", "mojo-status-strict", "library-check", "mojo-local", "ci-fuzz"], description = "What CI runs on every push: every test, no program of ours failing, every library routine checked, every Mojo program built by the pinned compiler, and a short fuzz (docs/ci.md)" } +ci-fuzz = { cmd = "node scripts/ci-fuzz.mjs", depends-on = ["build"], description = "The fuzzers' share of ci: mojo-fuzz and prove-fuzz on seed 1 and on a seed that moves with the CI run number or the date, each printed (docs/ci.md)" } +nightly = { cmd = "node scripts/ci-fuzz.mjs --nightly", depends-on = ["build", "mojo-status-strict"], description = "What the scheduled CI runs: mojo-status-strict, then mojo-fuzz and prove-fuzz longer, on four seeds that move with the date (docs/ci.md)" } +package = { cmd = "python scripts/package.py", description = "Zip the files git tracks (and new ones it does not ignore) into package/ for sharing" } diff --git a/scripts/asm-census.mjs b/scripts/asm-census.mjs new file mode 100644 index 0000000..ff71982 --- /dev/null +++ b/scripts/asm-census.mjs @@ -0,0 +1,70 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The assembly census: every function in every compiled listing the Mojo + * corpus holds (library/mojo/golden, the answers godbolt gave with `asm`), sent to + * the model machine the way "Send to x86-64 pane" sends it, and what the + * assembler said. Run after `pixi run build`: + * + * node scripts/asm-census.mjs # the summary and the reasons + * node scripts/asm-census.mjs --ops # every mnemonic, implemented or not, with counts + * + * docs/asm-coverage.md is this census written up; tests/asm-coverage.test.mjs + * holds the numbers as a ratchet. + */ +import fs from 'node:fs'; +import { forSimulator, intelListing } from '../dist/mojo/att.js'; +import { listingFunctions } from '../dist/services/godbolt.js'; +import { parseAsm } from '../dist/services/asm.js'; +import { OPS } from '../dist/engines/asm86/interp.js'; + +const R = new URL('..', import.meta.url).pathname; + +export function census() { + const ops = new Map(); // mnemonic -> { total, user, example } + const reasons = new Map(); // normalized assembler message -> count + let user = 0, userOk = 0, lib = 0, libOk = 0; + const userFailures = []; + for (const f of fs.readdirSync(R + 'library/mojo/golden')) { + if (!f.endsWith('.json')) continue; + const d = JSON.parse(fs.readFileSync(R + 'library/mojo/golden/' + f, 'utf8')); + if (!d.result?.asm) continue; + const fns = listingFunctions(d.result.asm); + for (const fn of fns) { + if (fn.kind === 'data') continue; + for (const r of intelListing(fn, fns)) { + if (!r.intel || r.intel.endsWith(':')) continue; + const op = r.intel.trim().replace(/^lock\s+/, '').split(/\s+/)[0]; + const e = ops.get(op) ?? { total: 0, user: 0, example: r.intel.trim() }; + e.total++; if (fn.kind === 'user') e.user++; + ops.set(op, e); + } + const { source } = forSimulator(fns, fn); + const p = parseAsm(source); + if (fn.kind === 'user') { user++; if (p.ok) userOk++; else userFailures.push(`${d.firstItem} ${fn.name}: ${p.diagnostics[0]?.message}`); } + else { lib++; if (p.ok) libOk++; } + for (const e of p.diagnostics.filter((x) => x.severity === 'error')) { + const k = e.message.replace(/line \d+: /, '').replace(/"[^"]*"/g, '…').replace(/\[rip \+ \S+/, '[rip + …').replace(/\(.*\)$/, ''); + reasons.set(k, (reasons.get(k) || 0) + 1); + } + } + } + const missing = [...ops].filter(([op]) => !OPS[op]).sort((a, b) => b[1].total - a[1].total); + return { ops, missing, reasons, user, userOk, lib, libOk, userFailures }; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const c = census(); + console.log(`${c.ops.size} distinct instructions in the corpus listings; ${c.missing.length} outside the model machine's set`); + console.log(`user functions: ${c.userOk} of ${c.user} assemble; library and runtime: ${c.libOk} of ${c.lib}`); + for (const f of c.userFailures) console.log(' user function refused:', f); + console.log('\nwhy the rest are refused:'); + for (const [k, n] of [...c.reasons].sort((a, b) => b[1] - a[1])) console.log(` ${String(n).padStart(4)} ${k}`); + if (process.argv.includes('--ops')) { + console.log('\ninstructions outside the set (count, in user functions, example):'); + for (const [op, e] of c.missing) console.log(` ${String(e.total).padStart(5)} ${String(e.user).padStart(4)} ${op.padEnd(14)} ${e.example}`); + console.log('\nevery instruction:'); + for (const [op, e] of [...c.ops].sort((a, b) => b[1].total - a[1].total)) console.log(` ${String(e.total).padStart(5)} ${op.padEnd(14)} ${OPS[op] ? '' : 'NOT MODELED'}`); + } +} diff --git a/scripts/ci-fuzz.mjs b/scripts/ci-fuzz.mjs new file mode 100644 index 0000000..2b89076 --- /dev/null +++ b/scripts/ci-fuzz.mjs @@ -0,0 +1,66 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The fuzzers as CI runs them: a fixed seed, so a failure that comes back is the same failure, and seeds that move, + * so every run looks somewhere new. Every seed is printed with the command that reproduces it. + * + * node scripts/ci-fuzz.mjs the gate's share (pixi run ci): a few minutes + * mojo-fuzz seed 1 and the moving seed, 25 programs of expressions each + * mojo-fuzz --statements seed 1 and the moving seed, 25 whole programs each + * prove-fuzz seed 1 and the moving seed, 300 programs each + * node scripts/ci-fuzz.mjs --nightly the scheduled run (pixi run nightly): about half an hour + * mojo-fuzz four moving seeds, 100 programs each + * mojo-fuzz --statements four moving seeds, 100 whole programs each (the size at which the first run of + * three seeds found a fault that 40 programs a seed had not) + * prove-fuzz four moving seeds, 3,000 programs each + * node scripts/ci-fuzz.mjs --seed N the same, with N as the moving seed + * node scripts/ci-fuzz.mjs --dry-run print the commands and do nothing + * + * The moving seed is the CI run's number when the CI says it (GITHUB_RUN_NUMBER, which Gitea Actions sets too), or + * else today's date in UTC as YYYYMMDD; the nightly seeds are the date and the three days after it, so each night + * starts where the last one would have gone next. Every fuzzer runs even when an earlier one fails; the exit status + * is 1 when any of them found something, 2 when one could not run. + */ +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const ROOT = fileURLToPath(new URL('..', import.meta.url)); +const argv = process.argv.slice(2); +const opt = (name) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : null; }; +const flag = (name) => argv.includes(`--${name}`); +const nightly = flag('nightly'); + +const today = Number(new Date().toISOString().slice(0, 10).replace(/-/g, '')); +const runNumber = Number(process.env.GITHUB_RUN_NUMBER); +let moving; +if (opt('seed') !== null) moving = { seed: Number(opt('seed')), from: '--seed' }; +else if (!nightly && Number.isInteger(runNumber) && runNumber > 0) moving = { seed: runNumber, from: 'the CI run number' }; +else moving = { seed: today, from: "today's date (UTC)" }; +if (!Number.isInteger(moving.seed) || moving.seed < 0) { console.error(`ci-fuzz: a seed is a whole number; got ${opt('seed')}`); process.exit(2); } + +const seeds = nightly ? [0, 1, 2, 3].map((k) => moving.seed + k) : [1, moving.seed]; +const rounds = [ + ...seeds.map((s) => ({ name: 'mojo-fuzz', script: 'scripts/mojo/fuzz.mjs', args: ['--seed', String(s), '--programs', nightly ? '100' : '25'] })), + ...seeds.map((s) => ({ name: 'mojo-fuzz --statements', script: 'scripts/mojo/fuzz.mjs', args: ['--seed', String(s), '--programs', nightly ? '100' : '25', '--statements'] })), + ...seeds.map((s) => ({ name: 'prove-fuzz', script: 'scripts/prove-fuzz.mjs', args: ['--seed', String(s), '--programs', nightly ? '3000' : '300'] })), +]; + +console.log(`ci-fuzz${nightly ? ' --nightly' : ''}: the moving seed is ${moving.seed}, from ${moving.from}; seeds ${seeds.join(', ')}`); +const results = []; +for (const r of rounds) { + const line = `node ${r.script} ${r.args.join(' ')}`; + console.log(`\n== ${r.name} seed ${r.args[1]} (reproduce: ${line})`); + if (flag('dry-run')) continue; + const started = Date.now(); + const p = spawnSync(process.execPath, [path.join(ROOT, r.script), ...r.args], { cwd: ROOT, stdio: 'inherit' }); + results.push({ ...r, line, status: p.status ?? 2, seconds: Math.round((Date.now() - started) / 1000) }); +} +if (flag('dry-run')) process.exit(0); + +console.log('\nci-fuzz summary'); +for (const r of results) console.log(` ${r.status === 0 ? 'clean ' : r.status === 1 ? 'FOUND ' : 'FAILED'} ${r.name} seed ${r.args[1]} (${r.seconds} s)${r.status ? `: ${r.line}` : ''}`); +const worst = results.some((r) => r.status === 1) ? 1 : results.some((r) => r.status !== 0) ? 2 : 0; +console.log(worst === 0 ? 'every fuzzer is clean' : 'reproduce a seed with the command beside it (docs/ci.md says what to do next)'); +process.exit(worst); diff --git a/scripts/cli.mjs b/scripts/cli.mjs new file mode 100644 index 0000000..a452423 --- /dev/null +++ b/scripts/cli.mjs @@ -0,0 +1,109 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * pplus: pplus-core without a browser. The same engines, translators, prover and fuzzer the app runs, on a file. + * + * pixi run cli check what the language's own reader says + * pixi run cli run [--call C] [--input F] run it, print what it prints + * pixi run cli prove [--speco F] every obligation: proved, broken or open + * pixi run cli fuzz [--seed N] [--budget N] the fuzzer's report per routine + * pixi run cli translate --to mojo|lingo|matho|speco + * + * The language comes from the extension: .pseudo, .lingo, .matho (or .tex), .speco, .mojo, .asm (or .s). Add --json + * for machine-readable output. Exit status: 0 when nothing is wrong, 1 when the command found a problem (an error, + * a broken obligation, a failing input, a run that stopped), 2 for a usage error. Build first (`pixi run build`): + * this reads dist/. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const ROOT = fileURLToPath(new URL('..', import.meta.url)); +const dist = (p) => pathToFileURL(path.join(ROOT, 'dist', p)).href; +const usage = (why) => { + if (why) console.error(`pplus: ${why}`); + console.error('usage: pplus [--call C] [--input F] [--speco F] [--seed N] [--budget N] [--to LANG] [--json]'); + process.exit(2); +}; +if (!fs.existsSync(path.join(ROOT, 'dist', 'common', 'workspace.js'))) usage('dist/ is missing; run `pixi run build` first'); + +const [cmd, file, ...rest] = process.argv.slice(2); +if (!cmd || !file) usage(); +const opt = (name) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; }; +const json = rest.includes('--json'); +const LANG_OF = { '.pseudo': 'pseudo', '.lingo': 'lingo', '.matho': 'matho', '.tex': 'matho', '.speco': 'speco', '.mojo': 'mojo', '.asm': 'asm', '.s': 'asm' }; +const lang = LANG_OF[path.extname(file).toLowerCase()]; +if (!lang) usage(`cannot tell the language of ${file} from its extension (${Object.keys(LANG_OF).join(', ')})`); +if (!fs.existsSync(file)) usage(`${file} does not exist`); + +const { Workspace } = await import(dist('common/workspace.js')); +const text = fs.readFileSync(file, 'utf8'); +const sources = { [lang]: text }; +const speco = opt('speco'); +if (speco) sources.speco = fs.readFileSync(speco, 'utf8'); +const input = opt('input') ? fs.readFileSync(opt('input'), 'utf8') : ''; +const ws = new Workspace({ sources, lang, call: opt('call') ?? null, input }); + +const out = (obj, lines) => { if (json) console.log(JSON.stringify(obj, null, 2)); else for (const l of lines) console.log(l); }; +const where = (d) => `${file}:${d.line ?? 1}:${d.col ?? 1}`; + +switch (cmd) { + case 'check': { + const ds = ws.diagnostics(lang); + out(ds, ds.length ? ds.map((d) => `${where(d)}: ${d.severity}: ${d.message}${d.id ? ` [${d.id}]` : ''}`) : [`${file}: no problems`]); + process.exit(ds.some((d) => d.severity === 'error') ? 1 : 0); + } + case 'run': { + const errors = ws.diagnostics(lang).filter((d) => d.severity === 'error'); + if (errors.length) { out({ errors }, errors.map((d) => `${where(d)}: error: ${d.message}`)); process.exit(1); } + const tr = ws.trace(); + if (!tr) { out({ error: ws.runError ?? 'nothing to run' }, [`pplus: ${ws.runError ?? 'nothing to run'}`]); process.exit(1); } + const lines = [...(tr.output ?? [])]; + if (tr.error) lines.push(`stopped at line ${tr.error.line}: ${tr.error.message}`); + out({ output: tr.output, error: tr.error, steps: tr.count }, lines); + process.exit(tr.error ? 1 : 0); + } + case 'prove': { + if (lang !== 'pseudo') usage('prove reads a .pseudo file (with --speco for a separate spec)'); + const p = ws.proof(); + if (!p) { out({ error: 'the pseudo does not parse' }, ['pplus: the pseudo does not parse; run `pplus check` first']); process.exit(1); } + const { report } = p; + const lines = []; + for (const u of report.units) { + lines.push(`${u.name}${u.partial ? ' (partial: not every path was examined)' : ''}`); + for (const o of u.obligations) lines.push(` ${o.status.padEnd(6)} line ${String(o.line).padEnd(4)} ${o.label}`); + for (const n of u.notes) lines.push(` note: ${n}`); + } + lines.push(`${report.counts.proved} proved, ${report.counts.broken} broken, ${report.counts.open} open`); + out(report, lines); + process.exit(report.counts.broken ? 1 : 0); + } + case 'fuzz': { + const reports = ws.fuzz({ seed: Number(opt('seed') ?? 1), budget: Number(opt('budget') ?? 300) }); + const lines = []; + for (const r of reports) { + if (r.problem) { lines.push(`${r.name}: ${r.problem}`); continue; } + lines.push(`${r.name}: ${r.cases.length} cases, ${r.failures.length} failure${r.failures.length === 1 ? '' : 's'}`); + for (const f of r.failures) lines.push(` ${f.call}: ${f.message}`); + } + out(reports.map((r) => ({ name: r.name, problem: r.problem, cases: r.cases.length, failures: r.failures })), lines.length ? lines : [`${file}: no routine to fuzz`]); + process.exit(reports.some((r) => r.failures.length) ? 1 : 0); + } + case 'translate': { + if (lang !== 'pseudo') usage('translate reads a .pseudo file'); + const to = opt('to'); + let result; + if (to === 'mojo') result = ws.derivedMojo(); + else if (to === 'speco') { const { extractSpec } = await import(dist('services/speco.js')); result = { source: extractSpec(text) ?? '' }; } + else if (to === 'lingo' || to === 'matho') { + const { SyncHub } = await import(dist('sync/hub.js')); + const f = new SyncHub({ pseudo: text }).fill(); + result = { source: f.sources[to] ?? '', stale: f.stale[to] }; + } else usage('--to must be mojo, lingo, matho or speco'); + if (result.stale) { out({ refused: result.stale }, [`pplus: ${to} cannot say this program: ${result.stale}`]); process.exit(1); } + if (json) console.log(JSON.stringify(result, null, 2)); else process.stdout.write(result.source.endsWith('\n') ? result.source : result.source + '\n'); + process.exit(0); + } + default: usage(`unknown command ${cmd}`); +} diff --git a/scripts/dafny.py b/scripts/dafny.py new file mode 100644 index 0000000..b29afa9 --- /dev/null +++ b/scripts/dafny.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Dafny, pinned: fetch the release this project uses, check it, and run it. + + pixi run dafny-fetch download Dafny into .tools/ once (network), checked by SHA-256 + pixi run dafny verify file.dfy run the pinned Dafny (fetches first if it is missing) + pixi run dafny --version + +Dafny is not on conda-forge, so pixi cannot install it directly; this script is the pin instead. It downloads the +official release zip for this machine from GitHub, refuses it unless its SHA-256 is the one written below, and unpacks +it into .tools/dafny-/ (ignored by git). The zip is self-contained: it carries its own .NET runtime and the Z3 +builds Dafny was tested with, so the solver is pinned too. To move to another Dafny, change VERSION and every hash in +RELEASES in one commit, and expect every stored proof answer to be asked again. + +Standard library only, like the corpus pipeline. +""" +from __future__ import annotations + +import hashlib +import os +import platform +import shutil +import stat +import subprocess +import sys +import urllib.request +import zipfile +from pathlib import Path + +VERSION = "4.11.0" +# platform -> (release asset, SHA-256 of the zip) +RELEASES = { + ("Linux", "x86_64"): ("dafny-4.11.0-x64-ubuntu-22.04.zip", "a46a9ff7cdd720f7955854c78e95df13f4cfe6b80691b05f8654fe19e8267179"), + ("Darwin", "arm64"): ("dafny-4.11.0-arm64-macos-13.zip", "c90c75e7d5db9c6ccbb7127840dfe43f0ac938b039a7ebed146d8ead383a572f"), +} +URL = "https://github.com/dafny-lang/dafny/releases/download/v{version}/{asset}" + +ROOT = Path(__file__).resolve().parent.parent +TOOLS = ROOT / ".tools" +HOME = TOOLS / f"dafny-{VERSION}" +EXE = HOME / "dafny" / "dafny" + + +def release(system: str | None = None, machine: str | None = None) -> tuple[str, str]: + key = (system or platform.system(), machine or platform.machine()) + if key not in RELEASES: + raise SystemExit(f"dafny: no pinned Dafny {VERSION} for {key[0]} {key[1]}; the pinned platforms are " + + ", ".join(f"{s} {m}" for s, m in RELEASES)) + return RELEASES[key] + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for block in iter(lambda: f.read(1 << 20), b""): + h.update(block) + return h.hexdigest() + + +def unpack(zip_path: Path, into: Path) -> None: + """Unzip, keeping each file's Unix permission bits (zipfile drops them), so dafny and z3 stay executable.""" + with zipfile.ZipFile(zip_path) as z: + for info in z.infolist(): + target = into / info.filename + z.extract(info, into) + mode = (info.external_attr >> 16) & 0o777 + if mode and not info.is_dir(): + os.chmod(target, mode | stat.S_IRUSR) + + +def fetch(quiet: bool = False) -> Path: + """Make sure the pinned Dafny is in .tools/, and return its executable.""" + if EXE.exists(): + return EXE + asset, want = release() + TOOLS.mkdir(exist_ok=True) + zip_path = TOOLS / asset + if not zip_path.exists() or sha256(zip_path) != want: + url = URL.format(version=VERSION, asset=asset) + if not quiet: + print(f"dafny: downloading {url}", file=sys.stderr) + part = zip_path.with_suffix(".part") + with urllib.request.urlopen(url) as r, part.open("wb") as f: + shutil.copyfileobj(r, f) + part.replace(zip_path) + got = sha256(zip_path) + if got != want: + zip_path.unlink() + raise SystemExit(f"dafny: {asset} has SHA-256 {got}, not the pinned {want}; refusing it") + staging = TOOLS / f".dafny-{VERSION}.unpacking" + shutil.rmtree(staging, ignore_errors=True) + unpack(zip_path, staging) + staging.replace(HOME) + zip_path.unlink() + if not quiet: + print(f"dafny: Dafny {VERSION} is in {HOME.relative_to(ROOT)}", file=sys.stderr) + return EXE + + +def main(argv: list[str]) -> int: + if argv[:1] == ["fetch"]: + fetch() + print(subprocess.run([str(EXE), "--version"], capture_output=True, text=True).stdout.strip()) + return 0 + if argv[:1] == ["path"]: + print(fetch(quiet=True)) + return 0 + exe = fetch() + return subprocess.run([str(exe), *argv]).returncode + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/engines-diff.mjs b/scripts/engines-diff.mjs new file mode 100644 index 0000000..d9647c4 --- /dev/null +++ b/scripts/engines-diff.mjs @@ -0,0 +1,74 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Compare this repository's engines with a checkout of the course site. + * + * pixi run engines-diff ../course/static + * + * is the course repository's static/ directory. This repository owns + * the engines and the course copies from here (pixi run engines-export), so + * this shows what the course has not taken yet. For each engine module (pseudo, + * asm86, shared) it compares the file the course would receive (src/engines/ + * as written, or dist/engines/ for a module written in TypeScript) with the + * course's copy (the course's stylesheets are compared in pplus-web); + * prints whether they are the same, differ only in whitespace and line + * endings, or differ in content (with the number of changed lines, whitespace + * ignored), and lists files only one side has. It changes nothing. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { engineModules, ENGINE_DIRS } from './engines-files.mjs'; + +const ROOT = fileURLToPath(new URL('..', import.meta.url)); +const dir = process.argv[2]; +if (!dir || !fs.existsSync(path.join(dir, 'js'))) { + console.error('usage: pixi run engines-diff '); + process.exit(2); +} + +const squash = (s) => s.replace(/\r/g, '').split('\n').map((l) => l.replace(/\s+/g, ' ').trim()).filter(Boolean); + +/** Lines changed, whitespace ignored: the size of the symmetric difference of the two line multisets. */ +function changed(a, b) { + const count = new Map(); + for (const l of a) count.set(l, (count.get(l) || 0) + 1); + for (const l of b) count.set(l, (count.get(l) || 0) - 1); + let n = 0; + for (const v of count.values()) n += Math.abs(v); + return n; +} + +function compare(ours, theirs) { + const a = fs.readFileSync(ours, 'utf8'), b = fs.readFileSync(theirs, 'utf8'); + if (a === b) return { kind: 'same' }; + const n = changed(squash(a), squash(b)); + return n === 0 ? { kind: 'whitespace' } : { kind: 'content', n }; +} + +const rows = []; +const strip = (t) => t.replace(/\n\/\/# sourceMappingURL=.*\n?$/, '\n'); +const mine = engineModules(ROOT); +for (const m of mine) { + const theirs = path.join(dir, 'js', m.dir, m.file); + if (!fs.existsSync(m.ours)) rows.push([m.label, 'not built: run pixi run build']); + else if (!fs.existsSync(theirs)) rows.push([m.label, 'only here']); + else { + const a = strip(fs.readFileSync(m.ours, 'utf8')), b = fs.readFileSync(theirs, 'utf8'); + const r = a === b ? { kind: 'same' } : (changed(squash(a), squash(b)) === 0 ? { kind: 'whitespace' } : { kind: 'content', n: changed(squash(a), squash(b)) }); + rows.push([m.label, r.kind === 'content' ? `differs: ${r.n} lines` : r.kind === 'whitespace' ? 'whitespace only' : 'same']); + } +} +for (const d of ENGINE_DIRS) { + const c = path.join(dir, 'js', d); + if (!fs.existsSync(c)) continue; + for (const f of fs.readdirSync(c).filter((x) => x.endsWith('.js')).sort()) { + if (!mine.some((m) => m.dir === d && m.file === f)) rows.push([`js/${d}/${f}`, 'only in the course']); + } +} + +const w = Math.max(...rows.map((r) => r[0].length)); +for (const [f, v] of rows) console.log(`${f.padEnd(w)} ${v}`); +const content = rows.filter((r) => r[1].startsWith('differs')).length; +console.log(`\n${rows.length} files: ${rows.filter((r) => r[1] === 'same').length} same, ${rows.filter((r) => r[1] === 'whitespace only').length} whitespace only, ${content} differ in content`); diff --git a/scripts/engines-export.mjs b/scripts/engines-export.mjs new file mode 100644 index 0000000..a44d751 --- /dev/null +++ b/scripts/engines-export.mjs @@ -0,0 +1,45 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Copy this repository's language engines into a checkout of the course site. + * + * pixi run engines-export ../course/static + * + * This repository owns the engines (docs/engines.md); the course copies from + * here. is the course repository's static/ directory. For each + * engine directory (pseudo, asm86, shared) every module goes to + * /js//: a JavaScript module as written in src/engines/, + * a TypeScript one (the prover, the parser, the checker, the interpreter) as built into dist/engines/, which + * is plain ES-module JavaScript the course's pages can import unchanged. + * Source maps are not copied. Prints what it wrote, and changes nothing else + * in the course. Review the course's diff before committing it there. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { engineModules } from './engines-files.mjs'; + +const ROOT = fileURLToPath(new URL('..', import.meta.url)); +const dir = process.argv[2]; +if (!dir || !fs.existsSync(path.join(dir, 'js'))) { + console.error('usage: pixi run engines-export '); + process.exit(2); +} + +let wrote = 0, same = 0, added = 0; +for (const m of engineModules(ROOT)) { + if (!fs.existsSync(m.ours)) { + console.error(`${m.label}: ${path.relative(ROOT, m.ours)} is missing; run pixi run build first`); + process.exit(1); + } + const target = path.join(dir, 'js', m.dir, m.file); + const text = fs.readFileSync(m.ours, 'utf8').replace(/\n\/\/# sourceMappingURL=.*\n?$/, '\n'); + const before = fs.existsSync(target) ? fs.readFileSync(target, 'utf8') : null; + if (before === text) { same++; continue; } + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, text); + if (before === null) { added++; console.log(`added js/${m.dir}/${m.file}${m.built ? ' (built from TypeScript)' : ''}`); } + else { wrote++; console.log(`updated js/${m.dir}/${m.file}${m.built ? ' (built from TypeScript)' : ''}`); } +} +console.log(`\n${wrote} updated, ${added} added, ${same} already the same`); diff --git a/scripts/engines-files.mjs b/scripts/engines-files.mjs new file mode 100644 index 0000000..d5c1da1 --- /dev/null +++ b/scripts/engines-files.mjs @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The engine modules this repository owns, and the file of each that the course receives: + * src/engines//.js as written, or dist/engines//.js for a module written in + * TypeScript. Shared by engines-diff and engines-export. + */ +import fs from 'node:fs'; +import path from 'node:path'; + +export const ENGINE_DIRS = ['pseudo', 'asm86', 'shared']; + +/** Every engine module: { dir, file, label, ours, built }. */ +export function engineModules(root) { + const out = []; + for (const dir of ENGINE_DIRS) { + const src = path.join(root, 'src/engines', dir); + for (const f of fs.readdirSync(src).sort()) { + if (f.endsWith('.js')) out.push({ dir, file: f, label: `src/engines/${dir}/${f}`, ours: path.join(src, f), built: false }); + else if (f.endsWith('.ts') && !f.endsWith('.d.ts')) { + const js = f.replace(/\.ts$/, '.js'); + out.push({ dir, file: js, label: `src/engines/${dir}/${f}`, ours: path.join(root, 'dist/engines', dir, js), built: true }); + } + } + } + return out; +} diff --git a/scripts/library.mjs b/scripts/library.mjs new file mode 100644 index 0000000..d5d6017 --- /dev/null +++ b/scripts/library.mjs @@ -0,0 +1,718 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The library's tools: check every routine of pplus-library (the submodule library/) at every level, record + * what was found in the library, and generate core's registry from it. + * + * node scripts/library.mjs check [--dir library] [--only stats.Mean] every check; writes status.json, STATUS.md, goldens + * node scripts/library.mjs emit [--check] library/ -> src/library/generated.ts + * node scripts/library.mjs harness [--input] print the program a reference is run as (its stdin) + * node scripts/library.mjs mojo-sources each Mojo reference's program and stdin, for godbolt + * + * The checks of one routine: + * structure routine.pseudo defines exactly the routine routine.json names, and nothing else (USE lines aside) + * pseudo every case runs; its result is recorded and becomes what the references are held to + * prover every obligation of the routine: pass when all are proved, open when some are not, fail when broken + * fuzz the fuzzer finds no input that breaks a contract or stops the run + * mojo the Mojo generated from the pseudo checks, runs in core's interpreter and prints what the pseudo prints + * and for each reference implementation, per way of running it: + * cpython python3 runs it on every case cc -O0, cc -O2 cc compiles and runs it on every case + * interpreter core's Mojo interpreter runs it compiler the pinned mojo on this machine builds and runs it + * godbolt the stored answer of the pinned compiler on Compiler Explorer + * and the proof, if there is one: the pinned Dafny verifies it (or its stored verdict for this exact text). + * + * Needs a build (dist/). Python, a C compiler and the pinned Mojo compiler (pixi brings it) are used when they are + * on the PATH; without them those checks are recorded as skipped, never as passed. A Mojo compiler of another + * version than the pin is not asked: its answer would say nothing about the pin. + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const CORE = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const args = process.argv.slice(2); +const cmd = args[0]; +const opt = (name, dflt = null) => { const i = args.indexOf(`--${name}`); return i >= 0 ? args[i + 1] : dflt; }; +const flag = (name) => args.includes(`--${name}`); +const LIB = path.resolve(opt('dir', path.join(CORE, 'library'))); +const GENERATED = path.join(CORE, 'src', 'library', 'generated.ts'); + +const sha256 = (s) => crypto.createHash('sha256').update(s).digest('hex'); +const readJson = (p) => JSON.parse(fs.readFileSync(p, 'utf8')); +const writeIfChanged = (p, text) => { if (!fs.existsSync(p) || fs.readFileSync(p, 'utf8') !== text) { fs.mkdirSync(path.dirname(p), { recursive: true }); fs.writeFileSync(p, text); return true; } return false; }; + +/* ------------------------------------------------------------------ */ +/* Reading the library */ +/* ------------------------------------------------------------------ */ + +/** Every module and routine on disk, with its sources, and the status last recorded for it. */ +export function load(dir = LIB) { + if (!fs.existsSync(path.join(dir, 'modules'))) throw new Error(`no library at ${dir}: check out the submodule (git submodule update --init) or pass --dir`); + const status = fs.existsSync(path.join(dir, 'status.json')) ? readJson(path.join(dir, 'status.json')).routines || {} : {}; + const modules = []; + for (const m of fs.readdirSync(path.join(dir, 'modules')).sort()) { + const mdir = path.join(dir, 'modules', m); + if (!fs.existsSync(path.join(mdir, 'module.json'))) continue; + const meta = readJson(path.join(mdir, 'module.json')); + const routines = []; + for (const r of fs.readdirSync(mdir).sort()) { + const rdir = path.join(mdir, r); + if (!fs.existsSync(path.join(rdir, 'routine.json'))) continue; + const rm = readJson(path.join(rdir, 'routine.json')); + const id = `${meta.name}.${rm.name}`; + const st = status[id] || {}; + routines.push({ + id, module: meta.name, name: rm.name, dir: path.relative(dir, rdir).split(path.sep).join('/'), + summary: rm.summary || '', tags: rm.tags || [], sources: rm.sources || [], + source: fs.readFileSync(path.join(rdir, 'routine.pseudo'), 'utf8'), + cases: (rm.cases || []).map((c, i) => ({ ...c, ...(st.cases?.[i]?.call === c.call ? { result: st.cases[i].result } : {}) })), + references: (rm.references || []).map((ref) => ({ + lang: ref.lang, level: ref.level, file: ref.file, note: ref.note || '', + ...(ref.rel !== undefined ? { rel: ref.rel } : {}), ...(ref.abs !== undefined ? { abs: ref.abs } : {}), + source: fs.readFileSync(path.join(rdir, ref.file), 'utf8'), + checks: st.references?.[`${ref.lang}/${ref.level}`] || {}, + })), + ...(rm.proof ? { proof: { lang: rm.proof.lang, file: rm.proof.file, note: rm.proof.note || '', source: fs.readFileSync(path.join(rdir, rm.proof.file), 'utf8'), check: st.proof || { status: 'unverified', detail: 'not checked yet' } } } : {}), + checks: st.checks || {}, + }); + } + modules.push({ name: meta.name, title: meta.title, summary: meta.summary, routines }); + } + return modules; +} + +/* ------------------------------------------------------------------ */ +/* Values in other languages */ +/* ------------------------------------------------------------------ */ + +const floatText = (v) => { const s = String(v); return /[.e]/.test(s) || !Number.isFinite(v) ? s : `${s}.0`; }; +const items = (v) => (Array.isArray(v) ? v : v?.data ? v.data : []); + +function pyLiteral(v, t) { + switch (t?.k) { + case 'FLOAT': return floatText(v).replace('e+', 'e'); + case 'INTEGER': return String(v); + case 'BOOL': return v ? 'True' : 'False'; + case 'CHAR': case 'STRING': return JSON.stringify(v); + case 'LIST': case 'ARRAY': return `[${items(v).map((x) => pyLiteral(x, t.of)).join(', ')}]`; + default: throw new Error(`no Python literal for ${t?.k}`); + } +} + +function mojoType(t) { + switch (t?.k) { + case 'FLOAT': return t.bits === 32 ? 'Float32' : 'Float64'; + case 'INTEGER': return t.bits ? `${t.signed ? 'Int' : 'UInt'}${t.bits}` : 'Int'; + case 'BOOL': return 'Bool'; + case 'CHAR': case 'STRING': return 'String'; + case 'LIST': case 'ARRAY': return `List[${mojoType(t.of)}]`; + default: throw new Error(`no Mojo type for ${t?.k}`); + } +} + +/* + * A compiled reference's cases arrive at run time. Its program holds no case: main reads every case's arguments + * from one line of stdin, so the compiler cannot work out a call's result while it builds the program (a call on + * literal arguments it may fold, with its own arithmetic and its own math library; built with contraction on, as + * Mojo is by default, the fold also skips the run-time code's fused multiply-adds: probes/float-contract-runtime), + * and what a runner prints is what the routine computes on inputs it did not know. A Mojo reference's program + * and line go to core's interpreter, the pinned compiler on this machine and godbolt; a C reference's to cc at + * each level. One line, read by one input(): on a pipe, Mojo + * 1.2.0's input() returns the first line and a second call raises EOF (probes/input-pipe-first-line, input-lines). + * + * The line is words joined by single spaces: the number of cases, then each case's arguments in order, then + * `end`. A whole number is its digits. A FLOAT is two whole numbers m and e with x = m * 2^e exactly, m odd and + * below 2^53 (so Float64(m) is exact) and every doubling or halving that rebuilds x exact as well + * (probes/float-scale-exact; C's ldexp): the program gets the case's number bit for bit, with no decimal + * conversion in between. A String is its text with %, space and line breaks written %25, %20, %0A, %0D + * (probes/split-empty-fields, string-replace-escapes). A List is its length, then its items. + */ + +/** The words one value is sent as. */ +function caseWords(v, t) { + switch (t?.k) { + case 'FLOAT': return floatWords(Number(v)); + case 'INTEGER': return [String(v)]; + case 'BOOL': return [v ? '1' : '0']; + case 'CHAR': case 'STRING': return [String(v).replace(/%/g, '%25').replace(/ /g, '%20').replace(/\n/g, '%0A').replace(/\r/g, '%0D')]; + case 'LIST': case 'ARRAY': { const xs = items(v); return [String(xs.length), ...xs.flatMap((x) => caseWords(x, t.of))]; } + default: throw new Error(`no way to send a ${t?.k} at run time`); + } +} + +/** A double as [m, e] with x = m * 2^e exactly, m odd (or 0) and below 2^53; an infinity as 2^1024 overflowing. */ +export function floatWords(x) { + if (Number.isNaN(x)) throw new Error('a NaN argument cannot be sent'); + if (!Number.isFinite(x)) return [x > 0 ? '1' : '-1', '1024']; + if (x === 0) return [Object.is(x, -0) ? '-0' : '0', '0']; + const dv = new DataView(new ArrayBuffer(8)); + dv.setFloat64(0, x); + const hi = dv.getUint32(0); const biased = (hi >>> 20) & 0x7ff; + let m = (BigInt(hi & 0xfffff) << 32n) | BigInt(dv.getUint32(4)); + let e = -1074; // a subnormal: m * 2^-1074 + if (biased) { m |= 1n << 52n; e = biased - 1075; } + while (m % 2n === 0n) { m /= 2n; e++; } + return [`${hi >>> 31 ? '-' : ''}${m}`, String(e)]; +} + +/** The line of stdin a compiled reference's program reads: every case's arguments. */ +export function caseInput(params, calls) { + const words = [String(calls.length), ...calls.flatMap((c) => c.args.flatMap((v, i) => caseWords(v, params[i].type))), 'end']; + return `${words.join(' ')}\n`; +} + +/** The helper that reads one value of type t, and the helpers it needs, defined into `defs`; returns its name. */ +function mojoReader(t, defs) { + const T = mojoType(t); + const name = `read_${T.replace(/List\[/g, 'list_').replace(/\]/g, '').toLowerCase()}`; + if (defs.has(name)) return name; + const head = `def ${name}(words: List[String], mut at: Int) raises -> ${T}:`; + switch (t?.k) { + case 'FLOAT': + if (t.bits === 32) throw new Error('no run-time reader for Float32 yet'); + defs.set(name, `${head} + # x = m * 2^e, sent as m and e: Float64(m) is exact (m is below 2^53), and so is each doubling or halving + var x = Float64(atol(words[at])) + if words[at] == "-0": + x = -x + var e = atol(words[at + 1]) + at += 2 + while e > 0: + x *= 2.0 + e -= 1 + while e < 0: + x *= 0.5 + e += 1 + return x`); + break; + case 'INTEGER': + if (t.bits) throw new Error(`no run-time reader for ${T} yet`); + defs.set(name, `${head}\n at += 1\n return atol(words[at - 1])`); + break; + case 'BOOL': + defs.set(name, `${head}\n at += 1\n return atol(words[at - 1]) != 0`); + break; + case 'CHAR': case 'STRING': + defs.set(name, `${head} + # the words are split at spaces, so a space is sent as %20, a line break as %0A or %0D, and % as %25 + at += 1 + return words[at - 1].replace("%20", " ").replace("%0A", "\\n").replace("%0D", "\\r").replace("%25", "%")`); + break; + case 'LIST': case 'ARRAY': { + const item = mojoReader(t.of, defs); + defs.set(name, `${head} + var n = atol(words[at]) + at += 1 + var items = ${T}() + for _ in range(n): + items.append(${item}(words, at)) + return items^`); + break; + } + default: throw new Error(`no run-time reader for ${t?.k}`); + } + return name; +} + +/** A Mojo reference's program: its source, unchanged, then the readers and a main that reads and prints each case. */ +function mojoHarness(ref, fname, params) { + const defs = new Map(); + const readers = params.map((p) => mojoReader(p.type, defs)); + for (const name of [...defs.keys(), 'main']) if (new RegExp(`\\b${name}\\b`).test(ref.source)) throw new Error(`the reference uses the name ${name}, which its harness defines`); + return `${ref.source.replace(/\s+$/, '')} + + +${[...defs.values()].join('\n\n\n')} + + +def main() raises: + # the cases arrive at run time, on one line of stdin: how many there are, each one's arguments, then end + var words = List[String]() + for w in input().split(" "): + words.append(String(w)) + var at = 1 + for _ in range(atol(words[0])): +${readers.map((r, i) => ` var a${i} = ${r}(words, at)`).join('\n')} + print(${fname}(${readers.map((_, i) => `a${i}`).join(', ')})) + if at != len(words) - 1 or words[at] != "end": + raise Error("the input line does not hold the cases it announces") +`; +} + +/** The readers every C harness has: they take the words of the line caseInput writes, in order. */ +const C_READERS = `/* The cases arrive at run time, on one line of stdin: how many there are, each one's arguments, then end. */ +static char *line_rest; + +static void read_line(void) { + size_t cap = 256, n = 0; + char *line = malloc(cap); + int ch; + while (line && (ch = getchar()) != EOF && ch != '\\n') { + if (n + 1 >= cap) line = realloc(line, cap *= 2); + if (line) line[n++] = (char)ch; + } + if (!line) exit(3); + line[n] = '\\0'; + line_rest = line; +} + +static char *read_word(void) { + if (!line_rest) { fputs("the input line ended early\\n", stderr); exit(3); } + char *word = line_rest, *space = strchr(line_rest, ' '); + if (space) { *space = '\\0'; line_rest = space + 1; } else line_rest = NULL; + return word; +} + +static long read_long(void) { return strtol(read_word(), NULL, 10); } + +/* x = m * 2^e, sent as m and e: (double)m is exact (m is below 2^53), and so is ldexp */ +static double read_double(void) { + const char *m = read_word(); + double x = ldexp((double)strtoll(m, NULL, 10), (int)strtol(read_word(), NULL, 10)); + return strcmp(m, "-0") == 0 ? -x : x; +}`; + +/** The reader a C harness has when the routine takes a CHAR or a STRING. */ +const C_READ_STRING = `/* a space is sent as %20, a line break as %0A or %0D, and % as %25 */ +static char *read_string(void) { + char *word = read_word(), *out = word; + for (const char *p = word; *p; p++) { + if (*p == '%' && p[1] && p[2]) { char hex[3] = { p[1], p[2], '\\0' }; *out++ = (char)strtol(hex, NULL, 16); p += 2; } + else *out++ = *p; + } + *out = '\\0'; + return word; +}`; + +/** The C statements that read one argument (a list as an array and its length), and the expressions it is passed as. */ +function cRead(t, name) { + switch (t?.k) { + case 'FLOAT': return { decls: [`double ${name} = read_double();`], args: [name] }; + case 'INTEGER': return { decls: [`long ${name} = read_long();`], args: [name] }; + case 'BOOL': return { decls: [`int ${name} = read_long() != 0;`], args: [name] }; + case 'CHAR': return { decls: [`char ${name} = read_string()[0];`], args: [name] }; + case 'STRING': return { decls: [`const char *${name} = read_string();`], args: [name] }; + case 'LIST': case 'ARRAY': { + if (t.of?.k !== 'FLOAT' && t.of?.k !== 'INTEGER') throw new Error('C lists hold numbers only'); + const [ct, read] = t.of.k === 'FLOAT' ? ['double', 'read_double'] : ['long', 'read_long']; + return { + decls: [`long ${name}_n = read_long();`, `${ct} *${name} = malloc(sizeof(${ct}) * (${name}_n ? ${name}_n : 1));`, `if (!${name}) return 3;`, `for (long i = 0; i < ${name}_n; i++) ${name}[i] = ${read}();`], + args: [name, `${name}_n`], free: name, + }; + } + default: throw new Error(`no C argument for ${t?.k}`); + } +} + +/** A C reference's program: its source, unchanged, then the readers and a main that reads and prints each case. */ +function cHarness(ref, fname, sub) { + for (const name of ['read_line', 'read_word', 'read_long', 'read_double', 'read_string', 'line_rest', 'main']) if (new RegExp(`\\b${name}\\b`).test(ref.source)) throw new Error(`the reference uses the name ${name}, which its harness defines`); + const fmt = sub.returns?.k === 'FLOAT' ? '%.17g' : '%ld'; + const reads = sub.params.map((p, i) => cRead(p.type, `a${i}`)); + const body = [...reads.flatMap((r) => r.decls), `printf("${fmt}\\n", ${fname}(${reads.flatMap((r) => r.args).join(', ')}));`, ...reads.filter((r) => r.free).map((r) => `free(${r.free});`)]; + return `#include +#include +#include +#include + +${ref.source.replace(/\s+$/, '')} + + +${C_READERS} +${sub.params.some((p) => p.type?.k === 'CHAR' || p.type?.k === 'STRING') ? `\n${C_READ_STRING}\n` : ''} +int main(void) { + read_line(); + long cases = read_long(); + for (long k = 0; k < cases; k++) { +${body.map((l) => ` ${l}`).join('\n')} + } + if (strcmp(read_word(), "end") != 0 || line_rest) { fputs("the input line does not hold the cases it announces\\n", stderr); return 3; } + return 0; +} +`; +} + +/* ------------------------------------------------------------------ */ +/* The checks */ +/* ------------------------------------------------------------------ */ + +const ok = (detail) => ({ status: 'pass', detail }); +const bad = (detail) => ({ status: 'fail', detail }); +const skip = (detail) => ({ status: 'skipped', detail }); +const open = (detail) => ({ status: 'open', detail }); +const unverified = (detail) => ({ status: 'unverified', detail }); + +function have(cmdName) { return spawnSync(cmdName, ['--version'], { encoding: 'utf8' }).status === 0; } + +/** The version of the `mojo` on the PATH, or null when there is none. */ +function mojoVersion() { + const p = spawnSync('mojo', ['--version'], { encoding: 'utf8' }); + return p.status === 0 ? (/Mojo\s+(\S+)/.exec(p.stdout) || [])[1] ?? null : null; +} + +/** + * Build a Mojo program with the local compiler and run it on `input`: { compiled, error, status, stdout }. `args` are + * every build argument, the pin's (mojoBuildArgs in src/services/godbolt.ts). + */ +function buildAndRunMojo(src, dir, args, input = '') { + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'example.mojo'), src); + const b = spawnSync('mojo', ['build', ...args.split(' ').filter(Boolean), 'example.mojo', '-o', 'example'], { cwd: dir, encoding: 'utf8', timeout: 180000 }); + if (b.status !== 0) return { compiled: false, error: ((b.stderr || '').split('\n').find((l) => /error:/.test(l)) || 'the build failed').replace(/^.*example\.mojo:/, 'line ').trim() }; + // core-dump limit 1: a program that traps is not piped to systemd-coredump (see scripts/mojo/local.mjs) + const limited = spawnSync('prlimit', ['--version'], { encoding: 'utf8' }).status === 0; + const r = limited ? spawnSync('prlimit', ['--core=1:1', './example'], { cwd: dir, input, encoding: 'utf8', timeout: 60000 }) : spawnSync('./example', [], { cwd: dir, input, encoding: 'utf8', timeout: 60000 }); + return { compiled: true, status: r.status ?? (r.signal ? `signal ${r.signal}` : 'timed out'), stdout: r.stdout || '' }; +} + +/** Does a printed value agree with the pseudo's, within the reference's tolerance? */ +function agrees(printed, want, t, rel = 0, abs = 0) { + const s = String(printed).trim(); + if (t?.k === 'FLOAT' || t?.k === 'INTEGER') { + const got = Number(s.replace(/^inf$/i, 'Infinity').replace(/^-inf$/i, '-Infinity').replace(/^nan$/i, 'NaN')); + if (Number.isNaN(got) && Number.isNaN(want)) return true; + if (Number.isNaN(got)) return false; + if (got === want) return true; + return Math.abs(got - want) <= Math.max(rel * Math.max(Math.abs(got), Math.abs(want)), abs); + } + if (t?.k === 'BOOL') return ['true', '1'].includes(s.toLowerCase()) === Boolean(want); + return s === String(want); +} + +function compareLines(stdout, wants, t, rel, abs) { + const got = stdout.replace(/\n$/, '').split('\n'); + if (got.length !== wants.length) return `printed ${got.length} line(s) for ${wants.length} case(s)`; + for (let i = 0; i < wants.length; i++) if (!agrees(got[i], wants[i].value, t, rel, abs)) return `case ${i + 1} (${wants[i].call}) printed ${got[i]}, and the pseudo gives ${wants[i].text}`; + return null; +} + +/** + * The program a reference implementation is run as: its source, plus a main that prints each case. A Mojo + * or C reference's program holds no case; it reads them from the line `harnessInput` gives (see caseInput). + */ +export function harness(lang, ref, fname, sub, calls) { + const params = sub.params; + if (lang === 'python') { + return `import importlib.util\nspec = importlib.util.spec_from_file_location("ref", __REF__)\nref = importlib.util.module_from_spec(spec)\nspec.loader.exec_module(ref)\n${calls.map((c) => `print(repr(ref.${fname}(${c.args.map((v, i) => pyLiteral(v, params[i].type)).join(', ')})))`).join('\n')}\n`; + } + if (lang === 'mojo') return mojoHarness(ref, fname, params); + if (lang === 'c') return cHarness(ref, fname, sub); + throw new Error(`no harness for ${lang}`); +} + +/** What a reference's program reads on stdin: every case's arguments for Mojo and C, nothing for Python. */ +export function harnessInput(lang, sub, calls) { + return lang === 'mojo' || lang === 'c' ? caseInput(sub.params, calls) : ''; +} + +export async function checkLibrary(dir = LIB, only = null) { + const { setLibrary } = await import(path.join(CORE, 'dist/library/registry.js')); + const svc = await import(path.join(CORE, 'dist/services/pseudo.js')); + const { parseExpression } = await import(path.join(CORE, 'dist/engines/pseudo/parse.js')); + const { evaluateConstant } = await import(path.join(CORE, 'dist/engines/pseudo/interp.js')); + const { formatValue } = await import(path.join(CORE, 'dist/engines/pseudo/lang.js')); + const { pseudoToMojo, snake } = await import(path.join(CORE, 'dist/sync/pseudo-to-mojo.js')); + const { checkMojo } = await import(path.join(CORE, 'dist/mojo/check.js')); + const { runMojo } = await import(path.join(CORE, 'dist/mojo/interp.js')); + const { sameOutput, outputDistance } = await import(path.join(CORE, 'dist/sync/compare.js')); + // every Mojo program is built with the pin's arguments (library/mojo/pin.json build.args): godbolt's, and on this + // machine the same, less the x86-64 CPU when this machine is not x86-64 + const { mojoBuildArgs } = await import(path.join(CORE, 'dist/services/godbolt.js')); + const godboltArgs = mojoBuildArgs(); + const buildArgs = mojoBuildArgs('', { x86_64: process.arch === 'x64' }); + + const modules = load(dir); + setLibrary(modules); + const pin = readJson(path.join(dir, 'mojo', 'pin.json')); + const dafny = (() => { try { const p = execFileSync('python3', [path.join(CORE, 'scripts', 'dafny.py'), 'path'], { encoding: 'utf8' }).trim(); return fs.existsSync(p) ? p : null; } catch { return null; } })(); + const dafnyVersion = (() => { try { return readJson(path.join(CORE, 'scripts', 'dafny.json')).version; } catch { return null; } })() || (() => { const m = /VERSION = "([^"]+)"/.exec(fs.readFileSync(path.join(CORE, 'scripts', 'dafny.py'), 'utf8')); return m ? m[1] : 'unknown'; })(); + const python = have('python3'); + const cc = have('cc'); + const localMojo = mojoVersion(); + const mojoHere = localMojo === pin.mojo.version; + const noCompiler = localMojo ? `the mojo on the PATH is ${localMojo}, and the pin is ${pin.mojo.version}` : 'no mojo on the PATH (pixi run library-check brings the pinned one)'; + /** Numbers the compiled program may print with other last digits than the pseudo: its own std.math (docs/trust.md). Its arithmetic is the pseudo's: it is built with contraction off (library/mojo/pin.json build.args). */ + const ROUNDING = 1e-9; + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-library-')); + const out = {}; + for (const m of modules) { + for (const r of m.routines) { + if (only && r.id !== only) { continue; } + const rdir = path.join(dir, r.dir); + const rec = { checks: {}, cases: [], references: {}, proof: undefined }; + out[r.id] = rec; + const st = svc.studyOf(r.source); + const program = st.program?.program; + // structure + const subs = program ? [...program.subs.values()] : []; + if (!st.ok) { rec.checks.structure = bad(`the pseudo does not check: ${st.diagnostics.filter((d) => d.severity === 'error').map((d) => `line ${d.line}: ${d.message}`).join('; ')}`); continue; } + if (subs.length !== 1 || subs[0].name.toUpperCase() !== r.name.toUpperCase()) { rec.checks.structure = bad(`routine.pseudo defines ${subs.map((s) => s.name).join(', ') || 'nothing'}; it must define exactly ${r.name} (helpers are routines of their own, brought in with USE)`); continue; } + if (program.main.body.length) { rec.checks.structure = bad(`routine.pseudo has lines outside ${r.name} (line ${program.main.body[0].line}); a routine file holds USE lines and the routine only`); continue; } + const sub = subs[0]; + rec.checks.structure = ok(`defines ${sub.kind === 'Function' ? 'FUNCTION' : 'PROCEDURE'} ${sub.name}${program.libs?.size ? `, using ${[...program.libs.values()].map((x) => x.lib.id).join(', ')}` : ''}`); + // pseudo: every case + const calls = []; + let caseFail = null; + for (const c of r.cases) { + const p = parseExpression(c.call); + let entry = { call: c.call }; + try { + if (p.diags.some((d) => d.severity === 'error') || p.expr.kind !== 'CallExpr' || p.expr.name.toUpperCase() !== r.name.toUpperCase()) throw new Error(`"${c.call}" is not a call of ${r.name}`); + const vals = p.expr.args.map((a) => evaluateConstant(program, a)); + const tr = svc.runPseudo(st, { call: c.call, maxSteps: 2000000 }); + if (tr.error) throw new Error(`${c.call} stops: ${tr.error.message}`); + const result = tr.raw.result; + entry = { call: c.call, result: formatValue(result, sub.returns, { max: 1000 }) }; + calls.push({ call: c.call, args: vals, value: result, text: entry.result }); + } catch (e) { caseFail = caseFail || e.message; } + rec.cases.push(entry); + } + rec.checks.pseudo = caseFail ? bad(caseFail) : ok(`${calls.length} case(s) run`); + // prover + const pr = svc.provePseudo(st); + const obs = pr.units.flatMap((u) => u.obligations); + const broken = obs.filter((o) => o.status === 'broken'); + const notProved = obs.filter((o) => o.status !== 'proved'); + rec.checks.prover = broken.length ? bad(`${broken.length} obligation(s) broken: ${broken.map((o) => `${o.kind}@${o.line} (${o.label})`).join('; ')}`) + : notProved.length ? open(`${obs.length - notProved.length} of ${obs.length} obligation(s) proved; open: ${notProved.map((o) => `${o.kind}@${o.line} (${o.label})`).join('; ')}`) + : ok(`all ${obs.length} obligation(s) proved`); + // fuzz + const props = svc.propertiesPseudo(st, r.name)[0]; + rec.checks.fuzz = !props ? skip('no routine to fuzz') : props.problem ? skip(`the fuzzer cannot generate inputs: ${props.problem}`) + : props.failures.length ? bad(`${props.failures.length} failing input(s), first: ${props.failures[0].call}: ${props.failures[0].message}`) + : ok(`no failure in ${props.checked} generated input(s)`); + // the Mojo generated from the pseudo, on every case + if (calls.length) { + const prog = `USE ${r.module}.${r.name}\n${calls.map((c) => `OUTPUT ${c.call}`).join('\n')}\n`; + const want = svc.runPseudo(svc.studyOf(prog), {}).output.join('\n'); + const mj = pseudoToMojo(prog); + if ('unsupported' in mj) rec.checks.mojo = bad(`the translator refused: ${mj.unsupported.map((d) => d.message).join('; ')}`); + else { + const gaps = mj.notes.filter((n) => n.id === 'translate-gap'); + const ck = checkMojo(mj.source); + const errs = ck.diagnostics.filter((d) => d.severity === 'error'); + if (gaps.length) rec.checks.mojo = skip(`the translator has no Mojo for: ${gaps.map((g) => g.message.replace(/^.*?\(|\)\..*$/g, '')).join('; ')}`); + else if (errs.length) rec.checks.mojo = bad(`the generated Mojo has checker errors: ${errs.map((d) => `L${d.line} ${d.message}`).join(' | ')}`); + else { + const run = runMojo(ck.module, mj.source, { maxSteps: 4000000 }); + // the pinned compiler on this machine, when there is one: the interpreter's word is not the compiler's + const built = mojoHere ? buildAndRunMojo(mj.source, path.join(tmp, `gen-${fname0(r)}`), buildArgs) : null; + const far = built?.compiled && built.status === 0 ? outputDistance(want, built.stdout) : null; + const compiled = !built ? null + : !built.compiled ? bad(`Mojo ${localMojo} rejects the generated Mojo: ${built.error}`) + : built.status !== 0 ? bad(`the generated Mojo, built by Mojo ${localMojo}, stopped (exit ${built.status})`) + : far > ROUNDING ? bad(`the generated Mojo, built by Mojo ${localMojo}, printed ${JSON.stringify(built.stdout.slice(0, 120))}; the pseudo printed ${JSON.stringify(want.slice(0, 120))}`) + : ok(far > 0 ? `built by Mojo ${localMojo}, the generated Mojo prints the pseudo's results to within ${far.toExponential(1)} of each value (the compiler's rounding and std.math)` : `built by Mojo ${localMojo}, the generated Mojo prints what the pseudo prints on every case`); + const interp = run.refused ? null : run.exitCode !== 0 ? bad(`the generated Mojo stopped in core's interpreter (exit ${run.exitCode})`) + : sameOutput(want, run.stdout) ? ok('the generated Mojo prints what the pseudo prints on every case, in core\'s interpreter') + // the interpreter computes std.math's exact functions as Mojo does (hypot, for one), so it is held to the + // pseudo as the compiler is: to the rounding of those functions, not to the last digit + : outputDistance(want, run.stdout) <= ROUNDING ? ok(`in core's interpreter the generated Mojo prints the pseudo's results to within ${outputDistance(want, run.stdout).toExponential(1)} of each value (std.math, computed as Mojo computes it)`) + : bad(`in core's interpreter the generated Mojo printed ${JSON.stringify(run.stdout.slice(0, 120))}; the pseudo printed ${JSON.stringify(want.slice(0, 120))}`); + rec.checks.mojo = compiled?.status === 'fail' ? compiled : interp?.status === 'fail' ? interp + : compiled && interp ? ok(`${compiled.detail}, and in core's interpreter`) + : compiled ? ok(`${compiled.detail} (core's interpreter declines ${run.refused})`) + : interp ? ok(`${interp.detail} (${noCompiler})`) + : skip(`core's Mojo interpreter declines ${run.refused}, and ${noCompiler}`); + } + } + } + // references + const fname = snake(r.name); + for (const ref of r.references) { + const checks = {}; + rec.references[`${ref.lang}/${ref.level}`] = checks; + if (!calls.length) { checks.cases = skip('no case ran in pseudo, so there is nothing to hold it to'); continue; } + const rel = ref.rel ?? 0; const abs = ref.abs ?? 0; + const tol = rel || abs ? ` (within rel ${rel}, abs ${abs})` : ' (exactly)'; + try { + if (ref.lang === 'python') { + if (!python) { checks.cpython = skip('python3 is not on the PATH'); continue; } + const h = harness('python', ref, fname, sub, calls).replace('__REF__', JSON.stringify(path.join(rdir, ref.file))); + const p = spawnSync('python3', ['-c', h], { encoding: 'utf8', timeout: 60000 }); + if (p.status !== 0) { checks.cpython = bad(`python3 stopped: ${(p.stderr || '').trim().split('\n').pop()}`); continue; } + const diff = compareLines(p.stdout, calls, sub.returns, rel, abs); + checks.cpython = diff ? bad(diff) : ok(`agrees with the pseudo on ${calls.length} case(s)${tol}`); + } else if (ref.lang === 'c') { + if (!cc) { checks['cc -O0'] = skip('no C compiler (cc) on the PATH'); continue; } + // the program reads its cases from stdin, so at -O2 too the answer is the compiled code's, not the constant folder's + const src = harness('c', ref, fname, sub, calls); + const stdin = harnessInput('c', sub, calls); + const file = path.join(tmp, `${fname}-${ref.level}.c`); + fs.writeFileSync(file, src); + for (const level of ['-O0', '-O2']) { + const bin = path.join(tmp, `${fname}-${ref.level}${level}`); + const c = spawnSync('cc', ['-std=c11', level, '-o', bin, file, '-lm'], { encoding: 'utf8' }); + if (c.status !== 0) { checks[`cc ${level}`] = bad(`cc ${level} failed: ${(c.stderr || '').trim().split('\n')[0]}`); continue; } + const p = spawnSync(bin, [], { input: stdin, encoding: 'utf8', timeout: 60000 }); + const diff = p.status !== 0 ? `exited ${p.status}` : compareLines(p.stdout, calls, sub.returns, rel, abs); + checks[`cc ${level}`] = diff ? bad(diff) : ok(`agrees with the pseudo on ${calls.length} case(s)${tol}`); + } + } else if (ref.lang === 'mojo') { + // the program reads its cases from stdin, so no runner's answer is the compiler working them out as it builds + const src = harness('mojo', ref, fname, sub, calls); + const stdin = harnessInput('mojo', sub, calls); + const ck = checkMojo(src); + const errs = ck.diagnostics.filter((d) => d.severity === 'error'); + if (errs.length) checks.interpreter = bad(`core's Mojo checker reports: ${errs.map((d) => `L${d.line} ${d.message}`).join(' | ')}`); + else { + const run = runMojo(ck.module, src, { input: [stdin.replace(/\n$/, '')], maxSteps: 4000000 }); + const diff = run.refused ? null : run.exitCode !== 0 ? `stopped (exit ${run.exitCode})` : compareLines(run.stdout, calls, sub.returns, rel, abs); + checks.interpreter = run.refused ? skip(`core's Mojo interpreter declines ${run.refused}`) : diff ? bad(diff) : ok(`agrees with the pseudo on ${calls.length} case(s) in core's interpreter${tol}`); + } + // the pinned compiler on this machine + if (!mojoHere) checks.compiler = skip(noCompiler); + else { + const built = buildAndRunMojo(src, path.join(tmp, `ref-${fname}-${ref.level}`), buildArgs, stdin); + const diff = !built.compiled ? `Mojo ${localMojo} rejects it: ${built.error}` : built.status !== 0 ? `built by Mojo ${localMojo}, it stopped (exit ${built.status})` : compareLines(built.stdout, calls, sub.returns, rel, abs); + checks.compiler = diff ? bad(diff) : ok(`built by Mojo ${localMojo}, agrees with the pseudo on ${calls.length} case(s)${tol}`); + } + // the compiler's stored answer for this exact program + const gfile = path.join(rdir, 'golden', `mojo-${ref.level}.json`); + const g = fs.existsSync(gfile) ? readJson(gfile) : null; + if (!g || g.source_sha256 !== sha256(src) || g.stdin !== stdin || g.args !== godboltArgs || g.compiler !== pin.godbolt.compiler) checks.godbolt = unverified(`no stored answer from ${pin.godbolt.compiler} for this program, input and build arguments: pixi run library-verify-mojo`); + else if (!g.result?.compiled) checks.godbolt = bad(`${pin.godbolt.compiler} did not compile it: ${(g.result?.buildStderr || '').split('\n')[0]}`); + else { + const diff = g.result.exitCode !== 0 ? `exited ${g.result.exitCode}` : compareLines(g.result.stdout, calls, sub.returns, rel, abs); + checks.godbolt = diff ? bad(diff) : ok(`${pin.godbolt.compiler} agrees with the pseudo on ${calls.length} case(s)${tol}`); + } + } else checks.cases = skip(`no runner for ${ref.lang} yet`); + } catch (e) { + checks.harness = bad(`could not build the harness: ${e.message}`); + } + } + // proof + if (r.proof) { + const gfile = path.join(rdir, 'golden', 'dafny.json'); + const src = r.proof.source; + const flags = ['verify', '--resource-limit', '20000000']; + if (dafny) { + const p = spawnSync(dafny, [...flags, path.join(rdir, r.proof.file)], { encoding: 'utf8', timeout: 600000 }); + const summary = /Dafny program verifier finished with (.*)/.exec(p.stdout || '')?.[1] || (p.stdout || p.stderr || '').trim().split('\n').pop(); + const pass = p.status === 0 && / 0 errors/.test(summary); + rec.proof = pass ? ok(`Dafny ${dafnyVersion}: ${summary}`) : bad(`Dafny ${dafnyVersion}: ${summary}`); + writeIfChanged(gfile, `${JSON.stringify({ tool: 'dafny', version: dafnyVersion, flags, source_sha256: sha256(src), verdict: pass ? 'verified' : 'failed', summary }, null, 2)}\n`); + } else { + const g = fs.existsSync(gfile) ? readJson(gfile) : null; + rec.proof = g && g.source_sha256 === sha256(src) ? (g.verdict === 'verified' ? ok(`Dafny ${g.version} (stored answer): ${g.summary}`) : bad(`Dafny ${g.version} (stored answer): ${g.summary}`)) + : unverified('the pinned Dafny is not fetched (pixi run dafny-fetch) and there is no stored verdict for this text'); + } + } + } + } + fs.rmSync(tmp, { recursive: true, force: true }); + return { modules, results: out }; +} + +const ORDER = ['structure', 'pseudo', 'prover', 'fuzz', 'mojo']; +const fname0 = (r) => r.id.replace(/[^A-Za-z0-9]+/g, '-'); + +function statusMarkdown(modules, results) { + const mark = (c) => (!c ? '·' : { pass: 'pass', fail: '**fail**', open: 'open', skipped: 'skipped', unverified: 'unverified' }[c.status] || c.status); + const lines = ['# Library status', '', '', '', + 'Each routine, at each level. *pass*: checked and agrees; *open*: the prover could not finish every obligation (nothing is wrong, nothing is proved); *skipped*: the tool is not available here, or it declines the construct; *unverified*: no stored answer for this exact text yet.', '']; + for (const m of modules) { + lines.push(`## ${m.title} (\`${m.name}\`)`, '', '| Routine | Cases | Prover | Fuzz | Generated Mojo | References | Proof |', '| --- | --- | --- | --- | --- | --- | --- |'); + for (const r of m.routines) { + const res = results[r.id]; + if (!res) continue; + const refs = Object.entries(res.references).map(([k, checks]) => `${k}: ${Object.entries(checks).map(([how, c]) => `${how} ${mark(c)}`).join(', ')}`).join('
'); + lines.push(`| \`${r.id}\` | ${mark(res.checks.pseudo)} | ${mark(res.checks.prover)} | ${mark(res.checks.fuzz)} | ${mark(res.checks.mojo)} | ${refs || '·'} | ${res.proof ? `dafny ${mark(res.proof)}` : '·'} |`); + } + lines.push(''); + } + lines.push('## Details', ''); + for (const m of modules) for (const r of m.routines) { + const res = results[r.id]; + if (!res) continue; + lines.push(`### ${r.id}`, ''); + for (const k of ORDER) if (res.checks[k]) lines.push(`- ${k}: ${res.checks[k].status}. ${res.checks[k].detail}`); + for (const [k, checks] of Object.entries(res.references)) for (const [how, c] of Object.entries(checks)) lines.push(`- ${k}, ${how}: ${c.status}. ${c.detail}`); + if (res.proof) lines.push(`- proof: ${res.proof.status}. ${res.proof.detail}`); + lines.push(''); + } + return lines.join('\n'); +} + +/** + * Every reference of every routine, with what it is run on: { id, ref, dir, fname, sub, calls, make, input }. + * make() is the program (harness) and input() its stdin (harnessInput). Needs a build (dist/). + */ +export async function referencePrograms(dir = LIB) { + const svc = await import(path.join(CORE, 'dist/services/pseudo.js')); + const { setLibrary } = await import(path.join(CORE, 'dist/library/registry.js')); + const { parseExpression } = await import(path.join(CORE, 'dist/engines/pseudo/parse.js')); + const { evaluateConstant } = await import(path.join(CORE, 'dist/engines/pseudo/interp.js')); + const { snake } = await import(path.join(CORE, 'dist/sync/pseudo-to-mojo.js')); + const modules = load(dir); + setLibrary(modules); + const programs = []; + for (const m of modules) for (const r of m.routines) { + const st = svc.studyOf(r.source); + if (!st.ok) continue; + const sub = [...st.program.program.subs.values()][0]; + const calls = r.cases.map((c) => ({ args: parseExpression(c.call).expr.args.map((a) => evaluateConstant(st.program.program, a)) })); + const fname = snake(r.name); + for (const ref of r.references) programs.push({ id: r.id, ref, dir: path.join(dir, r.dir), fname, sub, calls, make: () => harness(ref.lang, ref, fname, sub, calls), input: () => harnessInput(ref.lang, sub, calls) }); + } + return programs; +} + +/* ------------------------------------------------------------------ */ +/* Emitting core's registry */ +/* ------------------------------------------------------------------ */ + +function emitText(dir = LIB) { + const modules = load(dir); + return `// Generated by scripts/library.mjs emit from library/ (pplus-library). Do not edit; run pixi run library-emit.\nimport type { LibraryModule } from './types.js';\n\nexport const LIBRARY_MODULES: LibraryModule[] = ${JSON.stringify(modules, null, 2)};\n`; +} + +/* ------------------------------------------------------------------ */ +/* Command line */ +/* ------------------------------------------------------------------ */ + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + if (cmd === 'check') { + const only = opt('only'); + const { modules, results } = await checkLibrary(LIB, only); + const prior = fs.existsSync(path.join(LIB, 'status.json')) ? readJson(path.join(LIB, 'status.json')).routines || {} : {}; + const merged = only ? { ...prior, ...results } : results; + const sorted = Object.fromEntries(Object.keys(merged).sort().map((k) => [k, merged[k]])); + writeIfChanged(path.join(LIB, 'status.json'), `${JSON.stringify({ comment: 'Written by pixi run library-check in pplus-core. Do not edit.', routines: sorted }, null, 2)}\n`); + writeIfChanged(path.join(LIB, 'STATUS.md'), `${statusMarkdown(modules, sorted)}\n`); + let failed = 0; + for (const [id, res] of Object.entries(results)) { + const all = [...Object.entries(res.checks), ...Object.entries(res.references).flatMap(([k, c]) => Object.entries(c).map(([h, v]) => [`${k} ${h}`, v])), ...(res.proof ? [['proof', res.proof]] : [])]; + for (const [k, c] of all) if (c.status === 'fail') { failed++; console.log(`FAIL ${id} ${k}: ${c.detail}`); } + console.log(`${id}: ${all.map(([k, c]) => `${k}=${c.status}`).join(' ')}`); + } + console.log(failed ? `${failed} failing check(s)` : 'no failing check'); + process.exit(failed ? 1 : 0); + } else if (cmd === 'emit') { + const text = emitText(LIB); + if (flag('check')) { + const cur = fs.existsSync(GENERATED) ? fs.readFileSync(GENERATED, 'utf8') : ''; + if (cur !== text) { console.error('src/library/generated.ts is out of date with library/: run pixi run library-emit'); process.exit(1); } + console.log('src/library/generated.ts is up to date'); + } else console.log(writeIfChanged(GENERATED, text) ? 'wrote src/library/generated.ts' : 'src/library/generated.ts unchanged'); + } else if (cmd === 'harness' || cmd === 'mojo-sources') { + const programs = await referencePrograms(LIB); + if (cmd === 'harness') { + const [id, lang, level] = args.slice(1); + const p = programs.find((x) => x.id === id && x.ref.lang === lang && x.ref.level === level); + if (!p) { console.error(`no reference ${lang}/${level} for ${id}`); process.exit(2); } + process.stdout.write(flag('input') ? p.input() : p.make()); + } else { + // one entry per Mojo reference: the program and the stdin scripts/library_godbolt.py sends, and where its answer goes + const list = programs.filter((p) => p.ref.lang === 'mojo').map((p) => ({ id: `${p.id} mojo/${p.ref.level}`, source: p.make(), stdin: p.input(), golden: path.join(p.dir, 'golden', `mojo-${p.ref.level}.json`) })); + process.stdout.write(`${JSON.stringify(list, null, 2)}\n`); + } + } else { + console.error('usage: node scripts/library.mjs check [--only id] | emit [--check] | harness [--input] | mojo-sources'); + process.exit(2); + } +} diff --git a/scripts/library_godbolt.py b/scripts/library_godbolt.py new file mode 100644 index 0000000..06e7bfc --- /dev/null +++ b/scripts/library_godbolt.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Ask the pinned Mojo compiler (on Compiler Explorer) about every Mojo reference implementation of the library, +and store each answer beside the routine, in golden/mojo-.json. Network; standard library only. + + node scripts/library.mjs mojo-sources | python3 scripts/library_godbolt.py [--force] [--dry-run] + +Each program is the reference's source with a main that reads every case's arguments from one line of stdin and +prints each result (scripts/library.mjs builds both, and sends the same program and line to core's interpreter and +the local compiler): the program holds no case, so the answer is the compiled routine's on run-time inputs, not +the compiler's working out a call on literals while it builds. Each is built with the pin's build.args (library/mojo/ +pin.json), as every Mojo program is. An answer is keyed by the SHA-256 of that exact program, the stdin it was given, +the build arguments and the compiler id, so editing a reference or a case, or the build arguments, makes it +unverified until it is asked again; an answer already stored for the same program, stdin and arguments is not asked +again unless --force. +pixi run library-check then compares the stored output with the pseudo's results. +""" +from __future__ import annotations + +import hashlib +import json +import sys +from datetime import date +from pathlib import Path + +CORE = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(CORE / "scripts" / "mojo")) +from mojocorpus import godbolt # noqa: E402 +from mojocorpus.common import PipelineError, build_args, load_pin # noqa: E402 + + +def main(argv: list[str]) -> int: + force = "--force" in argv + dry = "--dry-run" in argv + pin = load_pin() + base, compiler, timeout = pin.base, pin.compiler, pin.timeout + args = build_args(pin.build) # a reference program has no arguments of its own + items = json.load(sys.stdin) + asked = failed = 0 + for it in items: + src, stdin = it["source"], it["stdin"] + sha = hashlib.sha256(src.encode("utf-8")).hexdigest() + out = Path(it["golden"]) + if out.exists() and not force: + old = json.loads(out.read_text()) + if old.get("source_sha256") == sha and old.get("stdin") == stdin and old.get("args") == args and old.get("compiler") == compiler: + print(f"[library] {it['id']}: stored answer is current") + continue + if dry: + print(f"[library] {it['id']}: would ask {base} ({compiler}), {len(src)} bytes and {len(stdin)} of stdin") + continue + try: + data = godbolt.post(base, compiler, godbolt.request_body(src, "run", stdin=stdin, args=args), timeout=timeout) + except PipelineError as e: # godbolt.post has already retried; leave this one unverified and go on + failed += 1 + print(f"[library] {it['id']}: {e}") + continue + result = godbolt.normalize("run", data) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps({"tool": "godbolt", "compiler": compiler, "mojoVersion": pin.version, + "source_sha256": sha, "stdin": stdin, "args": args, "asked": date.today().isoformat(), "result": result}, + indent=2, ensure_ascii=False) + "\n") + asked += 1 + print(f"[library] {it['id']}: compiled={result['compiled']} exit={result['exitCode']}") + print(f"[library] asked {asked} program(s){f', {failed} failed' if failed else ''}") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/mojo/corpus.py b/scripts/mojo/corpus.py new file mode 100644 index 0000000..3907477 --- /dev/null +++ b/scripts/mojo/corpus.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Build and check Programming+'s Mojo corpus. See docs/mojo-corpus.md. + + corpus.py harvest [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--processors a,b] [--keep] + corpus.py fetch [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--only a,b] + corpus.py emit + corpus.py verify [--corpus quest,manual,probes,local,diagnostics] [--item ID] [--limit N] [--force] [--dry-run] + corpus.py status [--strict [--strict-corpus a,b]] + corpus.py mojo-run FILE # run one program on the local compiler, built as everywhere else + +Typical use: + python3 scripts/mojo/corpus.py harvest # after editing library/mojo/pin.json (needs network) + python3 scripts/mojo/corpus.py harvest --local-dir ~/zips # zips for the sources the pin leaves UNRESOLVED + python3 scripts/mojo/corpus.py harvest --local-dir ~/zips --offline # zips for every source, no network + python3 scripts/mojo/corpus.py verify # ask godbolt about anything not yet verified + python3 scripts/mojo/corpus.py status # offline: judge every item, write library/mojo/golden/STATUS.md +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from mojocorpus import emit_ts, verify # noqa: E402 +from mojocorpus.common import PipelineError, build_args, host_is_x86_64, load_pin, log, rel # noqa: E402 +from mojocorpus.fetch import fetch_all, make_workdir # noqa: E402 +from mojocorpus.harvest import detect_local_zips, harvest # noqa: E402 + + +def _csv(s: str | None) -> set[str] | None: + return {x.strip() for x in s.split(",") if x.strip()} if s else None + + +def _locals(args, pin) -> dict[str, Path]: + out: dict[str, Path] = {} + if getattr(args, "local_dir", None): + out.update(detect_local_zips(pin, Path(args.local_dir).expanduser(), include_resolved=getattr(args, "offline", False))) + for spec in getattr(args, "local", None) or []: + name, sep, path = spec.partition("=") + if not sep: + raise PipelineError(f"--local wants NAME=PATH, got {spec!r}") + out[name] = Path(path).expanduser() + return out + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(prog="corpus.py", description=__doc__.split("\n\n")[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + + def source_opts(p): + p.add_argument("--local", action="append", metavar="NAME=ZIP", help="use a zip on disk for a source instead of downloading it") + p.add_argument("--local-dir", metavar="DIR", help="use every zip in DIR, matched to sources by their marker files") + p.add_argument("--update", action="store_true", help="re-resolve refs (and accept local zips) instead of reusing the lock") + p.add_argument("--allow-hash-change", action="store_true", help="accept a GitHub zip whose hash differs from the lock") + p.add_argument("--offline", action="store_true", help="let --local-dir zips stand in even for sources the pin fetches from GitHub") + + p = sub.add_parser("harvest", help="fetch sources, run processors, write the corpus and generated TypeScript") + source_opts(p) + p.add_argument("--processors", help="only these processors (comma-separated)") + p.add_argument("--keep", action="store_true", help="keep the temporary directory") + p.add_argument("--workdir", help="use this directory instead of a temporary one (kept)") + + p = sub.add_parser("fetch", help="resolve refs, download or take local zips, and write the lock") + source_opts(p) + p.add_argument("--only", help="only these sources (comma-separated)") + + sub.add_parser("emit", help="regenerate src/mojo/generated/*.ts from the pin and corpus") + + p = sub.add_parser("verify", help="ask godbolt about items that have no golden yet (network)") + p.add_argument("--corpus", default=",".join(verify.CORPORA_DEFAULT), help=f"which corpora (default {','.join(verify.CORPORA_DEFAULT)}; also: diagnostics)") + p.add_argument("--item", action="append", help="only this item id (a trailing * matches a prefix)") + p.add_argument("--limit", type=int, help="ask about at most N programs") + p.add_argument("--force", action="store_true", help="ask again even when a golden exists") + p.add_argument("--concurrency", type=int, help="parallel requests (default from the pin; at most 4)") + p.add_argument("--dry-run", action="store_true", help="count and show the first request; send nothing") + + p = sub.add_parser("status", help="judge every item from the goldens (offline)") + p.add_argument("--strict", action="store_true", help=f"exit 1 if an item we wrote fails ({','.join(verify.CORPORA_OURS)})") + p.add_argument("--strict-corpus", help="with --strict: the corpora held to no failures instead (comma-separated; 'all' for every one)") + + p = sub.add_parser("prune", help="remove stored answers nothing reads: another compiler's, and those of edited or removed programs") + p.add_argument("--dry-run", action="store_true", help="count them; remove nothing") + + p = sub.add_parser("mojo-run", help="run one Mojo file on the local compiler, built with the pin's build.args and then the file's own # args:") + p.add_argument("file") + + args = ap.parse_args(argv) + try: + pin = load_pin() + if args.cmd == "harvest": + harvest(pin, _locals(args, pin), update=args.update, processors=_csv(args.processors), keep=args.keep, + workdir=Path(args.workdir) if args.workdir else None, allow_hash_change=args.allow_hash_change) + verify.status(pin) + elif args.cmd == "fetch": + wd, keep = make_workdir(False, None) + try: + fetch_all(pin, wd, _locals(args, pin), update=args.update, only=_csv(args.only), allow_hash_change=args.allow_hash_change) + finally: + if not keep: + import shutil + shutil.rmtree(wd, ignore_errors=True) + elif args.cmd == "emit": + for f in emit_ts.emit(pin): + log(f"[emit] {rel(f)}") + elif args.cmd == "verify": + r = verify.run(pin, sorted(_csv(args.corpus) or []), force=args.force, limit=args.limit, + only_ids=set(args.item) if args.item else None, dry_run=args.dry_run, concurrency=args.concurrency) + log(f"[verify] asked {r['asked']}, failed {r['failed']} of {r['todo']}") + doc = verify.status(pin) + _print_summary(doc) + emit_ts.emit(pin) # programs.ts carries the verdicts, so it follows every verify + return 1 if r["failed"] else 0 + elif args.cmd == "prune": + r = verify.prune(pin, dry_run=args.dry_run) + log(f"[prune] {'would remove' if args.dry_run else 'removed'} {r['otherCompiler']} answer(s) of other compilers than {pin.compiler} " + f"and {r['orphaned']} of programs since edited or removed; {r['kept']} kept") + elif args.cmd == "mojo-run": + import subprocess + own = verify.parse_header(Path(args.file).read_text(encoding="utf-8")).get("args", "") + return subprocess.call(["mojo", "run", *build_args(pin.build, own, x86_64=host_is_x86_64()).split(), args.file]) + elif args.cmd == "status": + doc = verify.status(pin) + _print_summary(doc) + emit_ts.emit(pin) + if args.strict: + corpora = verify.CORPORA_ALL if args.strict_corpus == "all" else (_csv(args.strict_corpus) or verify.CORPORA_OURS) + failed = verify.strict_failures(doc, corpora) + for r in failed: + log(f"[strict] {r['id']}: {r['why']}") + if failed: + return 1 + return 0 + except PipelineError as e: + log(f"error: {e}") + return 2 + + +def _print_summary(doc) -> None: + for c, counts in sorted(doc["summary"].items()): + log(f"[status] {c:12} " + " ".join(f"{k} {v}" for k, v in sorted(counts.items()))) + log("[status] written library/mojo/golden/STATUS.md") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/mojo/fuzz-programs.mjs b/scripts/mojo/fuzz-programs.mjs new file mode 100644 index 0000000..99ef275 --- /dev/null +++ b/scripts/mojo/fuzz-programs.mjs @@ -0,0 +1,667 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The generator behind `node scripts/mojo/fuzz.mjs --statements`: small whole pseudo programs, typed, so that most + * of what it writes is a program the pseudo's checker accepts and a run takes to its end. A program is a list of + * nodes (a statement, or a few that belong together) so the fuzzer can leave nodes out to repair or shrink it. + * + * programGenerator(rand).program() the top-level nodes of one program, from a seeded rand() in [0, 1) + * render(nodes, removed) its text without the removed nodes, and each line's chain of nodes + * allNodes(nodes) every node, outermost first + * + * What it keeps out, because the notation and Mojo do not share it: the order a SET or a map is visited in (a FOR + * EACH over one only adds things up), whole numbers past 2^53 (values stay small; a run that stops is left out), + * division by zero (the same), and printing a record or a map of collections, which Mojo cannot print or prints in + * its own order. + */ +/* ------------------------------------------------------------------ */ +/* Random whole programs (--statements) */ +/* ------------------------------------------------------------------ */ + +/** + * A statement is a node: the lines it writes at its own indentation and the blocks under it, in order, so a + * program can be repaired and shrunk by dropping nodes. `label` names what made it, for grouping findings. + */ +export const node = (label, ...parts) => ({ label, parts }); + +/** The text of a program with some nodes left out, and for each line the innermost node it belongs to. */ +export function render(top, removed = new Set()) { + const lines = []; const owner = []; + const walk = (nodes, indent, parentStack) => { + for (const n of nodes) { + if (removed.has(n)) continue; + const stack = [...parentStack, n]; + for (const p of n.parts) { + if (typeof p === 'string') { lines.push(`${' '.repeat(indent)}${p}`); owner.push(stack); } + else walk(p, indent + 1, stack); + } + } + }; + walk(top, 0, []); + return { text: `${lines.join('\n')}\n`, owner }; +} +/** Every node of a program, outermost first. */ +export function allNodes(top) { + const out = []; + const walk = (nodes) => { for (const n of nodes) { out.push(n); for (const p of n.parts) if (typeof p !== 'string') walk(p); } }; + walk(top); + return out; +} + +const SCALAR = ['INTEGER', 'FLOAT', 'STRING', 'CHAR', 'BOOL']; +const SIZED_T = ['INT8', 'INT16', 'INT32', 'INT64', 'UINT8', 'UINT16', 'UINT32', 'UINT64', 'FLOAT32', 'FLOAT64']; +const isInt = (t) => t.k === 'INTEGER' || /^U?INT\d+$/.test(t.k); +const isNum = (t) => isInt(t) || t.k === 'FLOAT' || t.k === 'FLOAT32' || t.k === 'FLOAT64'; +const isScalar = (t) => SCALAR.includes(t.k) || SIZED_T.includes(t.k); +const tt = (t) => { + switch (t.k) { + case 'LIST': return `LIST OF ${tt(t.of)}`; + case 'ARRAY': return `ARRAY[${t.dims.join(', ')}] OF ${tt(t.of)}`; + case 'HASHMAP': return `HASHMAP OF ${tt(t.key)} : ${tt(t.val)}`; + case 'SET': return `SET OF ${tt(t.of)}`; + case 'REC': return t.rec.name; + default: return t.k; + } +}; +const same = (a, b) => tt(a) === tt(b); +const printable = (t) => isScalar(t) || ((t.k === 'LIST' || t.k === 'ARRAY' || t.k === 'SET') && printable(t.of) && t.of.k !== 'REC') + || (t.k === 'HASHMAP' && isScalar(t.val)); +/** Element types a SORT or SORTED puts in order the same way in both languages. */ +const sortable = (t) => ['INTEGER', 'FLOAT', 'STRING', 'CHAR'].includes(t.k); + +/** Names Mojo or its standard library gives a meaning to; a pseudo name may be one of them. */ +const ODD_NAMES = ['Await', 'Async', 'Borrowed', 'Nonlocal', 'Origin', 'Some', 'Never', 'Swap', 'Hash', 'Iter', 'Zip', 'Any', 'All', 'Reversed', + 'Enumerate', 'Exit', 'Rebind', 'Abort', 'Span', 'Pointer', 'Tuple', 'Optional', 'Variant', 'Error', 'Writer', 'Scalar', 'Simd', 'Object', + 'Constrained', 'Lambda', 'Escaping', 'Capturing', 'Where', 'Isinstance', 'Let', 'Debug_assert', 'Bitcast', 'Materialize', 'Dict', 'Range', 'Slice', 'Iterator', + 'Hashable', 'Copyable', 'Movable', 'Sized', 'Writable', 'Stringable', 'Equatable', 'Comparable', 'Defaultable', 'Absolute', 'Uint', 'Element', 'Value', 'Size', 'Count']; +const CAMEL = ['RunTotal', 'BestScore', 'MaxSeen', 'LastWord', 'GcCount', 'StepSize', 'HitCount', 'WordList', 'NextPos', 'TopItem']; +const snakeCaps = (n) => n.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toUpperCase(); + +class TooDeep extends Error { } + +export function programGenerator(rand) { + const pick = (xs) => xs[Math.floor(rand() * xs.length)]; + const some = (p) => rand() < p; + const int = (lo, hi) => lo + Math.floor(rand() * (hi - lo + 1)); + const weighted = (pairs) => { const total = pairs.reduce((n, [, w]) => n + w, 0); let r = rand() * total; for (const [x, w] of pairs) { r -= w; if (r < 0) return x; } return pairs[0][0]; }; + let counter = 0; let tags = 0; + const used = new Set(); + const twins = []; + /** A name for something new: mostly a stem and a number, sometimes CamelCase, a word Mojo knows, or the snake-case twin of a name already made. */ + const fresh = (stem) => { + if (some(0.04) && twins.length) { const t = twins.shift(); if (!used.has(t.toUpperCase())) { used.add(t.toUpperCase()); return t; } } + if (some(0.05)) { const w = pick(ODD_NAMES); if (!used.has(w.toUpperCase())) { used.add(w.toUpperCase()); return w; } } + let n; + if (some(0.08)) { n = `${pick(CAMEL)}${++counter}`; twins.push(snakeCaps(n)); } else n = `${stem}${++counter}`; + used.add(n.toUpperCase()); + return n; + }; + const records = []; const subs = []; const constants = []; + + /* ---------- types ---------- */ + const scalarT = () => ({ k: weighted([['INTEGER', 5], ['FLOAT', 3], ['STRING', 3], ['CHAR', 1], ['BOOL', 1.5], [pick(SIZED_T), 1.2]]) }); + const elemT = () => weighted([[{ k: 'INTEGER' }, 4], [{ k: 'FLOAT' }, 1.5], [{ k: 'STRING' }, 2], [{ k: 'CHAR' }, 0.5], [{ k: 'BOOL' }, 0.5], + [{ k: 'LIST', of: { k: 'INTEGER' } }, 1], ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 1.2]] : [])]); + const keyT = () => ({ k: pick(['STRING', 'INTEGER', 'CHAR']) }); + const anyT = () => weighted([ + [scalarT(), 9], [{ k: 'LIST', of: elemT() }, 6], [{ k: 'ARRAY', of: { k: pick(['INTEGER', 'FLOAT', 'STRING', 'BOOL']) }, dims: [int(2, 4)] }, 1], + [{ k: 'ARRAY', of: { k: pick(['INTEGER', 'CHAR']) }, dims: [int(2, 3), int(2, 3)] }, 1], + [{ k: 'HASHMAP', key: keyT(), val: weighted([[{ k: 'INTEGER' }, 3], [{ k: 'STRING' }, 1], [{ k: 'FLOAT' }, 1], [{ k: 'LIST', of: { k: 'INTEGER' } }, 1]]) }, 2.5], + [{ k: 'SET', of: { k: pick(['INTEGER', 'STRING', 'CHAR']) } }, 2], ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 2.5]] : []), + ]); + + /* ---------- expressions ---------- */ + const lit = { + INTEGER: () => String(pick([0, 1, 2, 3, 4, 5, 7, 10, 12, 100, -1, -3])), + FLOAT: () => pick(['0.5', '1.5', '2.25', '-0.75', '3.0', '10.0', '0.1', '-2.5', '4']), + STRING: () => pick(['"ab"', '"GATTACA"', '"x,y,z"', '""', '"Hello"', '"a"', '"tac"', '"b,a"']), + CHAR: () => pick(["'a'", "'G'", "'z'", "'b'", "','", "'T'"]), + BOOL: () => pick(['TRUE', 'FALSE']), + }; + const keyLit = (t) => (t.k === 'STRING' ? pick(['"a"', '"b"', '"c"', '"ab"']) : t.k === 'CHAR' ? pick(["'a'", "'b'", "'c'", "'d'"]) : String(int(0, 4))); + const paren = (s) => (/^[\w."']+$|^\w+\(.*\)$/.test(s) && !/^-/.test(s) ? s : `(${s})`); + const neg = (s) => (s.startsWith('-') ? `(${s})` : s); + + /** An expression of type t over what `env` holds; d bounds its depth. A chain of calls and records that does not end is given up on. */ + let nest = 0; + function ex(env, t, d = 1) { + if (nest > 40) throw new TooDeep(); + nest++; + try { return ex1(env, t, d); } finally { nest--; } + } + function ex1(env, t, d) { + const vars = env.filter((v) => same(v.t, t)); + const v = () => pick(vars).name; + const has = (pred) => env.filter((x) => pred(x.t)); + const pickVar = (pred) => { const xs = has(pred); return xs.length ? pick(xs) : null; }; + const fns = subs.filter((s) => s.returns && same(s.returns, t) && s.callable(env)); + const call = () => { const f = pick(fns); return `${f.name}(${f.params.map((p) => (p.small ? `(${ex(env, p.t, -1)}) MOD 5` : ex(env, p.t, -1))).join(', ')})`; }; + const idx = (len) => `(${ex(env, { k: 'INTEGER' }, -1)}) MOD ${len}`; + // an element of a list or array of type t, a field of type t + const elem = () => { + const opts = []; + for (const x of has((u) => u.k === 'LIST' && same(u.of, t))) opts.push(() => `${x.name}[${idx(`LENGTH(${x.name})`)}]`); + for (const x of has((u) => u.k === 'ARRAY' && same(u.of, t))) opts.push(() => (x.t.dims.length === 2 ? `${x.name}[${idx(x.t.dims[0])}, ${idx(x.t.dims[1])}]` : `${x.name}[${idx(x.t.dims[0])}]`)); + for (const x of has((u) => u.k === 'REC')) for (const [f, ft] of x.t.rec.fields) { if (same(ft, t)) opts.push(() => `${x.name}.${f}`); else if (ft.k === 'REC') for (const [g, gt] of ft.rec.fields) if (same(gt, t)) opts.push(() => `${x.name}.${f}.${g}`); } + for (const x of has((u) => u.k === 'LIST' && u.of.k === 'REC')) for (const [f, ft] of x.t.of.rec.fields) if (same(ft, t)) opts.push(() => `${x.name}[${idx(`LENGTH(${x.name})`)}].${f}`); + for (const x of has((u) => u.k === 'LIST' && u.of.k === 'LIST' && same(u.of.of, t))) opts.push(() => `${x.name}[${idx(`LENGTH(${x.name})`)}][0]`); + return opts; + }; + const leaf = () => { + const opts = [[() => lit[t.k](), 2]]; + if (vars.length) opts.push([v, 5]); + // below depth 0 only a literal or a variable, so an index or an argument does not grow without end + if (d < 0) return weighted(opts)(); + const el = elem(); if (el.length) opts.push([() => pick(el)(), 2]); + if (fns.length) opts.push([call, 1]); + return weighted(opts)(); + }; + if (d < 0 && !isScalar(t) && vars.length) return v(); + // below depth 0, a collection with no variable to come from is a literal of literals and variables + if (d < 0 && t.k === 'LIST') return `[${ex(env, t.of, -1)}]`; + if (d < 0 && t.k === 'SET') return `{${keyLit(t.of)}}`; + if (d < 0 && t.k === 'HASHMAP') return `{${keyLit(t.key)}: ${ex(env, t.val, -1)}}`; + if (d < 0 && SIZED_T.includes(t.k)) return vars.length && some(0.6) ? v() : t.k.startsWith('FLOAT') ? '0.5' : String(int(0, 9)); + if (isScalar(t) && SIZED_T.includes(t.k)) { + // a sized value: a variable of the type, a small literal, a conversion of a small whole number, or + and * among its own kind + const small = () => (t.k === 'FLOAT32' || t.k === 'FLOAT64' ? pick(['0.5', '1.25', '3.0', '0.1']) : String(int(0, 9))); + const opts = [[small, 2], [() => `${t.k}(${t.k.startsWith('FLOAT') ? ex(env, { k: 'FLOAT' }, 0) : `(${ex(env, { k: 'INTEGER' }, 0)}) MOD 7 + 3`})`, 2]]; + if (vars.length) opts.push([v, 4]); + if (d > 0) opts.push([() => `${paren(ex(env, t, d - 1))} + ${paren(ex(env, t, d - 1))}`, 2], [() => `${paren(ex(env, t, d - 1))} * ${small()}`, 1]); + if (t.k === 'FLOAT64') return weighted(opts)(); + return weighted(opts)(); + } + if (d <= 0 || some(0.3)) { + if (t.k === 'INTEGER' || t.k === 'FLOAT' || t.k === 'STRING' || t.k === 'CHAR' || t.k === 'BOOL') return leaf(); + } + const I = (dd = d - 1) => ex(env, { k: 'INTEGER' }, dd); + const F = (dd = d - 1) => ex(env, { k: 'FLOAT' }, dd); + const S = (dd = d - 1) => ex(env, { k: 'STRING' }, dd); + const lists = (pred) => has((u) => (u.k === 'LIST' || u.k === 'ARRAY') && pred(u.of)); + switch (t.k) { + case 'INTEGER': { + const opts = [ + [() => `${paren(I())} + ${paren(I())}`, 3], [() => `${paren(I())} - ${paren(I())}`, 2], [() => `${paren(I())} * ${paren(I())}`, 2], + [() => `${paren(I())} DIV ${pick(['2', '3', '(-2)', '5'])}`, 1], [() => `${paren(I())} MOD ${pick(['2', '3', '(-4)', '7'])}`, 1], + [() => `LENGTH(${S()})`, 1], [() => `ABS(${I()})`, 0.5], [() => `MIN(${I()}, ${I()})`, 0.5], [() => `MAX(${I()}, ${I()})`, 0.5], [() => `INT(${F()})`, 0.3], + [() => `ROUND(${F()})`, 0.3], [() => leaf(), 2], + ]; + const ls = lists((o) => o.k === 'INTEGER'); if (ls.length) opts.push([() => `SUM_OF(${pick(ls).name})`, 0.7], [() => `INDEX_OF(${pick(ls).name}, ${I(0)})`, 0.5], [() => `MAX(${pick(ls).name})`, 0.3]); + const any = has((u) => ['LIST', 'SET', 'HASHMAP', 'ARRAY'].includes(u.k)); if (any.length) opts.push([() => `LENGTH(${pick(any).name})`, 1.5]); + const cs = has((u) => u.k === 'CHAR'); if (cs.length) opts.push([() => `ORD(${pick(cs).name})`, 0.3]); + const ss = has((u) => u.k === 'STRING'); if (ss.length) opts.push([() => `INDEX_OF(${pick(ss).name}, ${pick(['"a"', "'T'", '","'])})`, 0.4]); + const sz = has((u) => /^U?INT\d+$/.test(u.k)); if (sz.length) opts.push([() => `INTEGER(${pick(sz).name})`, 0.4]); + return weighted(opts)(); + } + case 'FLOAT': { + const opts = [[() => `${paren(F())} + ${paren(F())}`, 2], [() => `${paren(F())} * ${paren(F())}`, 1.5], [() => `${paren(F())} - ${paren(I())}`, 1], + [() => `${paren(I())} / ${pick(['2', '4', '8', '(-5)'])}`, 1], [() => `TO_FLOAT(${I()})`, 0.5], [() => `${paren(I())} * 0.5`, 1], [() => `ABS(${F()})`, 0.4], [() => `MAX(${F()}, ${I()})`, 0.4], [() => leaf(), 2]]; + const ls = lists((o) => o.k === 'FLOAT'); if (ls.length) opts.push([() => `SUM_OF(${pick(ls).name})`, 0.5], [() => `MEAN_OF(${pick(ls).name})`, 0.5], [() => `MIN(${pick(ls).name})`, 0.3]); + const f32 = has((u) => u.k === 'FLOAT32'); if (f32.length) opts.push([() => `FLOAT(${pick(f32).name})`, 0.4]); + return weighted(opts)(); + } + case 'BOOL': { + const cmp = () => pick(['==', '!=', '<', '<=', '>', '>=']); + const opts = [[() => `${paren(I())} ${cmp()} ${paren(I())}`, 3], [() => `${paren(F())} ${cmp()} ${paren(F())}`, 1], [() => `${paren(S())} ${pick(['==', '!=', '<'])} ${paren(S())}`, 1], + [() => `${paren(ex(env, t, d - 1))} AND ${paren(ex(env, t, d - 1))}`, 1], [() => `${paren(ex(env, t, d - 1))} OR ${paren(ex(env, t, d - 1))}`, 1], [() => `NOT (${ex(env, t, d - 1)})`, 0.7], [() => leaf(), 2]]; + const cols = has((u) => ['LIST', 'SET'].includes(u.k) && isScalar(u.of)); if (cols.length) opts.push([() => { const c = pick(cols); return `${ex(env, c.t.of, 0)} IN ${c.name}`; }, 1]); + const maps = has((u) => u.k === 'HASHMAP'); if (maps.length) opts.push([() => { const m = pick(maps); return `${keyLit(m.t.key)} IN ${m.name}`; }, 1]); + const cs = has((u) => u.k === 'CHAR'); if (cs.length) opts.push([() => `${pick(cs).name} IN ${S(0)}`, 0.4]); + return weighted(opts)(); + } + case 'STRING': { + const opts = [[() => `${paren(S())} & ${paren(S())}`, 3], [() => `${paren(S())} & TO_STRING(${I()})`, 1], [() => `UPPER(${S()})`, 0.6], [() => `LOWER(${S()})`, 0.4], + [() => { const s = S(0); return `SLICE(${s}, 0, MIN(2, LENGTH(${s})))`; }, 0.6], [() => `REPLACE(${S()}, "a", "o")`, 0.4], [() => `TO_STRING(${I()})`, 0.5], [() => leaf(), 2]]; + const ls = lists((o) => o.k === 'STRING'); if (ls.length) opts.push([() => `JOIN(${pick(ls).name}, "-")`, 0.6]); + const cs = has((u) => u.k === 'CHAR'); if (cs.length) opts.push([() => `${pick(cs).name} & ""`, 0.5]); + return weighted(opts)(); + } + case 'CHAR': { + const ss = has((u) => u.k === 'STRING'); + const opts = [[() => leaf(), 3]]; + if (ss.length) opts.push([() => { const s = pick(ss).name; return `${s}[${idx(`LENGTH(${s})`)}]`; }, 2]); + return weighted(opts)(); + } + case 'LIST': { + const opts = []; + if (vars.length) opts.push([v, 5]); + const items = () => Array.from({ length: int(1, 3) }, () => ex(env, t.of, 0)); + opts.push([() => `[${items().join(', ')}]`, 2.5]); + if (vars.length && sortable(t.of)) opts.push([() => `SORTED(${v()})`, 1]); + if (vars.length) opts.push([() => { const x = v(); return `SLICE(${x}, 0, MIN(${int(1, 3)}, LENGTH(${x})))`; }, 1]); + if (isScalar(t.of)) opts.push([() => `FILL(${int(0, 3)}, ${ex(env, t.of, 0)})`, 0.6]); + if (t.of.k === 'STRING') opts.push([() => `SPLIT(${S(0)}, ",")`, 1]); + const maps = has((u) => u.k === 'HASHMAP' && same(u.key, t.of)); if (maps.length) opts.push([() => `KEYS(${pick(maps).name})`, 1]); + const sets = has((u) => u.k === 'SET' && same(u.of, t.of)); if (sets.length) opts.push([() => `SORTED(${pick(sets).name})`, 1]); + if (fns.length) opts.push([call, 1]); + return weighted(opts)(); + } + case 'HASHMAP': { + const opts = [[() => `{${Array.from({ length: int(1, 2) }, () => `${keyLit(t.key)}: ${ex(env, t.val, 0)}`).filter((x, i, a) => a.findIndex((y) => y.split(':')[0] === x.split(':')[0]) === i).join(', ')}}`, 1]]; + if (vars.length) opts.push([v, 3]); + if (fns.length) opts.push([call, 1]); + return weighted(opts)(); + } + case 'SET': { + const opts = [[() => `{${[...new Set(Array.from({ length: int(1, 3) }, () => keyLit(t.of)))].join(', ')}}`, 1]]; + if (vars.length) opts.push([v, 3]); + if (fns.length) opts.push([call, 1]); + return weighted(opts)(); + } + case 'REC': { + const opts = []; + if (vars.length) opts.push([v, 4]); + const el = elem(); if (el.length) opts.push([() => pick(el)(), 2]); + if (fns.length) opts.push([call, 1]); + if (!opts.length) return null; + return weighted(opts)(); + } + case 'ARRAY': return vars.length ? v() : null; + default: return leaf(); + } + } + /** Can `ex` make a value of type t from this env? (A record or an array needs a variable to come from.) */ + const canMake = (env, t) => !(t.k === 'ARRAY' || t.k === 'REC') || env.some((x) => same(x.t, t)) || subs.some((s) => s.returns && same(s.returns, t) && s.callable(env)); + + /* ---------- statements ---------- */ + const writable = (env, pred = () => true) => env.filter((x) => !x.ro && pred(x.t)); + const tagged = (env, values, label = 'output') => { + const tag = ++tags; + if (values.length === 1 && some(0.2)) return node(label, `OUTPUT ${values[0]}`); + return { ...node(label, `OUTPUT "@${tag} ", ${values.join(', " ", ')}`), tag }; + }; + const printables = (env) => env.filter((x) => printable(x.t)); + const showSome = (env, label) => { + const ps = printables(env); + const vals = []; + for (let i = int(1, 3); i > 0; i--) vals.push(ps.length && some(0.6) ? pick(ps).name : ex(env, scalarT(), 1)); + return tagged(env, vals, label); + }; + + /** DECLARE, with its value: the one-line form, the two-line form, several names, an ARRAY filled by a loop, a record field by field. */ + function declare(env, ctx, t = anyT()) { + const name = fresh({ INTEGER: 'N', FLOAT: 'X', STRING: 'S', CHAR: 'C', BOOL: 'Ok', LIST: 'Xs', ARRAY: 'A', HASHMAP: 'H', SET: 'Seen', REC: 'R' }[t.k] ?? 'W'); + const out = []; + if (t.k === 'ARRAY') { + out.push(node('declare-array', `DECLARE ${name} : ${tt(t)}`)); + const i = fresh('I'); + const inner = [...env, { name: i, t: { k: 'INTEGER' }, ro: true }]; + if (t.dims.length === 1) out.push(node('fill-array', `FOR ${i} <- 0 TO ${t.dims[0]}`, [node('fill-array', `${name}[${i}] <- ${ex(inner, t.of, 1)}`)], `NEXT ${i}`)); + else { + const j = fresh('J'); + const inner2 = [...inner, { name: j, t: { k: 'INTEGER' }, ro: true }]; + out.push(node('fill-array', `FOR ${i} <- 0 TO ${t.dims[0]}`, [node('fill-array', `FOR ${j} <- 0 TO ${t.dims[1]}`, [node('fill-array', `${name}[${i}, ${j}] <- ${ex(inner2, t.of, 1)}`)], `NEXT ${j}`)], `NEXT ${i}`)); + } + } else if (t.k === 'REC') { + const value = canMake(env, t) && some(0.4) ? ex(env, t, 1) : null; + if (value) out.push(node('declare-record', `DECLARE ${name} : ${tt(t)} <- ${value}`)); + else { + out.push(node('declare-record', `DECLARE ${name} : ${tt(t)}`)); + for (const [f, ft] of t.rec.fields) { + if (ft.k === 'REC') { for (const [g, gt] of ft.rec.fields) out.push(node('field', `${name}.${f}.${g} <- ${ex(env, gt, 1)}`)); } + else out.push(node('field', `${name}.${f} <- ${ex(env, ft, 1)}`)); + } + } + } else if (some(0.65) && canMake(env, t)) out.push(node('declare', `DECLARE ${name} : ${tt(t)} <- ${ex(env, t, 1)}`)); + else if (some(0.3) && isScalar(t)) { + const other = fresh(t.k === 'INTEGER' ? 'M' : 'V'); + out.push(node('declare-two', `DECLARE ${name}, ${other} : ${tt(t)}`), node('assign', `${name} <- ${ex(env, t, 1)}`), node('assign', `${other} <- ${ex(env, t, 1)}`)); + env.push({ name: other, t }); + } else { + out.push(node('declare', `DECLARE ${name} : ${tt(t)}`)); + if (t.k !== 'LIST' || some(0.7)) out.push(node('assign', `${name} <- ${ex(env, t, 1)}`)); + } + env.push({ name, t }); + return out; + } + + /** A place of type t to assign to, or null: a variable, an element, a field, a map entry. */ + function place(env) { + const opts = []; + for (const x of writable(env)) { + if (x.t.k !== 'ARRAY') opts.push([() => [x.name, x.t], 3]); + if (x.t.k === 'LIST') opts.push([() => [`${x.name}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name})]`, x.t.of], 2]); + if (x.t.k === 'LIST' && x.t.of.k === 'LIST') opts.push([() => [`${x.name}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name})][0]`, x.t.of.of], 1]); + if (x.t.k === 'ARRAY') opts.push([() => [x.t.dims.length === 2 ? `${x.name}[${int(0, x.t.dims[0] - 1)}, ${int(0, x.t.dims[1] - 1)}]` : `${x.name}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD ${x.t.dims[0]}]`, x.t.of], 2]); + if (x.t.k === 'HASHMAP') opts.push([() => [`${x.name}[${keyLit(x.t.key)}]`, x.t.val], 2]); + if (x.t.k === 'REC') for (const [f, ft] of x.t.rec.fields) { + opts.push([() => [`${x.name}.${f}`, ft], 1.5]); + if (ft.k === 'LIST') opts.push([() => [`${x.name}.${f}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name}.${f})]`, ft.of], 0.7]); + if (ft.k === 'REC') for (const [g, gt] of ft.rec.fields) opts.push([() => [`${x.name}.${f}.${g}`, gt], 0.7]); + } + if (x.t.k === 'LIST' && x.t.of.k === 'REC') for (const [f, ft] of x.t.of.rec.fields) opts.push([() => [`${x.name}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name})].${f}`, ft], 1]); + } + return opts.length ? weighted(opts)() : null; + } + function assign(env) { + const p = place(env); + if (!p || !canMake(env, p[1])) return []; + const [target, t] = p; + // a running update reads the place it writes + if (isNum(p[1]) && some(0.4)) return [node('update', `${target} <- ${target} ${pick(['+', '-', '*'])} ${neg(ex(env, t, 0))}`)]; + if (t.k === 'STRING' && some(0.3)) return [node('update', `${target} <- ${pick([`${target} & ${ex(env, t, 0)}`, `${ex(env, t, 0)} & ${target}`])}`)]; + return [node('assign', `${target} <- ${ex(env, t, 1)}`)]; + } + + /** The collection built-ins: APPEND, INSERT, REMOVE, SORT, ADD, DELETE, and a tally into a map. */ + function builtin(env) { + const opts = []; + for (const x of writable(env, (u) => u.k === 'LIST')) { + if (canMake(env, x.t.of)) opts.push([() => [node('append', `APPEND(${x.name}, ${ex(env, x.t.of, 1)})`)], 3]); + if (canMake(env, x.t.of)) opts.push([() => [node('insert', `INSERT(${x.name}, (${ex(env, { k: 'INTEGER' }, 0)}) MOD (LENGTH(${x.name}) + 1), ${ex(env, x.t.of, 1)})`)], 1.5]); + opts.push([() => [node('remove', `IF LENGTH(${x.name}) > 1 THEN`, [node('remove', `REMOVE(${x.name}, (${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name}))`)], 'ENDIF')], 1.5]); + if (sortable(x.t.of)) opts.push([() => [node('sort', `SORT(${x.name})`)], 1]); + if (x.t.of.k === 'LIST') opts.push([() => [node('append-inner', `IF LENGTH(${x.name}) > 0 THEN`, [node('append-inner', `APPEND(${x.name}[0], ${ex(env, x.t.of.of, 0)})`)], 'ENDIF')], 1]); + } + for (const x of writable(env, (u) => u.k === 'REC')) for (const [f, ft] of x.t.rec.fields) if (ft.k === 'LIST') opts.push([() => [node('append-field', `APPEND(${x.name}.${f}, ${ex(env, ft.of, 1)})`)], 1]); + for (const x of writable(env, (u) => u.k === 'SET')) { + opts.push([() => [node('add', `ADD(${x.name}, ${keyLit(x.t.of)})`)], 2]); + opts.push([() => [node('delete-set', `DELETE(${x.name}, ${keyLit(x.t.of)})`)], 1]); + } + for (const x of writable(env, (u) => u.k === 'HASHMAP')) { + const k = keyLit(x.t.key); + if (x.t.val.k === 'INTEGER') opts.push([() => [node('tally', `IF ${k} IN ${x.name} THEN`, [node('tally', `${x.name}[${k}] <- ${x.name}[${k}] + 1`)], 'ELSE', [node('tally', `${x.name}[${k}] <- 1`)], 'ENDIF')], 2]); + if (x.t.val.k === 'LIST') opts.push([() => [node('map-list', `IF NOT (${k} IN ${x.name}) THEN`, [node('map-list', `${x.name}[${k}] <- []`)], 'ENDIF'), node('map-list', `APPEND(${x.name}[${k}], ${ex(env, x.t.val.of, 0)})`)], 2]); + opts.push([() => [node('delete-map', `DELETE(${x.name}, ${k})`)], 1]); + opts.push([() => { const y = writable(env, (u) => same(u, x.t.val)); return y.length ? [node('read-map', `IF ${k} IN ${x.name} THEN`, [node('read-map', `${pick(y).name} <- ${x.name}[${k}]`)], 'ENDIF')] : [node('read-map', `IF ${k} IN ${x.name} THEN`, [tagged(env, [`${x.name}[${k}]`], 'read-map')], 'ENDIF')]; }, 1.5]); + } + return opts.length ? weighted(opts)() : []; + } + + /** Copies: a list, a record, a list in a list or in a map, changed after the copy is made, and both shown. */ + function copies(env) { + const opts = []; + for (const x of writable(env, (u) => u.k === 'LIST' && isScalar(u.of) && printable(u))) opts.push(() => { + const m = fresh('Copy'); + const out = [node('copy-list', `DECLARE ${m} : ${tt(x.t)} <- ${x.name}`), node('copy-list', `APPEND(${x.name}, ${ex(env, x.t.of, 0)})`)]; + out.push(tagged(env, [m, x.name], 'copy-list')); + env.push({ name: m, t: x.t }); + return out; + }); + for (const x of writable(env, (u) => u.k === 'REC')) opts.push(() => { + const m = fresh('Twin'); + const [f, ft] = pick(x.t.rec.fields); + const out = [node('copy-record', `DECLARE ${m} : ${tt(x.t)}`), node('copy-record', `${m} <- ${x.name}`)]; + if (ft.k === 'LIST') out.push(node('copy-record', `APPEND(${x.name}.${f}, ${ex(env, ft.of, 0)})`)); + else if (ft.k === 'REC') { const [g, gt] = pick(ft.rec.fields); out.push(node('copy-record', `${x.name}.${f}.${g} <- ${ex(env, gt, 0)}`)); } + else out.push(node('copy-record', `${x.name}.${f} <- ${ex(env, ft, 0)}`)); + const show = (r) => (ft.k === 'REC' ? `${r}.${f}.${ft.rec.fields[0][0]}` : `${r}.${f}`); + if (printable(ft.k === 'REC' ? ft.rec.fields[0][1] : ft)) out.push(tagged(env, [show(m), show(x.name)], 'copy-record')); + env.push({ name: m, t: x.t }); + return out; + }); + for (const x of writable(env, (u) => u.k === 'LIST' && u.of.k === 'LIST')) for (const y of writable(env, (u) => same(u, x.t.of))) opts.push(() => [ + node('copy-nested', `APPEND(${x.name}, ${y.name})`), node('copy-nested', `APPEND(${y.name}, ${ex(env, y.t.of, 0)})`), tagged(env, [x.name, y.name], 'copy-nested')]); + for (const x of writable(env, (u) => u.k === 'LIST' && u.of.k === 'REC')) for (const y of writable(env, (u) => same(u, x.t.of))) opts.push(() => { + const [f, ft] = pick(y.t.rec.fields); + if (!isScalar(ft)) return []; + return [node('copy-into-list', `APPEND(${x.name}, ${y.name})`), node('copy-into-list', `${y.name}.${f} <- ${ex(env, ft, 0)}`), tagged(env, [`${x.name}[LENGTH(${x.name}) - 1].${f}`, `${y.name}.${f}`], 'copy-into-list')]; + }); + for (const x of writable(env, (u) => u.k === 'HASHMAP' && u.val.k === 'LIST')) for (const y of writable(env, (u) => same(u, x.t.val))) opts.push(() => { + const k = keyLit(x.t.key); + return [node('copy-map', `${x.name}[${k}] <- ${y.name}`), node('copy-map', `APPEND(${y.name}, ${ex(env, y.t.of, 0)})`), tagged(env, [`${x.name}[${k}]`, y.name], 'copy-map')]; + }); + return opts.length ? pick(opts)() : []; + } + + function cond(env) { return ex(env, { k: 'BOOL' }, int(1, 2)); } + function block(env, ctx, n) { + const inner = [...env]; + const out = []; + for (let i = 0; i < n; i++) out.push(...stmt(inner, ctx)); + return out.length ? out : [showSome(inner, 'output')]; + } + + function ifStmt(env, ctx) { + const parts = [`IF ${cond(env)} THEN`, block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 3))]; + for (let i = int(0, 2); i > 0; i--) parts.push(`ELSEIF ${cond(env)} THEN`, block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 2))); + if (some(0.5)) parts.push('ELSE', block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 2))); + parts.push('ENDIF'); + return [node('if', ...parts)]; + } + + function caseStmt(env, ctx) { + const kind = weighted([['INTEGER', 4], ['CHAR', 1], ['STRING', 1], ['FLOAT', 1]]); + const subj = ex(env, { k: kind }, 1); + const parts = [`CASE OF ${subj}`]; + const lab = () => { + if (kind === 'INTEGER') return weighted([[() => String(int(-2, 12)), 3], [() => `${int(0, 4)}, ${int(5, 9)}`, 2], [() => { const a = int(-5, 6); return `${a} TO ${a + int(1, 8)}`; }, 3]])(); + if (kind === 'FLOAT') { const a = int(-3, 5); return weighted([[`${a} TO ${a + int(1, 6)}`, 3], [`${a}.5 TO ${a + 2}.25`, 1], [`${a}.5`, 1]]); } + if (kind === 'CHAR') return weighted([[lit.CHAR(), 2], [`${lit.CHAR()}, ${lit.CHAR()}`, 1], ["'a' TO 'n'", 1]]); + return weighted([[lit.STRING(), 2], [`${lit.STRING()}, ${lit.STRING()}`, 1]]); + }; + const seen = new Set(); + for (let i = int(1, 3); i > 0; i--) { + const l = lab(); + if (seen.has(l)) continue; + seen.add(l); + if (some(0.5)) { const b = stmt([...env], { ...ctx, depth: ctx.depth + 1, oneLine: true }); if (b.length === 1 && b[0].parts.length === 1) { parts.push(node('case-arm', `${l} : ${b[0].parts[0]}`)); continue; } } + parts.push(node('case-arm', `${l} :`, block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 2)))); + } + if (some(0.6)) parts.push(node('case-arm', 'OTHERWISE :', block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 2)))); + parts.push('ENDCASE'); + // the arms are nodes of their own, each at the CASE's indentation plus one; the labels are lines of theirs + return [{ label: 'case', parts: [parts[0], parts.slice(1, -1), parts.at(-1)] }]; + } + + function loopBody(env, ctx, label, n) { + const body = block(env, { ...ctx, depth: ctx.depth + 1, loop: label }, n); + // a BREAK or CONTINUE under a condition, somewhere in the body + if (some(0.35)) { const at = int(0, body.length); body.splice(at, 0, node('jump', `IF ${cond(env)} THEN`, [node('jump', pick(['BREAK', 'CONTINUE']))], 'ENDIF')); } + return body; + } + + function forStmt(env, ctx) { + const i = fresh('I'); + const inner = [...env, { name: i, t: { k: 'INTEGER' }, ro: true }]; + let head; + const r = rand(); + if (r < 0.15) { + // the bound is a variable the body changes: the notation works the bounds out once + const ns = writable(env, (u) => u.k === 'INTEGER'); + if (ns.length) { + const n = pick(ns).name; + const body = loopBody(inner, ctx, 'for', int(1, 2)); + body.unshift(node('for-bound', `${n} <- ${n} + 1`)); + return [node('for-bound', `FOR ${i} <- 0 TO MIN(${n}, 6)`, body, `NEXT ${i}`)]; + } + } + if (r < 0.4) { const a = int(4, 9); head = `FOR ${i} <- ${a} TO ${a - int(2, 6)} STEP ${pick(['-1', '-2', '-3'])}`; } + else if (r < 0.6) head = `FOR ${i} <- ${int(-2, 2)} TO ${int(3, 9)} STEP ${pick(['2', '3'])}`; + else if (r < 0.8) head = `FOR ${i} <- 0 TO ${int(0, 5)}`; + else { + const ls = env.filter((x) => ['LIST', 'STRING'].includes(x.t.k)); + head = ls.length ? `FOR ${i} <- 0 TO LENGTH(${pick(ls).name})` : `FOR ${i} <- 1 TO ${int(2, 5)}`; + } + return [node('for', head, loopBody(inner, ctx, 'for', int(1, 3)), `NEXT ${i}`)]; + } + + function forEach(env, ctx) { + const x = fresh('E'); + const opts = []; + for (const c of env.filter((u) => u.t.k === 'LIST' || u.t.k === 'ARRAY' || u.t.k === 'STRING')) opts.push([c, 3]); + opts.push([{ name: lit.STRING(), t: { k: 'STRING' }, literal: true }, 0.5], [{ name: `[${int(-2, 2)}, ${int(2, 6)})`, t: { k: 'RANGE' }, literal: true }, 1]); + for (const c of env.filter((u) => u.t.k === 'HASHMAP' || u.t.k === 'SET')) opts.push([c, 2]); + const c = weighted(opts); + const elT = c.t.k === 'STRING' ? { k: 'CHAR' } : c.t.k === 'RANGE' ? { k: 'INTEGER' } : c.t.k === 'HASHMAP' ? c.t.key : c.t.of; + // the notation walks a one-dimensional collection only + if (c.t.k === 'ARRAY' && c.t.dims.length === 2) return []; + const inner = [...env, { name: x, t: elT, ro: true }]; + if (c.t.k === 'HASHMAP' || c.t.k === 'SET') { + // a SET or a map is visited in an order the notation does not promise: the body only adds things up + const acc = fresh('Acc'); + const contrib = elT.k === 'INTEGER' ? pick([x, `${x} * ${x}`, `${x} MOD 3`]) : elT.k === 'STRING' ? `LENGTH(${x})` : `ORD(${x})`; + const val = c.t.k === 'HASHMAP' && c.t.val.k === 'INTEGER' ? ` + ${c.name}[${x}]` : c.t.k === 'HASHMAP' && c.t.val.k === 'LIST' ? ` + LENGTH(${c.name}[${x}])` : ''; + const body = [node('each-unordered', `${acc} <- ${acc} + ${contrib}${val}`)]; + if (some(0.4)) body.push(node('each-unordered', `IF ${contrib} > 1 THEN`, [node('each-unordered', `${acc} <- ${acc} + 100`)], 'ENDIF')); + // the body changes what it walks, in a way whose end does not depend on the order + if (!c.ro && some(0.3)) { + if (c.t.k === 'HASHMAP' && c.t.val.k === 'INTEGER') body.push(node('each-change', `${c.name}[${x}] <- ${c.name}[${x}] * 2`)); + if (c.t.k === 'SET') body.push(node('each-change', `ADD(${c.name}, ${keyLit(c.t.of)})`)); + } + env.push({ name: acc, t: { k: 'INTEGER' } }); + return [node('each-unordered', `DECLARE ${acc} : INTEGER <- 0`), node('each-unordered', `FOR EACH ${x} IN ${c.name}`, body, `NEXT ${x}`), tagged(env, [acc], 'each-unordered')]; + } + const body = loopBody(inner, ctx, 'each', int(1, 3)); + // the body changes what it walks: the notation walks it as it was when the loop began + if (!c.literal && !c.ro && some(0.3)) { + if (c.t.k === 'LIST' && canMake(inner, c.t.of)) body.push(node('each-change', `APPEND(${c.name}, ${ex(inner, c.t.of, 0)})`)); + if (c.t.k === 'STRING') body.push(node('each-change', `${c.name} <- ${pick([`"z" & ${c.name}`, `SLICE(${c.name}, 1, LENGTH(${c.name}))`, `${c.name} & "q"`])}`)); + if (c.t.k === 'ARRAY' && c.t.dims.length === 1) body.push(node('each-change', `${c.name}[0] <- ${ex(inner, c.t.of, 0)}`)); + } + return [node(`each-${c.t.k.toLowerCase()}`, `FOR EACH ${x} IN ${c.name}`, body, `NEXT ${x}`)]; + } + + function whileStmt(env, ctx) { + const k = fresh('K'); + const inner = [...env, { name: k, t: { k: 'INTEGER' }, ro: true }]; + const limit = int(1, 5); + const body = loopBody(inner, ctx, 'while', int(1, 3)); + body.unshift(node('while', `${k} <- ${k} + 1`)); + const extra = some(0.4) ? ` AND ${paren(cond(env))}` : ''; + env.push({ name: k, t: { k: 'INTEGER' }, ro: true }); + return [node('while', `DECLARE ${k} : INTEGER <- 0`), node('while', `WHILE ${k} < ${limit}${extra}`, body, 'ENDWHILE')]; + } + function repeatStmt(env, ctx) { + const k = fresh('K'); + const inner = [...env, { name: k, t: { k: 'INTEGER' }, ro: true }]; + const body = loopBody(inner, ctx, 'repeat', int(1, 3)); + body.unshift(node('repeat', `${k} <- ${k} + 1`)); + const extra = some(0.4) ? ` OR ${paren(cond(env))}` : ''; + env.push({ name: k, t: { k: 'INTEGER' }, ro: true }); + return [node('repeat', `DECLARE ${k} : INTEGER <- 0`), node('repeat', 'REPEAT', body, `UNTIL ${k} >= ${int(1, 5)}${extra}`)]; + } + + function callStmt(env) { + const procs = subs.filter((s) => !s.returns && s.callable(env)); + if (!procs.length) return []; + const p = pick(procs); + const taken = new Set(); + const args = []; + for (const q of p.params) { + if (q.byref) { + const vs = writable(env, (u) => same(u, q.t)).filter((x) => !taken.has(x.name)); + if (!vs.length) return []; + const x = pick(vs); taken.add(x.name); args.push(x.name); + } else if (!isScalar(q.t) && some(0.6) && env.some((u) => same(u.t, q.t))) { + // a collection or a record passed by value: the routine's changes must not reach it + const x = pick(env.filter((u) => same(u.t, q.t))); args.push(x.name); + } else args.push(ex(env, q.t, 1)); + } + const out = [node('call', `CALL ${p.name}(${args.join(', ')})`)]; + const shown = p.params.map((q, i) => (printable(q.t) && env.some((u) => u.name === args[i]) ? args[i] : null)).filter(Boolean); + if (shown.length) out.push(tagged(env, shown, 'call')); + return out; + } + + /** One statement (or a few that belong together), for a body whose names are `env`. */ + function stmt(env, ctx) { + const deep = ctx.depth >= 2; + const opts = [ + [() => declare(env, ctx), ctx.depth ? 1 : 3], [() => assign(env), 4], [() => [showSome(env, 'output')], 5], [() => builtin(env), 3], + [() => callStmt(env), 2.5], [() => copies(env), ctx.depth ? 0.5 : 1.5], + ]; + // compound statements, fewer the deeper the body + const w = ctx.depth === 0 ? 1 : 0.4; + if (!ctx.oneLine && !deep) opts.push([() => ifStmt(env, ctx), 2 * w], [() => caseStmt(env, ctx), 1 * w], [() => forStmt(env, ctx), 1.5 * w], [() => forEach(env, ctx), 1.5 * w], + [() => whileStmt(env, ctx), 0.8 * w], [() => repeatStmt(env, ctx), 0.8 * w]); + if (ctx.sub?.returns && ctx.depth > 0 && !ctx.oneLine) opts.push([() => (canMake(env, ctx.sub.returns) ? [node('return', `RETURN ${ex(env, ctx.sub.returns, 1)}`)] : []), 0.6]); + if (ctx.sub && !ctx.sub.returns && ctx.depth > 0) opts.push([() => [node('return', 'RETURN')], 0.4]); + for (let tries = 0; tries < 5; tries++) { + try { + const got = weighted(opts)(); + if (got.length) return got; + } catch (e) { if (!(e instanceof TooDeep)) throw e; } + } + return [showSome(env, 'output')]; + } + + /* ---------- records, constants, routines ---------- */ + function record() { + const name = some(0.15) ? pick(ODD_NAMES.filter((w) => !used.has(w.toUpperCase()))) ?? fresh('Rec') : fresh(pick(['Point', 'Item', 'Cell', 'Pair', 'Node'])); + used.add(name.toUpperCase()); + const fields = []; + const fnames = new Set(); + for (let i = int(2, 4); i > 0; i--) { + let f = some(0.15) ? pick(['Copy', 'Hash', 'Value', 'Len', 'Pop', 'Items', 'Self', 'Init', 'Write_to', 'Count', 'Size', 'Name', 'Data']) : pick(['X', 'Y', 'Name', 'Score', 'Tags', 'Flag', 'Total', 'Label']); + if (fnames.has(f.toUpperCase())) f = `${f}${i}`; + fnames.add(f.toUpperCase()); + const ft = weighted([[{ k: 'INTEGER' }, 3], [{ k: 'FLOAT' }, 1], [{ k: 'STRING' }, 2], [{ k: 'BOOL' }, 1], [{ k: 'LIST', of: { k: 'INTEGER' } }, 1.5], + [{ k: 'LIST', of: { k: 'STRING' } }, 0.5], ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 1]] : [])]); + fields.push([f, ft]); + } + const rec = { name, fields }; + records.push(rec); + return node('type', `TYPE ${name}`, fields.map(([f, ft]) => node('type', `DECLARE ${f} : ${tt(ft)}`)), 'ENDTYPE'); + } + function constant(globals) { + const t = { k: pick(['INTEGER', 'INTEGER', 'FLOAT', 'STRING']) }; + const name = fresh(pick(['Lim', 'Rate', 'Word', 'Base'])); + const v = t.k === 'INTEGER' ? String(int(2, 9)) : t.k === 'FLOAT' ? pick(['0.5', '2.5', '1.25']) : pick(['"ab"', '"GC"']); + globals.push({ name, t, ro: true }); + return node('constant', `CONSTANT ${name} = ${v}`); + } + const paramT = () => weighted([[{ k: 'INTEGER' }, 4], [{ k: 'FLOAT' }, 1.5], [{ k: 'STRING' }, 2], [{ k: 'BOOL' }, 0.7], [{ k: 'CHAR' }, 0.5], [{ k: pick(SIZED_T) }, 0.5], + [{ k: 'LIST', of: { k: 'INTEGER' } }, 2.5], [{ k: 'LIST', of: { k: 'STRING' } }, 1], [{ k: 'LIST', of: { k: 'LIST', of: { k: 'INTEGER' } } }, 0.4], + [{ k: 'HASHMAP', key: { k: 'STRING' }, val: { k: 'INTEGER' } }, 0.8], [{ k: 'SET', of: { k: 'INTEGER' } }, 0.5], ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 1.5]] : [])]); + /** A routine: a FUNCTION (by-value parameters, an answer, an early RETURN) or a PROCEDURE (BYREF and by-value parameters), sometimes recursive with a bound. */ + function routine(globals) { + const isFn = some(0.55); + const name = fresh(isFn ? pick(['Calc', 'Score', 'Pick', 'Build', 'Mix']) : pick(['Update', 'Grow', 'Bump', 'Fill', 'Walk'])); + const recursive = some(0.25); + const params = []; + if (recursive) params.push({ name: fresh('Depth'), t: { k: 'INTEGER' } }); + for (let i = int(recursive ? 0 : 1, 3); i > 0; i--) params.push({ name: fresh('P'), t: paramT(), byref: !isFn && some(0.55) }); + const returns = isFn ? weighted([[{ k: 'INTEGER' }, 4], [{ k: 'FLOAT' }, 1], [{ k: 'STRING' }, 1.5], [{ k: 'BOOL' }, 1], [{ k: 'LIST', of: { k: 'INTEGER' } }, 1.5], + ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 1]] : [])]) : null; + const me = { name, params, returns, index: subs.length, callable: () => true }; + const env = [...globals, ...params.map((p) => ({ name: p.name, t: p.t }))]; + const ctx = { depth: 0, sub: me, loop: null }; + const body = []; + if (recursive) { + const n = params[0].name; + const stop = isFn ? (canMake(env, returns) ? `RETURN ${ex(env, returns, 0)}` : null) : 'RETURN'; + if (!stop) return null; + body.push(node('recursion', `IF ${n} <= 0 THEN`, [node('recursion', stop)], 'ENDIF')); + } + for (let i = int(1, 4); i > 0; i--) body.push(...stmt(env, ctx)); + if (recursive) { + const args = params.map((p, i) => (i === 0 ? `${p.name} - 1` : p.byref ? p.name : ex(env, p.t, 0))); + // a call to itself; in a FUNCTION its answer is used + subs.push(me); + if (isFn) { + if (isNum(returns)) body.push(node('recursion', `RETURN ${name}(${args.join(', ')}) ${pick(['+', '-', '*'])} ${neg(ex(env, returns, 0))}`)); + else if (returns.k === 'STRING') body.push(node('recursion', `RETURN ${ex(env, returns, 0)} & ${name}(${args.join(', ')})`)); + else body.push(node('recursion', `RETURN ${name}(${args.join(', ')})`)); + } else body.push(node('recursion', `CALL ${name}(${args.join(', ')})`)); + subs.pop(); + } else if (isFn) { + if (!canMake(env, returns)) return null; + body.push(node('return', `RETURN ${ex(env, returns, 1)}`)); + } + const head = `${isFn ? 'FUNCTION' : 'PROCEDURE'} ${name}(${params.map((p) => `${p.byref ? 'BYREF ' : ''}${p.name} : ${tt(p.t)}`).join(', ')})${isFn ? ` RETURNS ${tt(returns)}` : ''}`; + // a recursive routine is called with a small first argument only + if (recursive) me.params = [{ name: params[0].name, t: { k: 'INTEGER' }, small: true }, ...params.slice(1)]; + subs.push(me); + return node(isFn ? 'function' : 'procedure', head, body, isFn ? 'ENDFUNCTION' : 'ENDPROCEDURE'); + } + + /** A whole program: records, constants, routines, then the main lines. */ + function program() { + const top = []; + const globals = []; + for (let i = weighted([[0, 1], [1, 3], [2, 2]]); i > 0; i--) top.push(record()); + for (let i = int(0, 2); i > 0; i--) top.push(constant(globals)); + const tried = (fn) => { try { return fn(); } catch (e) { if (e instanceof TooDeep) return null; throw e; } }; + for (let i = weighted([[0, 1], [1, 2], [2, 3], [3, 2]]); i > 0; i--) { const n = subs.length; const r = tried(() => routine(globals)); if (r) top.push(r); else subs.length = n; } + const env = [...globals]; + const ctx = { depth: 0, sub: null, loop: null }; + // a few values to work with, then statements until the program is long enough + for (let i = int(2, 4); i > 0; i--) top.push(...(tried(() => declare(env, ctx)) ?? [])); + const want = int(20, 55); + while (render(top).text.split('\n').length < want) top.push(...stmt(env, ctx)); + return top; + } + return { program }; +} + diff --git a/scripts/mojo/fuzz.mjs b/scripts/mojo/fuzz.mjs new file mode 100644 index 0000000..978f0e8 --- /dev/null +++ b/scripts/mojo/fuzz.mjs @@ -0,0 +1,596 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Fuzz the pseudo-to-Mojo translator, the Mojo checker and the Mojo interpreter against the pinned compiler. + * + * The regression programs show what someone thought to write down. This writes programs nobody did: typed pseudo + * expressions made at random from the notation's operators and built-ins over a handful of variables, one per + * OUTPUT line. A line the pseudo itself stops on (a division by zero, a whole number past 2^53) or the checker + * refuses is dropped, so every line left has an answer in the notation. The lines are translated to Mojo, built + * and run by the local `mojo`, and each printed line is held to the pseudo's: + * + * rejected the compiler does not accept the Mojo the translator wrote for this line + * stopped the built program stops at this line; the pseudo did not + * differs the built program prints another value (more than ROUNDING apart: see scripts/mojo/local.mjs) + * interpreter core's Mojo interpreter prints another line than the built program, or ends another way + * checker core's Mojo checker reports an error on a program the compiler accepts + * + * Each finding is printed once per shape, with the smallest expression that shows it and how many others did. + * A finding the translator cannot fix is a line it must refuse (`not translated`) or a note it must attach; add + * the expression to library/pseudo/programs or tests/sync-mojo.test.mjs when you fix one. + * + * With --statements it writes whole programs instead: records, constants, routines with by-value and BYREF + * parameters, loops of every kind, IF and CASE, the collection built-ins, copies changed after they are made, and a + * tagged OUTPUT at many points. The pseudo's checker and a run are the filter: a line the checker refuses, a run stops + * on, or the translator has no Mojo for is left out and the rest settled again; a program whose output depends on the + * order SET members or map keys are visited is dropped. Each built program is held to the pseudo as a whole (the first + * output line that differs, a rejection, a stop), and the interpreter and the checker to the built program. Each + * shape is shown once, its smallest program shrunk by leaving out statements while the finding stays. + * + * node scripts/mojo/fuzz.mjs [--seed N] [--programs N] [--lines N] [--depth N] [--jobs N] [--keep DIR] [--json] + * node scripts/mojo/fuzz.mjs --statements [--seed N] [--programs N] [--shrink BUILDS] [--shrink-seconds N] [--progress] [--dry] [--jobs N] [--keep DIR] [--json] + * + * The same seed gives the same programs. Exit status: 0 when nothing was found, 1 otherwise, 2 when the local + * compiler is missing or is not the pinned version. Needs a build (dist/). + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn, spawnSync } from 'node:child_process'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const ROOT = fileURLToPath(new URL('../..', import.meta.url)); +const dist = (p) => pathToFileURL(path.join(ROOT, 'dist', p)).href; +const argv = process.argv.slice(2); +const opt = (name, dflt = null) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : dflt; }; +const flag = (name) => argv.includes(`--${name}`); +if (!fs.existsSync(path.join(ROOT, 'dist', 'mojo', 'check.js'))) { console.error('mojo-fuzz: dist/ is missing; run `pixi run build` first'); process.exit(2); } + +const { MOJO_VERSION } = await import(dist('mojo/generated/pin.js')); +const { checkMojo } = await import(dist('mojo/check.js')); +const { runMojo } = await import(dist('mojo/interp.js')); +const { studyOf, runPseudo } = await import(dist('services/pseudo.js')); +const { pseudoToMojo } = await import(dist('sync/pseudo-to-mojo.js')); +const { outputDistance } = await import(dist('sync/compare.js')); +// built as every program is: the pin's arguments (library/mojo/pin.json build.args), less an x86-64 CPU on another machine +const { mojoBuildArgs } = await import(dist('services/godbolt.js')); +const BUILD_ARGS = mojoBuildArgs('', { x86_64: process.arch === 'x64' }).split(' ').filter(Boolean); +const { run: runProgram } = await import(dist('engines/pseudo/interp.js')); +const { programGenerator, render, allNodes } = await import(pathToFileURL(path.join(ROOT, 'scripts', 'mojo', 'fuzz-programs.mjs')).href); + +const ROUNDING = 1e-9; +/** + * Mojo 1.2.0 computes **, exp() and log2() on a Float64 to 9 to 13 significant digits, and a later step can make + * that visible (a FLOOR next to a whole number, a SIN of a large angle, a difference of two near-equal values). A + * line that uses one of them agrees when its numbers are within LOOSE of each other, or both within NEAR_ZERO of + * nothing; it is counted apart. Everything else is held to ROUNDING. + */ +const INEXACT = /\^|POW\(|EXP\(|EXPM1\(|LOG2\(|SINH\(|COSH\(|TANH\(/; +const LOOSE = 1e-5; +const NEAR_ZERO = 1e-8; +const nearZero = (a, b) => { const NUM = /-?\d+(?:\.\d+)?(?:e[+-]?\d+)?/gi; const xs = (a.match(NUM) ?? []).map(Number); const ys = (b.match(NUM) ?? []).map(Number); return xs.length === ys.length && xs.every((x, i) => x === ys[i] || (Math.abs(x) <= NEAR_ZERO && Math.abs(ys[i]) <= NEAR_ZERO)); }; + +/** + * Findings that are the compiler's own, each with what was observed; they are printed apart and do not fail a + * run. Take one off when a new pin fixes it (the fuzzer says when a listed one no longer occurs in a run that + * could have shown it only by its absence, so read the list on every bump). + */ +const UPSTREAM = [ + { kind: 'stopped', test: (f) => /SIGSEGV|SIGABRT/.test(f.detail) && /\^|POW\(/.test(f.expr), + why: 'Mojo 1.2.0: a Float64 raised to a whole exponent of about 1e10 or more overflows the stack of the built program, which can print its earlier output again before it dies (library/mojo/probes/float-pow-huge)' }, + { kind: 'stopped', test: (f) => /timeout/.test(f.detail) && /\^|POW\(/.test(f.expr) && /1e10|6\.02e23|4503599627370497/.test(f.expr), + why: 'Mojo 1.2.0: a Float64 raised to a very large negative whole exponent does not come back in 30 seconds (library/mojo/probes/float-pow-huge)' }, + { kind: 'stopped', test: (f) => /timeout/.test(f.detail) && /GCD\(|LCM\(/.test(f.expr) && /4503599627370497|1000000 \* 1000/.test(f.expr), + why: 'Mojo 1.2.0: gcd() of an even number and an odd one subtracts instead of dividing (std/math/math.mojo shifts the first argument by the shared power of two only), so gcd(6, 4503599627370597) takes about 10^15 steps' }, + { kind: 'differs', test: (f) => /\^|POW\(/.test(f.expr) && /1e10|6\.02e23|4503599627370497/.test(f.expr), + why: 'Mojo 1.2.0: a Float64 raised to a very large exponent comes out wrong where it does not stop (0.5 ** 4503599627370497.0 prints 1.0, not 0.0), and a power of ten is not exact, which a FLOOR or a MOD after it shows (library/mojo/probes/float-pow-huge)' }, + { kind: 'differs', test: (f) => /\b(SIN|COS|TAN)\(/.test(f.expr) && /COSH\(|EXP\(|\^|POW\(/.test(f.expr), + why: 'Mojo 1.2.0: cosh() and exp() of a Float64 far from zero are good to about 10 digits (cosh(100.0) prints 1.3440585709627154e+43; the true value is 1.3440585709080678e+43; library/mojo/probes/float-cosh-accuracy), and the SIN, COS or TAN of so large a value depends on every one of its digits (Mojo\'s own sin, cos and tan of 1e22 are correct)' }, + { kind: 'stopped', test: (f) => /SIGSEGV|SIGABRT/.test(f.detail) && /^\s*\w+(?:\[[^\]]*\])?\.\w+ <- /m.test(f.program ?? '') && /\b(FOR|WHILE|REPEAT)\b/.test(f.program ?? ''), + why: 'Mojo 1.2.0: a struct whose List field is given a copy inside a loop, and which nothing reads after the loop, is freed twice: the built program crashes on the second or third pass (var r = Origin(List[Int]()); for e in range(3): var f = List[Int](); f.append(2); r.hash = f.copy()). Reading r after the loop, or moving f^ instead of copying, avoids it (library/mojo/probes/struct-field-loop-copy)' }, + { kind: 'rejected', test: (f) => /did not finish/.test(f.detail), + why: 'Mojo 1.2.0: `mojo build` does not finish on some small programs: a struct field set in a loop that holds an inner loop breaking on it, and set again after the loop (var p = Pair(True); for i in range(2): p.flag = True; for e in range(3): if p.flag: break; then p.flag = False; a minute and more, where a Bool variable in its place builds at once)' }, +]; +const SEED = Number(opt('seed', '1')); +const PROGRAMS = Number(opt('programs', flag('statements') ? '100' : '20')); +const LINES = Number(opt('lines', '40')); +const DEPTH = Number(opt('depth', '3')); +const JOBS = Math.max(1, Number(opt('jobs', '2')) || 2); +const STATEMENTS = flag('statements'); +const SHRINK_BUILDS = Number(opt('shrink', '80')); +// shrinking also runs the pseudo for every candidate, and a candidate that lost the line ending its loop runs to the +// step limit: the builds were bounded and the time was not (seed 20261112 shrank one finding for over ten minutes) +const SHRINK_SECONDS = Number(opt('shrink-seconds', '90')); +const DRY = flag('dry'); +const BUILD_TIMEOUT = Number(opt('build-timeout', STATEMENTS ? '90' : '240')) * 1000; + +/* ------------------------------------------------------------------ */ +/* Random pseudo */ +/* ------------------------------------------------------------------ */ + +/** mulberry32: small, seeded, the same on every machine. */ +function rng(seed) { + let a = seed >>> 0; + return () => { a = (a + 0x6D2B79F5) >>> 0; let t = a; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; }; +} + +const INTS = ['0', '1', '2', '3', '5', '7', '10', '12', '64', '100', '255', '1000', '65536', '1000000', '(-1)', '(-2)', '(-7)', '(-100)']; +const SMALL = ['0', '1', '2', '3', '4', '5', '8']; +const FLOATS = ['0.0', '1.0', '0.5', '1.5', '2.5', '3.5', '0.1', '0.25', '2.0', '3.25', '10.0', '100.0', '1e-3', '1e10', '6.02e23', '1e-10', '0.49999999999999994', '4503599627370497.0', '(-0.5)', '(-1.5)', '(-2.5)', '(-0.0)', '(-7.25)']; +const STRINGS = ['"GATTACA"', '"tac"', '"A"', '""', '"hello, world"', '"a,b,,c"', '"Zebra"', '"T"']; + +function generator(rand) { + const pick = (xs) => xs[Math.floor(rand() * xs.length)]; + const some = (p) => rand() < p; + const I = (d) => { + if (d <= 0 || some(0.2)) return pick([...INTS, 'I', 'J', 'K', 'I', 'J']); + const a = () => I(d - 1), f = () => F(d - 1); + return pick([ + () => `${a()} + ${a()}`, () => `${a()} - ${a()}`, () => `${a()} * ${a()}`, () => `(${a()}) DIV (${a()})`, () => `(${a()}) MOD (${a()})`, + () => `(${a()}) ^ ${pick(SMALL)}`, () => `ABS(${a()})`, () => `MIN(${a()}, ${a()})`, () => `MAX(${a()}, ${a()})`, () => `MIN(${a()}, ${a()}, ${a()})`, + () => `MIN(${LI(d - 1)})`, () => `MAX(${LI(d - 1)})`, () => `LENGTH(${S(d - 1)})`, () => `LENGTH(${LI(d - 1)})`, () => `LENGTH(${LF(d - 1)})`, + () => `ROUND(${f()})`, () => `FLOOR(${f()})`, () => `CEIL(${f()})`, () => `INT(${f()})`, () => `SIGN(${a()})`, + () => `CLAMP(${a()}, ${a()}, ${a()})`, () => `GCD(${a()}, ${a()})`, () => `LCM(${a()}, ${a()})`, + () => `INDEX_OF(${S(d - 1)}, ${S(0)})`, () => `INDEX_OF(${LI(d - 1)}, ${a()})`, () => `ORD(C)`, () => `SUM_OF(${LI(d - 1)})`, + () => `LI[(${a()}) MOD LENGTH(LI)]`, () => `DOT(${LI(0)}, ${LI(0)})`, + () => `BIT_AND(${a()}, ${a()})`, () => `BIT_OR(${a()}, ${a()})`, () => `BIT_XOR(${a()}, ${a()})`, () => `BIT_NOT(${a()})`, + () => `SHIFT_LEFT(${a()}, ${pick(SMALL)})`, () => `SHIFT_RIGHT(${a()}, ${pick(SMALL)})`, () => `CHOOSE(${pick(SMALL)}, ${pick(SMALL)})`, + () => `TO_INTEGER("${pick(['12', '0', '-7', '007'])}")`, () => `LENGTH(SPLIT(${S(d - 1)}, ${S(0)}))`, + ])(); + }; + const F = (d) => { + if (d <= 0 || some(0.2)) return pick([...FLOATS, 'X', 'Y', 'Z', 'X', 'Y']); + const a = () => F(d - 1), i = () => I(d - 1); + return pick([ + () => `${a()} + ${a()}`, () => `${a()} - ${a()}`, () => `${a()} * ${a()}`, () => `(${a()}) / (${a()})`, () => `(${i()}) / (${i()})`, + () => `${i()} * ${a()}`, () => `${a()} + ${i()}`, () => `${i()} - ${a()}`, () => `(${a()}) ^ ${pick(SMALL)}`, () => `(${a()}) ^ (${a()})`, () => `(${i()}) ^ (${a()})`, + () => `ABS(${a()})`, () => `SQRT(${a()})`, () => `EXP(${a()})`, () => `LOG(${a()})`, () => `LOG2(${a()})`, () => `LOG10(${a()})`, + () => `SIN(${a()})`, () => `COS(${a()})`, () => `TAN(${a()})`, () => `ATAN2(${a()}, ${a()})`, () => `HYPOT(${a()}, ${a()})`, () => `POW(${a()}, ${a()})`, + () => `CBRT(${a()})`, () => `EXPM1(${a()})`, () => `LOG1P(${a()})`, () => `ASIN(${a()})`, () => `ACOS(${a()})`, () => `ATAN(${a()})`, + () => `SINH(${a()})`, () => `COSH(${a()})`, () => `TANH(${a()})`, () => `TRUNC(${a()})`, () => `SIGN(${a()})`, () => `COPYSIGN(${a()}, ${a()})`, + () => `CLAMP(${a()}, ${a()}, ${a()})`, () => `FMA(${a()}, ${a()}, ${a()})`, () => `MIN(${a()}, ${a()})`, () => `MAX(${a()}, ${i()})`, () => `MIN(${i()}, ${a()})`, + () => `MEAN_OF(${LF(d - 1)})`, () => `MEAN_OF(${LI(d - 1)})`, () => `VARIANCE_OF(${LF(d - 1)})`, () => `STDEV_OF(${LI(d - 1)})`, () => `MEDIAN_OF(${LF(d - 1)})`, () => `MEDIAN_OF(${LI(d - 1)})`, + () => `NORM_OF(${LF(d - 1)})`, () => `SUM_OF(${LF(d - 1)})`, () => `DOT(${LF(0)}, ${LF(0)})`, () => `LF[(${i()}) MOD LENGTH(LF)]`, () => `MIN(${LF(d - 1)})`, () => `MAX(${LF(d - 1)})`, + () => `TO_FLOAT(${i()})`, () => 'PI()', () => 'E()', () => `ULP(${a()})`, () => `NEXT_AFTER(${a()}, ${a()})`, () => `TO_FLOAT("${pick(['2.5', '1e3', '-0.125', '10'])}")`, + ])(); + }; + const num = (d) => (some(0.5) ? I(d) : F(d)); + const B = (d) => { + if (d <= 0 || some(0.15)) return pick(['TRUE', 'FALSE', 'P', 'Q']); + const cmp = () => pick(['==', '!=', '<', '<=', '>', '>=']); + return pick([ + () => `${num(d - 1)} ${cmp()} ${num(d - 1)}`, () => `${I(d - 1)} ${cmp()} ${I(d - 1)}`, () => `${F(d - 1)} ${cmp()} ${F(d - 1)}`, + () => `${num(d - 1)} ${pick(['<', '<='])} ${num(d - 1)} ${pick(['<', '<='])} ${num(d - 1)}`, + () => `(${B(d - 1)}) AND (${B(d - 1)})`, () => `(${B(d - 1)}) OR (${B(d - 1)})`, () => `NOT (${B(d - 1)})`, + () => `${I(d - 1)} IN ${LI(d - 1)}`, () => `${num(d - 1)} IN [${I(0)}, ${I(0)})`, () => `C IN ${S(d - 1)}`, + () => `IS_CLOSE(${F(d - 1)}, ${F(d - 1)}, 1e-9, 0.0)`, () => `${S(d - 1)} ${cmp()} ${S(d - 1)}`, () => `IS_FINITE(${F(d - 1)})`, + ])(); + }; + const S = (d) => { + if (d <= 0 || some(0.25)) return pick([...STRINGS, 'S', 'T', 'S', 'T']); + const a = () => S(d - 1); + return pick([ + () => `${a()} & ${a()}`, () => `UPPER(${a()})`, () => `LOWER(${a()})`, () => `SLICE(S, ${pick(SMALL)}, ${pick(SMALL)})`, () => `REPLACE(${a()}, ${S(0)}, ${S(0)})`, + () => `TO_STRING(${I(d - 1)})`, () => `TO_STRING(${F(d - 1)})`, () => `TO_STRING(${B(d - 1)})`, () => `JOIN(${LS(d - 1)}, ${S(0)})`, + () => `${a()} & TO_STRING(${I(d - 1)})`, () => `"" & S[(${I(d - 1)}) MOD LENGTH(S)]`, + ])(); + }; + const LI = (d) => { + if (d <= 0 || some(0.4)) return pick(['LI', 'LI', `[${I(0)}, ${I(0)}, ${I(0)}]`]); + return pick([() => `SORTED(${LI(d - 1)})`, () => `SLICE(LI, ${pick(SMALL)}, ${pick(SMALL)})`, () => `FILL(${pick(SMALL)}, ${I(d - 1)})`, () => `[${I(d - 1)}, ${I(d - 1)}]`])(); + }; + const LF = (d) => { + if (d <= 0 || some(0.4)) return pick(['LF', 'LF', `[${F(0)}, ${F(0)}, ${F(0)}]`]); + return pick([() => `SORTED(${LF(d - 1)})`, () => `SLICE(LF, ${pick(SMALL)}, ${pick(SMALL)})`, () => `FILL(${pick(SMALL)}, ${F(d - 1)})`, () => `[${F(d - 1)}, ${I(d - 1)}]`])(); + }; + const LS = (d) => (d <= 0 || some(0.5) ? pick(['LS', `[${S(0)}, ${S(0)}]`]) : pick([() => `SPLIT(${S(d - 1)}, ${S(0)})`, () => `SORTED(${LS(d - 1)})`])()); + const KINDS = [[I, 4], [F, 5], [B, 3], [S, 2], [LI, 1], [LF, 1], [LS, 1]]; + const total = KINDS.reduce((n, [, w]) => n + w, 0); + const expr = (d) => { let r = rand() * total; for (const [g, w] of KINDS) { r -= w; if (r < 0) return g(d); } return I(d); }; + const preamble = () => [ + `DECLARE I : INTEGER <- ${pick(['7', '1', '12', '100', '0'])}`, `DECLARE J : INTEGER <- ${pick(['-3', '-1', '5', '2', '-10'])}`, `DECLARE K : INTEGER <- ${pick(['0', '3', '1', '-2'])}`, + `DECLARE X : FLOAT <- ${pick(['2.5', '0.5', '1.0', '3.75', '10.0'])}`, `DECLARE Y : FLOAT <- ${pick(['-0.75', '-2.5', '0.1', '1.5', '-1.0'])}`, `DECLARE Z : FLOAT <- ${pick(['1e-3', '0.0', '2.0', '1e6', '0.3'])}`, + `DECLARE P : BOOL <- ${pick(['TRUE', 'FALSE'])}`, `DECLARE Q : BOOL <- ${pick(['TRUE', 'FALSE'])}`, + `DECLARE S : STRING <- ${pick(['"GATTACA"', '"hello"', '"a,b,c"'])}`, `DECLARE T : STRING <- ${pick(['"tac"', '"T"', '"l"', '","'])}`, `DECLARE C : CHAR <- ${pick(["'A'", "'l'", "','"])}`, + `DECLARE LI : LIST OF INTEGER <- ${pick(['[4, -1, 9, 0, 7]', '[3, 3, 1]', '[10, 20]'])}`, `DECLARE LF : LIST OF FLOAT <- ${pick(['[1.5, -2.25, 0.5, 3.0]', '[0.1, 0.2, 0.3]', '[2.0, 4.0]'])}`, + `DECLARE LS : LIST OF STRING <- ${pick(['["pear", "apple", "fig"]', '["b", "a"]'])}`, + ]; + return { expr, preamble }; +} + +/* ------------------------------------------------------------------ */ +/* The local compiler */ +/* ------------------------------------------------------------------ */ + +function compilerVersion() { + const p = spawnSync('mojo', ['--version'], { encoding: 'utf8' }); + return p.status === 0 ? (/Mojo\s+(\S+)/.exec(p.stdout) || [])[1] ?? null : null; +} +const exec = (cmd, args, { cwd, timeout = 60000 } = {}) => new Promise((resolve) => { + const c = spawn(cmd, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; let stderr = ''; let timedOut = false; let capped = false; + const t = setTimeout(() => { timedOut = true; c.kill('SIGKILL'); }, timeout); + // at most 4 MiB of output is kept; `capped` says some was dropped, so nobody compares a line that was cut + c.stdout.on('data', (d) => { if (stdout.length < 1 << 22) stdout += d; else capped = true; }); + c.stderr.on('data', (d) => { if (stderr.length < 1 << 22) stderr += d; }); + c.on('error', (e) => { clearTimeout(t); resolve({ status: null, signal: null, stdout, stderr: String(e), timedOut, capped }); }); + c.on('close', (status, signal) => { clearTimeout(t); resolve({ status, signal, stdout, stderr, timedOut, capped }); }); +}); +const HAVE_PRLIMIT = spawnSync('prlimit', ['--version'], { encoding: 'utf8' }).status === 0; + +async function buildAndRun(source, dir, runTimeout = 30000) { + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'example.mojo'), source); + fs.rmSync(path.join(dir, 'example'), { force: true }); + const b = await exec('mojo', ['build', ...BUILD_ARGS, '--disable-warnings', 'example.mojo', '-o', 'example'], { cwd: dir, timeout: BUILD_TIMEOUT }); + if (b.timedOut) return { compiled: false, stderr: `the compiler did not finish in ${BUILD_TIMEOUT / 1000} seconds` }; + if (b.status !== 0 || !fs.existsSync(path.join(dir, 'example'))) return { compiled: false, stderr: b.stderr || b.stdout }; + // a trap must not become a core dump on someone's desktop (see scripts/mojo/local.mjs) + const r = HAVE_PRLIMIT ? await exec('prlimit', ['--core=1:1', './example'], { cwd: dir, timeout: runTimeout }) : await exec('./example', [], { cwd: dir, timeout: runTimeout }); + return { compiled: true, status: r.timedOut ? 'timeout' : r.status ?? `signal ${r.signal}`, stdout: r.stdout, stderr: r.stderr, capped: r.capped }; +} + +/* ------------------------------------------------------------------ */ +/* One program */ +/* ------------------------------------------------------------------ */ + +const counts = { candidates: 0, refusedByChecker: 0, stoppedInPseudo: 0, notTranslated: 0, lines: 0, agree: 0, rounding: 0, inexact: 0, builds: 0 }; +const findings = []; +const gaps = new Map(); +const outputOf = (pre, e) => { + const st = studyOf(`${pre}\nOUTPUT ${e}\n`); + if (!st.ok) return { refused: true }; + const tr = runPseudo(st, { maxSteps: 200000 }); + return tr.error ? { stops: tr.error.kind } : { line: tr.raw.output.join('\n') }; +}; + +async function oneProgram(n, workdir) { + const rand = rng(SEED * 100003 + n); + const g = generator(rand); + const pre = g.preamble().join('\n'); + // lines that have an answer in the notation + let lines = []; + for (let tries = 0; lines.length < LINES && tries < LINES * 12; tries++) { + const e = g.expr(1 + Math.floor(rand() * DEPTH)); + counts.candidates++; + const o = outputOf(pre, e); + if (o.refused) { counts.refusedByChecker++; continue; } + if (o.stops) { counts.stoppedInPseudo++; continue; } + if (o.line.includes('\n')) continue; + lines.push({ expr: e, want: o.line }); + } + const preLines = pre.split('\n').length; + for (let round = 0; round < 12 && lines.length; round++) { + const pseudo = `${pre}\n${lines.map((l) => `OUTPUT ${l.expr}`).join('\n')}\n`; + const mj = pseudoToMojo(pseudo); + if ('unsupported' in mj) { findings.push({ kind: 'rejected', expr: lines[0].expr, detail: `the translator refused a program the pseudo runs: ${mj.unsupported[0]?.message}` }); return; } + const lineOf = (pseudoLine) => lines[pseudoLine - preLines - 1]; + // a line the translator says it has no Mojo for is not a finding: it said so + const gapLines = new Set(mj.notes.filter((x) => x.id === 'translate-gap').map((x) => x.line)); + if (gapLines.size) { + for (const x of mj.notes.filter((y) => y.id === 'translate-gap')) { const why = (/\((.*)\)\. The generated/.exec(x.message) || [])[1] ?? x.message; gaps.set(why, (gaps.get(why) || 0) + 1); } + counts.notTranslated += gapLines.size; + lines = lines.filter((_, i) => !gapLines.has(preLines + 1 + i)); + continue; + } + counts.builds++; + const r = await buildAndRun(mj.source, path.join(workdir, `p${n}`)); + if (!r.compiled) { + const bad = new Map(); + const errLines = r.stderr.split('\n'); + errLines.forEach((l, i) => { + let m = /example\.mojo:(\d+):\d+: error: (.*)$/.exec(l); + if (!m) return; + if (/function instantiation failed/.test(m[2])) { + // the error is reported at the def; the note that follows names the call, and the next one the reason + const rest = errLines.slice(i + 1); + const call = rest.map((x) => /example\.mojo:(\d+):\d+: note: call expansion failed/.exec(x)).find(Boolean); + const why = rest.map((x) => /note: constraint failed: (.*)$/.exec(x)).find(Boolean); + if (call) m = [m[0], call[1], `function instantiation failed${why ? `: ${why[1]}` : ''}`]; + } + const from = mj.map[Number(m[1]) - 1]; + const hit = from ? lineOf(from) : null; + if (hit && !bad.has(hit)) bad.set(hit, { message: m[2].slice(0, 160), mojo: (mj.source.split('\n')[Number(m[1]) - 1] || '').trim().slice(0, 200) }); + }); + if (!bad.size) { findings.push({ kind: 'rejected', expr: '(a whole program)', detail: `${(errLines.find((l) => /error:/.test(l)) || errLines[0] || '').slice(0, 200)}`, program: pseudo }); return; } + for (const [l, why] of bad) findings.push({ kind: 'rejected', expr: l.expr, detail: why.message, mojo: why.mojo }); + lines = lines.filter((l) => !bad.has(l)); + continue; + } + const got = r.stdout.replace(/\n$/, '').split('\n'); + if (r.status !== 0) { + // a trap says where it happened, on standard output, after everything printed before it; without that, the first + // line that did not print what the pseudo gives is where the built program stopped. Counting the lines printed + // is not enough: a program that overflows its stack can print what it had printed again, thousands of times, + // before it dies (seed 201: a Float64 to a huge whole power, then 3,673 lines from 40 prints, exit 134). + const said = /^At: .*example\.mojo:(\d+):\d+: (.*)$/.exec(got.at(-1) ?? ''); + const from = said ? mj.map[Number(said[1]) - 1] : null; + const holds = (x, g) => { const far = outputDistance(x.want, g); return far <= ROUNDING || (INEXACT.test(x.expr) && Number.isFinite(far) && (far <= LOOSE || nearZero(x.want, g))); }; + const firstOff = r.stdout === '' ? 0 : lines.findIndex((x, i) => !holds(x, got[i] ?? '')); + const l = from ? lineOf(from) : lines[firstOff < 0 ? Math.min(got.length, lines.length) : firstOff]; + if (said) r.stderr = `${said[2]}\n${r.stderr}`; + if (l) { findings.push({ kind: 'stopped', expr: l.expr, detail: `the built program stopped (exit ${r.status}): ${(r.stderr.trim().split('\n').find((x) => /Error|error|At:/.test(x)) || '').slice(0, 160)}; the pseudo gives ${l.want}` }); lines = lines.filter((x) => x !== l); continue; } + findings.push({ kind: 'stopped', expr: '(a whole program)', detail: `the built program stopped (exit ${r.status}) and nothing says where`, program: pseudo }); + return; + } + // the interpreter and the checker, against the same build + const ck = checkMojo(mj.source); + const errs = ck.diagnostics.filter((d) => d.severity === 'error'); + if (errs.length) for (const d of errs) { const hit = mj.map[d.line - 1] ? lineOf(mj.map[d.line - 1]) : null; findings.push({ kind: 'checker', expr: hit?.expr ?? '(the declarations)', detail: d.message.slice(0, 160) }); } + else { + const run = runMojo(ck.module, mj.source, { maxSteps: 400000, record: false }); + if (!run.refused && !run.limited) { + const mine = run.stdout.replace(/\n$/, '').split('\n'); + for (let i = 0; i < lines.length; i++) if ((mine[i] ?? '') !== (got[i] ?? '')) { findings.push({ kind: 'interpreter', expr: lines[i].expr, detail: `the interpreter prints ${JSON.stringify((mine[i] ?? '').slice(0, 80))}; the built program prints ${JSON.stringify((got[i] ?? '').slice(0, 80))}` }); } + } + } + for (let i = 0; i < lines.length; i++) { + counts.lines++; + const far = outputDistance(lines[i].want, got[i] ?? ''); + if (far === 0) counts.agree++; + else if (far <= ROUNDING) counts.rounding++; + else if (INEXACT.test(lines[i].expr) && Number.isFinite(far) && (far <= LOOSE || nearZero(lines[i].want, got[i] ?? ''))) counts.inexact++; + else findings.push({ kind: 'differs', expr: lines[i].expr, detail: `the pseudo gives ${JSON.stringify(lines[i].want.slice(0, 80))}; the built program prints ${JSON.stringify((got[i] ?? '').slice(0, 80))}` }); + } + return; + } +} + +/* ------------------------------------------------------------------ */ +/* One whole program (--statements) */ +/* ------------------------------------------------------------------ */ + +const STMT = { interpRan: 0, interpDeclined: 0, candidates: 0, firstTry: 0, survived: 0, dropped: 0, orderDependent: 0, linesGenerated: 0, linesKept: 0, programs: 0, outputs: 0, agree: 0, rounding: 0, builds: 0, shrinkBuilds: 0, noMain: 0, tooMuch: 0, pseudoFaults: [] }; +const coverage = new Map(); +const innermost = (owner, line) => (owner[line - 1] ?? []).at(-1) ?? null; +const pseudoRun = (st, shuffle = false) => runProgram(st.program.program, { record: false, contracts: true, maxSteps: 200000, seed: shuffle ? 2 : 1, shuffle }); + +/** + * A program the notation runs to its end, or why not: the checker's errors, a stop, a run that prints something + * else when SET members and map keys are visited in another order, the lines the translator says it has no Mojo for. + */ +function settle(top, removed) { + const r = render(top, removed); + const st = studyOf(r.text); + if (!st.ok) return { bad: 'checker', lines: st.diagnostics.filter((d) => d.severity === 'error').map((d) => d.line), r }; + const tr = pseudoRun(st); + if (tr.error) return { bad: tr.error.kind === 'internal' ? 'internal' : 'stops', lines: [tr.error.line].filter(Boolean), r, error: tr.error }; + const sh = pseudoRun(st, true); + if (sh.error || sh.output.join('\n') !== tr.output.join('\n')) return { bad: 'order', lines: [], r }; + const mj = pseudoToMojo(r.text); + if ('unsupported' in mj) return { bad: 'unsupported', lines: [], r }; + const gapNotes = mj.notes.filter((x) => x.id === 'translate-gap'); + if (gapNotes.length) return { bad: 'gap', lines: gapNotes.map((x) => x.line), notes: gapNotes, r }; + return { r, want: tr.output, mj }; +} + +/** What the built program says about a settled program: the first finding, or null when everything agrees. */ +async function judgeBuilt(s, dir, top, removed) { + const { r, want, mj } = s; + const lineText = (l) => (r.text.split('\n')[l - 1] ?? '').trim(); + const nodeAt = (l) => innermost(r.owner, l); + const tagNode = (outLine) => { const m = /^@(\d+) /.exec(outLine ?? ''); if (!m) return null; return allNodes(top).find((x) => x.tag === Number(m[1]) && !removed.has(x)) ?? null; }; + const b = await buildAndRun(mj.source, dir, 10000); + // a program that prints more than the harness keeps (seed 20261112: 5.6 MB from a loop over a list it grows) was + // compared up to a line cut in the middle and reported as different there. It is counted apart instead. + if (b.compiled && b.capped) { STMT.tooMuch++; return null; } + if (!b.compiled) { + const errLines = b.stderr.split('\n'); + for (let i = 0; i < errLines.length; i++) { + let m = /example\.mojo:(\d+):\d+: error: (.*)$/.exec(errLines[i]); + if (!m) continue; + if (/function instantiation failed/.test(m[2])) { + const rest = errLines.slice(i + 1); + const call = rest.map((x) => /example\.mojo:(\d+):\d+: note: call expansion failed/.exec(x)).find(Boolean); + const why = rest.map((x) => /note: constraint failed: (.*)$/.exec(x)).find(Boolean); + if (call) m = [m[0], call[1], `function instantiation failed${why ? `: ${why[1]}` : ''}`]; + } + const from = mj.map[Number(m[1]) - 1]; + return { kind: 'rejected', shape: m[2].replace(/'[^']*'/g, "'…'").replace(/\d+/g, 'N').slice(0, 120), detail: m[2].slice(0, 200), mojo: (mj.source.split('\n')[Number(m[1]) - 1] || '').trim().slice(0, 200), expr: from ? lineText(from) : '(no pseudo line)', at: from ? nodeAt(from) : null }; + } + return { kind: 'rejected', shape: /did not finish/.test(b.stderr) ? 'the compiler did not finish' : 'no error line', detail: (errLines.find((l) => /error:|did not finish/.test(l)) || errLines[0] || '').slice(0, 200), expr: '(a whole program)', at: null }; + } + const got = b.stdout.replace(/\n$/, '').split('\n').filter((x, i, a) => !(x === '' && i === a.length - 1 && b.stdout === '')); + const gotLines = b.stdout === '' ? [] : got; + // the interpreter and the checker, against the same build + const ck = checkMojo(mj.source); + const errs = ck.diagnostics.filter((d) => d.severity === 'error'); + let side = null; + if (errs.length) { const d = errs[0]; const from = mj.map[d.line - 1]; side = { kind: 'checker', shape: d.message.replace(/'[^']*'/g, "'…'").replace(/\d+/g, 'N').slice(0, 120), detail: d.message.slice(0, 200), mojo: (mj.source.split('\n')[d.line - 1] || '').trim(), expr: from ? lineText(from) : '(no pseudo line)', at: from ? nodeAt(from) : null }; } + else { + const run = runMojo(ck.module, mj.source, { maxSteps: 2000000, record: false }); + if (run.refused || run.limited) STMT.interpDeclined++; else STMT.interpRan++; + if (!run.refused && !run.limited) { + const mine = run.stdout === '' ? [] : run.stdout.replace(/\n$/, '').split('\n'); + const i = mine.findIndex((x, k) => x !== gotLines[k]); + const at = i >= 0 ? i : mine.length !== gotLines.length ? Math.min(mine.length, gotLines.length) : -1; + const endsOk = (b.status === 0) === !run.error; + if (at >= 0 || !endsOk) side = { kind: 'interpreter', shape: tagNode(gotLines[at] ?? want[at])?.label ?? 'end', detail: at >= 0 ? `output line ${at + 1}: the interpreter prints ${JSON.stringify((mine[at] ?? '(nothing)').slice(0, 100))}; the built program prints ${JSON.stringify((gotLines[at] ?? '(nothing)').slice(0, 100))}` : `the interpreter ends ${run.error ? `with ${String(run.error.message ?? run.error).slice(0, 80)}` : 'normally'}; the built program exits ${b.status}`, expr: '(see the program)', at: null }; + } + } + if (b.status !== 0) { + const said = /^At: .*example\.mojo:(\d+):\d+: (.*)$/m.exec(b.stdout); + const from = said ? mj.map[Number(said[1]) - 1] : null; + const why = b.status === 'timeout' ? 'it did not end within 10 seconds' : `exit ${b.status}: ${(b.stderr.trim().split('\n').find((x) => /Error|error|At:/.test(x)) || said?.[2] || '').slice(0, 160)}`; + const nd = from ? nodeAt(from) : tagNode(want[gotLines.length]); + return { kind: 'stopped', shape: `${nd?.label ?? 'somewhere'} ${b.status === 'timeout' ? 'timeout' : 'trap'}`, detail: `the built program stopped (${why}) after ${gotLines.length} of ${want.length} lines`, expr: from ? lineText(from) : want[gotLines.length] ?? '', at: nd }; + } + for (let i = 0; i < Math.max(want.length, gotLines.length); i++) { + const far = outputDistance(want[i] ?? '(nothing)', gotLines[i] ?? '(nothing)'); + STMT.outputs++; + if (far === 0) { STMT.agree++; continue; } + if (far <= ROUNDING) { STMT.rounding++; continue; } + const nd = tagNode(want[i]) ?? tagNode(gotLines[i]); + return { kind: 'differs', shape: nd?.label ?? 'untagged', detail: `output line ${i + 1}: the pseudo gives ${JSON.stringify((want[i] ?? '(nothing)').slice(0, 100))}; the built program prints ${JSON.stringify((gotLines[i] ?? '(nothing)').slice(0, 100))}`, expr: nd ? render([nd]).text.trim() : '(an untagged line)', at: nd }; + } + return side; +} + +/** + * Leave out nodes one at a time, keeping each removal after which the program still settles and the built program + * still shows the same kind and shape of finding; stop when nothing more can go or the build budget is spent. + */ +async function shrink(f, workdir) { + const removed = new Set(f.removed); + let budget = SHRINK_BUILDS; + const until = Date.now() + SHRINK_SECONDS * 1000; + const holds = async (rem) => { + const s = settle(f.top, rem); + if (s.bad) return false; + if (budget-- <= 0) return false; + STMT.shrinkBuilds++; + const g = await judgeBuilt(s, path.join(workdir, 'shrink'), f.top, rem); + return !!g && g.kind === f.kind && g.shape === f.shape; + }; + for (let changed = true; changed && budget > 0 && Date.now() < until;) { + changed = false; + for (const nd of allNodes(f.top)) { + if (removed.has(nd) || budget <= 0 || Date.now() >= until) continue; + const rem = new Set(removed); rem.add(nd); + if (await holds(rem)) { removed.add(nd); changed = true; } + } + } + return render(f.top, removed).text; +} + +const wholeFindings = []; +async function oneWholeProgram(n, workdir) { + const rand = rng(SEED * 100003 + n + 0x51a7e); + const top = programGenerator(rand).program(); + const removed = new Set(); + STMT.candidates++; + const generated = render(top).text.split('\n').length - 1; + STMT.linesGenerated += generated; + let clean = true; let builtOnce = false; + for (let round = 0; round < 16; round++) { + const s = settle(top, removed); + if (s.bad) { + clean = false; + if (s.bad === 'order') { STMT.orderDependent++; return; } + if (s.bad === 'internal') STMT.pseudoFaults.push({ message: s.error.message, line: s.error.line, program: s.r.text }); + if (s.bad === 'gap') for (const x of s.notes) { const why = (/\((.*)\)\. The generated/.exec(x.message) || [])[1] ?? x.message; gaps.set(why, (gaps.get(why) || 0) + 1); } + const drop = s.lines.map((l) => innermost(s.r.owner, l)).filter(Boolean); + if (!drop.length) { STMT.dropped++; return; } + for (const d of drop) removed.add(d); + continue; + } + if (!builtOnce) { + builtOnce = true; + STMT.programs++; if (clean) STMT.firstTry++; STMT.survived++; + STMT.linesKept += s.r.text.split('\n').length - 1; + for (const nd of allNodes(top)) if (!removed.has(nd)) coverage.set(nd.label, (coverage.get(nd.label) || 0) + 1); + } + if (DRY) { if (flag('show')) console.log(`--- program ${n}\n${s.r.text}`); return; } + // when every line of the main part was left out, routines and declarations are all that remain: the translation + // is a module with no main, which is right for it and which no compiler builds into a program. Nothing to compare. + if (!/^def main\(/m.test(s.mj.source)) { STMT.noMain++; return; } + STMT.builds++; + const f = await judgeBuilt(s, path.join(workdir, `p${n}`), top, removed); + if (!f) return; + wholeFindings.push({ ...f, program: s.r.text, top, removed: new Set(removed), seedProgram: n }); + if (flag('progress')) process.stderr.write(`program ${n}: ${f.kind} ${f.shape}: ${f.detail.slice(0, 160)}\n`); + // a line the compiler rejects is left out and the rest is built again, so one program can show several things + if (f.kind === 'rejected' && f.at) { removed.add(f.at); continue; } + return; + } + if (!builtOnce) STMT.dropped++; +} + +let wholeOurs = []; +/** The statement mode's report: counts, coverage, each shape once with its smallest program shrunk, the compiler's own apart. */ +async function reportStatements(workdir) { + const upstream = wholeFindings.filter((f) => UPSTREAM.some((u) => u.kind === f.kind && u.test(f))); + wholeOurs = wholeFindings.filter((f) => !upstream.includes(f)); + const shapes = new Map(); + for (const f of wholeOurs) { + const key = `${f.kind} ${f.shape}`; + const s = shapes.get(key) ?? { kind: f.kind, shape: f.shape, count: 0, example: f }; + s.count++; + if (f.program.length < s.example.program.length) s.example = f; + shapes.set(key, s); + } + const sorted = [...shapes.values()].sort((a, b) => a.kind.localeCompare(b.kind) || b.count - a.count); + for (const s of sorted) s.shrunk = SHRINK_BUILDS > 0 ? await shrink(s.example, workdir) : s.example.program; + const pct = (a, b) => `${b ? Math.round((100 * a) / b) : 0}%`; + if (flag('json')) { + console.log(JSON.stringify({ compiler: `Mojo ${version}`, seed: SEED, counts: { ...STMT, pseudoFaults: STMT.pseudoFaults.length }, coverage: Object.fromEntries(coverage), shapes: sorted.map(({ example, ...x }) => ({ ...x, detail: example.detail, mojo: example.mojo, line: example.expr })), upstream: upstream.map(({ top, removed, at, ...x }) => x), gaps: Object.fromEntries(gaps), pseudoFaults: STMT.pseudoFaults }, null, 2)); + return; + } + console.log(`Mojo ${version}, seed ${SEED}, statements: ${STMT.candidates} programs generated; ${STMT.survived} (${pct(STMT.survived, STMT.candidates)}) settled into a program the notation runs to its end (${STMT.firstTry} as generated, the rest after leaving out the lines it refused or stopped on), keeping ${pct(STMT.linesKept, STMT.linesGenerated)} of their lines; ${STMT.orderDependent} depended on the order a SET or map is visited and ${STMT.dropped} could not be settled`); + if (STMT.tooMuch) console.log(`${STMT.tooMuch} built program(s) printed more than 4 MiB, too much to compare here`); + if (STMT.noMain) console.log(`${STMT.noMain} settled program(s) had no line left outside their routines, so there was no program to build`); + console.log(`${STMT.builds} builds (${STMT.shrinkBuilds} more to shrink); ${STMT.outputs} output lines held to the pseudo, up to the first that differs (${STMT.agree} print the same, ${STMT.rounding} differ in their last digits only)`); + console.log(`the interpreter ran ${STMT.interpRan} built programs to compare with the compiler and declined ${STMT.interpDeclined}`); + console.log(`constructs in the programs built: ${[...coverage].sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k} ${v}`).join(', ')}`); + for (const kind of ['rejected', 'stopped', 'differs', 'interpreter', 'checker']) { + const ss = sorted.filter((s) => s.kind === kind); + if (!ss.length) continue; + console.log(`\n${kind}: ${ss.reduce((n, s) => n + s.count, 0)} program(s), ${ss.length} shape(s)`); + for (const s of ss) { + console.log(` [${s.count}x] ${s.shape}: ${s.example.detail}`); + if (s.example.expr) console.log(` at: ${s.example.expr.split('\n')[0]}`); + if (s.example.mojo) console.log(` Mojo: ${s.example.mojo}`); + console.log(` smallest program (program ${s.example.seedProgram}, shrunk):`); + for (const l of s.shrunk.replace(/\n$/, '').split('\n')) console.log(` ${l}`); + } + } + if (gaps.size) { console.log('\nnot translated (the translator says so; those lines were left out):'); for (const [why, k] of [...gaps].sort((a, b) => b[1] - a[1])) console.log(` [${k}x] ${why}`); } + for (const u of UPSTREAM) { + const fs_ = upstream.filter((f) => u.kind === f.kind && u.test(f)); + if (fs_.length) console.log(`\nthe compiler's own (${fs_.length} program(s), not counted): ${u.why}\n e.g. ${fs_[0].expr}`); + } + if (STMT.pseudoFaults.length) { + console.log(`\nthe pseudo engine's own faults (${STMT.pseudoFaults.length}; reported, not counted): ${STMT.pseudoFaults[0].message} (line ${STMT.pseudoFaults[0].line})`); + for (const l of STMT.pseudoFaults[0].program.replace(/\n$/, '').split('\n')) console.log(` ${l}`); + } + if (!wholeOurs.length) console.log('\nnothing found'); +} + +/* ------------------------------------------------------------------ */ +/* Run */ +/* ------------------------------------------------------------------ */ + +const version = compilerVersion(); +if (!version) { console.error('mojo-fuzz: no `mojo` on the PATH; run this as `pixi run mojo-fuzz` (pixi brings the pinned compiler)'); process.exit(2); } +if (version !== MOJO_VERSION) { console.error(`mojo-fuzz: the local compiler is Mojo ${version}, and the pin is ${MOJO_VERSION}`); process.exit(2); } +const keep = opt('keep'); +const workdir = keep ? (fs.mkdirSync(keep, { recursive: true }), path.resolve(keep)) : fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-mojo-fuzz-')); +let next = 0; +await Promise.all(Array.from({ length: JOBS }, async () => { while (next < PROGRAMS) { const n = next++; await (STATEMENTS ? oneWholeProgram : oneProgram)(n, workdir); if (!flag('json') && process.stderr.isTTY) process.stderr.write(`\r${next}/${PROGRAMS} `); } })); +if (STATEMENTS) await reportStatements(workdir); +if (!keep) fs.rmSync(workdir, { recursive: true, force: true }); +if (STATEMENTS) process.exit(wholeOurs.length ? 1 : 0); + +/** The outermost operator or built-in of an expression: findings that share it and a message are one shape. */ +const head = (e) => (/^([A-Z_0-9]+)\(/.exec(e) || [])[1] ?? ((/\b(DIV|MOD|AND|OR|NOT|IN)\b|[&^*/+\-]|[<>=!]=?/.exec(e.replace(/\([^()]*\)/g, '_')) || [])[0] ?? 'value'); +const upstream = findings.filter((f) => UPSTREAM.some((u) => u.kind === f.kind && u.test(f))); +const ours = findings.filter((f) => !upstream.includes(f)); +const shapes = new Map(); +for (const f of ours) { + const what = f.kind === 'rejected' ? f.detail.replace(/'[^']*'/g, "'…'").replace(/\d+/g, 'N') : head(f.expr); + const key = `${f.kind} ${what}`; + const s = shapes.get(key) ?? { kind: f.kind, what, count: 0, example: f }; + s.count++; + if (f.expr.length < s.example.expr.length) s.example = f; + shapes.set(key, s); +} +const sorted = [...shapes.values()].sort((a, b) => a.kind.localeCompare(b.kind) || b.count - a.count); +if (flag('json')) console.log(JSON.stringify({ compiler: `Mojo ${version}`, seed: SEED, counts, shapes: sorted, upstream, gaps: Object.fromEntries(gaps) }, null, 2)); +else { + console.log(`Mojo ${version}, seed ${SEED}: ${PROGRAMS} programs, ${counts.lines} lines held to the pseudo (${counts.agree} print the same, ${counts.rounding} differ in their last digits only, ${counts.inexact} more by what Mojo's **, exp and log2 lose)`); + console.log(`of ${counts.candidates} candidate lines the pseudo's checker refused ${counts.refusedByChecker}, a pseudo run stopped on ${counts.stoppedInPseudo}, and the translator said "not translated" for ${counts.notTranslated}`); + for (const kind of ['rejected', 'stopped', 'differs', 'interpreter', 'checker']) { + const ss = sorted.filter((s) => s.kind === kind); + if (!ss.length) continue; + console.log(`\n${kind}: ${ss.reduce((n, s) => n + s.count, 0)} line(s), ${ss.length} shape(s)`); + for (const s of ss) { console.log(` [${s.count}x] OUTPUT ${s.example.expr}`); console.log(` ${s.example.detail}`); if (s.example.mojo) console.log(` ${s.example.mojo}`); } + } + if (gaps.size) { console.log('\nnot translated (the translator says so; not a finding):'); for (const [why, k] of [...gaps].sort((a, b) => b[1] - a[1])) console.log(` [${k}x] ${why}`); } + for (const u of UPSTREAM) { + const fs_ = upstream.filter((f) => u.kind === f.kind && u.test(f)); + if (fs_.length) console.log(`\nthe compiler's own (${fs_.length} line(s), not counted): ${u.why}\n e.g. OUTPUT ${fs_.reduce((a, b) => (b.expr.length < a.expr.length ? b : a)).expr}`); + } + if (!ours.length) console.log('\nnothing found'); +} +process.exit(ours.length ? 1 : 0); diff --git a/scripts/mojo/listings.mjs b/scripts/mojo/listings.mjs new file mode 100644 index 0000000..16e2a68 --- /dev/null +++ b/scripts/mojo/listings.mjs @@ -0,0 +1,151 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Turn godbolt's saved answers into x86-64 panes. + * + * For every program in library/mojo/local/ that has a golden with assembly (the header + * says `# mode: asm` or `# mode: both`), find the function the exercise is + * about, translate it to the model machine's syntax the way "Send to x86-64 + * pane" does (src/mojo/att.ts), and write src/mojo/generated/listings.ts. The + * catalog hands that listing to any exercise that names the program and + * leaves its x86-64 pane empty (src/core/catalog.ts). + * + * node scripts/mojo/listings.mjs # all programs + * node scripts/mojo/listings.mjs demo # all programs, and say what happened to this one + * + * Needs `pixi run build` first (it imports from dist/). Offline: it reads the + * goldens only. A program with no matching golden (never verified, or edited + * since) simply has no listing yet, and the script says so. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { fileURLToPath } from 'node:url'; + +const ROOT = fileURLToPath(new URL('../..', import.meta.url)); + +/** The header keys scripts/mojo/mojocorpus/verify.py reads, and emit_ts.py strips from what the pane shows. */ +const HEADER = /^#\s*(?:probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(.*)$/; + +/** What emit_ts.py does to a program before the Mojo pane shows it: drop the header, then leading blank lines. Returns the source and how many lines went. */ +export function stripHeader(text) { + const lines = text.split('\n'); + let dropped = 0; + while (lines.length && HEADER.test(lines[0])) { lines.shift(); dropped++; } + while (lines.length && lines[0] === '') { lines.shift(); dropped++; } + return { source: lines.join('\n'), dropped }; +} + +/** The `# key: value` lines at the top of a program (only the ones verify.py knows), the last of each key winning. */ +export function readHeader(text) { + const h = {}; + for (const line of text.split('\n')) { + if (!line.startsWith('#')) break; + const m = /^#\s*(probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(.*)$/.exec(line); + if (m) h[m[1]] = m[2].trim(); + } + return h; +} + +/** godbolt marks the program's own lines (`main`) and the library's; only the program's become `; mojo line N`. */ +function programLine(a, offset) { + const mine = a.main === true || (a.main !== false && (!a.file || String(a.file).split('/').pop() === 'example.mojo')); + if (!mine || typeof a.line !== 'number') return null; + const n = a.line - offset; + return n >= 1 ? n : null; +} + +/** The golden that answers exactly this program: same source, mode and arguments (all of them: the pin's, then the program's own), and it carries a listing. */ +export function findGolden(goldenDir, text, mode, args) { + const want = crypto.createHash('sha256').update(Buffer.from(text, 'utf8')).digest('hex'); + for (const f of fs.readdirSync(goldenDir)) { + if (!f.endsWith('.json')) continue; + const raw = fs.readFileSync(path.join(goldenDir, f), 'utf8'); + if (!raw.includes(want)) continue; // cheap test before parsing the whole file + const g = JSON.parse(raw); + if (g.sourceSha256 === want && g.mode === mode && (g.args ?? '') === args && Array.isArray(g.result?.asm)) return g; + } + return null; +} + +/** The function to start from: the first `def` in the program that is not `main`, as it appears in the listing. */ +export function pickEntry(paneSource, functions) { + const user = functions.filter((f) => f.kind === 'user' && !/^main\(/.test(f.name)); + for (const m of paneSource.matchAll(/^def\s+(\w+)/gm)) { + const hit = user.find((f) => f.name.startsWith(m[1] + '(')); + if (hit) return hit; + } + return user[0] ?? null; +} + +/** + * The listing for one program text, or { why } when there is none. + * `att` and `godbolt` are the compiled modules, passed in so tests can load them from dist/; `mojoBuildArgs` puts + * the pin's build arguments before the program's own, as godbolt was asked. + */ +export function buildListing(text, goldenDir, { forSimulator, simulatorSetup, listingFunctions, mojoBuildArgs }) { + const h = readHeader(text); + const mode = h.mode ?? 'run', args = mojoBuildArgs(h.args ?? ''); + if (mode === 'run') return { why: 'its header says `# mode: run`; use `# mode: both` to keep the compiler\'s listing' }; + const g = findGolden(goldenDir, text, mode, args); + if (!g) return { why: 'no godbolt answer with a listing for this exact source yet (run `pixi run play `, which asks godbolt)' }; + if (!g.result.compiled) return { why: 'godbolt could not compile it, so there is no listing' }; + const { source: pane, dropped } = stripHeader(text); + const asm = g.result.asm.map((a) => ({ text: a.text ?? '', line: programLine(a, dropped) })); + const fns = listingFunctions(asm); + const entry = pickEntry(pane, fns); + if (!entry) return { why: `the listing holds none of the program's own functions${/-O0/.test(args) ? '' : ' (the optimizer may have inlined them all; put `-O0` in `# args:` to keep each one)'}` }; + // the entry and the program's own functions it calls, exactly what "Send to x86-64 pane" sends for it + // (docs/mojo-corpus.md, "The assembly"); the whole program would be every function the compiler kept at -O0 + const { source, data } = forSimulator(fns, entry); + return { entry: entry.name, asm: source, setup: simulatorSetup(entry, data), build: args }; +} + +/** listings.ts with its header and interface kept and the table rewritten, programs in name order so the diff is only what changed. */ +export function render(listings) { + const sorted = Object.fromEntries(Object.keys(listings).sort().map((k) => [k, listings[k]])); + const head = fs.readFileSync(path.join(ROOT, 'src/mojo/generated/listings.ts'), 'utf8').split('export const MOJO_LISTINGS')[0]; + return `${head}export const MOJO_LISTINGS: Readonly> = ${JSON.stringify(sorted, null, 4)};\n`; +} + +/** + * The listing of every program in `localDir` that has one, by name, and why each other program has none. + * What main() writes, and what tests/listings.test.mjs holds the committed listings.ts to. + */ +export function buildAll(localDir, goldenDir, mods) { + const listings = {}, why = {}; + for (const f of fs.readdirSync(localDir).filter((x) => x.endsWith('.mojo')).sort()) { + const name = f.replace(/\.mojo$/, ''); + const r = buildListing(fs.readFileSync(path.join(localDir, f), 'utf8'), goldenDir, mods); + if (r.why) why[name] = r.why; + else listings[name] = { entry: r.entry, asm: r.asm, setup: r.setup, build: r.build }; + } + return { listings, why }; +} + +async function main() { + let att, godbolt; + try { + att = await import(path.join(ROOT, 'dist/mojo/att.js')); + godbolt = await import(path.join(ROOT, 'dist/services/godbolt.js')); + } catch (e) { + console.error('listings: dist/ is missing or stale; run `pixi run build` first.\n ' + e.message); + process.exit(1); + } + const mods = { forSimulator: att.forSimulator, simulatorSetup: att.simulatorSetup, listingFunctions: godbolt.listingFunctions, mojoBuildArgs: godbolt.mojoBuildArgs }; + const localDir = path.join(ROOT, 'library/mojo/local'), goldenDir = path.join(ROOT, 'library/mojo/golden'); + const named = process.argv.slice(2).filter((a) => !a.startsWith('-')); + const { listings, why } = buildAll(localDir, goldenDir, mods); + for (const name of named) if (why[name]) console.log(`listings: ${name}: no listing — ${why[name]}`); + for (const [name, l] of Object.entries(listings)) { + if (named.includes(name) || !named.length) console.log(`listings: ${name}: ${l.entry}, ${l.asm.split('\n').length} lines of x86-64`); + } + const out = path.join(ROOT, 'src/mojo/generated/listings.ts'); + const next = render(listings); + const changed = fs.readFileSync(out, 'utf8') !== next; + if (changed) fs.writeFileSync(out, next); + console.log(`listings: ${Object.keys(listings).length} program(s) with a listing; ${changed ? 'listings.ts rewritten' : 'listings.ts unchanged'}`); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main(); diff --git a/scripts/mojo/local.mjs b/scripts/mojo/local.mjs new file mode 100644 index 0000000..2592439 --- /dev/null +++ b/scripts/mojo/local.mjs @@ -0,0 +1,345 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Hold core to the pinned Mojo compiler on this machine. + * + * Godbolt's stored answers cover the programs someone wrote down. This covers every Mojo program core can + * produce or holds, built and run by the local `mojo` (pixi provides the pinned one), with no network: + * + * own library/mojo/probes, local and stress, as written + * generated the Mojo the translator writes for every exercise, every pseudo regression program, and every + * library routine on its cases + * reference every Mojo reference implementation of the library, in the harness library-check runs it as + * + * For each program it asks four questions, and prints every "no": + * + * checker does the checker stay silent on a program the compiler accepts? (AGENTS.md, rule 4) + * interpreter does the interpreter print what the built program prints, and end the same way, or decline? + * translator does the compiler accept generated Mojo, and does it print what the pseudo prints? + * godbolt does this compiler agree with godbolt's stored answer for the same text? (a different + * answer means the two compilers, or the two machines, differ: worth knowing, never a bug of ours) + * + * One kind of difference is reported and does not fail the pass: generated Mojo whose numbers differ from the + * pseudo's only in their last digits (at most ROUNDING apart, relatively). The compiled program's std.math is + * its own (hypot differs in the last digit, exp and ** are good to about 3e-13 in Mojo 1.2.0), so those digits + * are the compiler's to decide; docs/trust.md says so. Its arithmetic is not: every program is built with the + * pin's arguments (library/mojo/pin.json build.args), contraction off among them, so a * b + c is rounded twice, + * as the interpreter and the pseudo round it. + * + * node scripts/mojo/local.mjs [--only own|generated|reference] [--item ] [--jobs N] [--json] + * [--no-cache] [--keep DIR] [--list] + * + * Exit status: 0 when every answer is yes (or a listed, explained exception), 1 otherwise, 2 when the local + * compiler is missing or is not the pinned version. Needs a build (dist/). Answers are cached in + * .tools/mojo-local/ by compiler version, arguments, input and source, so a second pass is quick; --no-cache builds + * everything again and keeps none of it. + * + * A built program that traps (an index out of range ends in an illegal instruction) is run with its core-dump + * limit set to 1, the value the kernel reads as "do not pipe this dump anywhere", so a deliberate crash does + * not reach systemd-coredump or the desktop. + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { spawn, spawnSync } from 'node:child_process'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const ROOT = fileURLToPath(new URL('../..', import.meta.url)); +const dist = (p) => pathToFileURL(path.join(ROOT, 'dist', p)).href; +const argv = process.argv.slice(2); +const opt = (name, dflt = null) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : dflt; }; +const flag = (name) => argv.includes(`--${name}`); +const sha256 = (s) => crypto.createHash('sha256').update(s).digest('hex'); +const readJson = (p) => JSON.parse(fs.readFileSync(p, 'utf8')); +const norm = (s) => String(s ?? '').replace(/\r\n/g, '\n').replace(/\n+$/, ''); + +if (!fs.existsSync(path.join(ROOT, 'dist', 'mojo', 'check.js'))) { console.error('mojo-local: dist/ is missing; run `pixi run build` first'); process.exit(2); } + +const { MOJO_VERSION, GODBOLT_COMPILER } = await import(dist('mojo/generated/pin.js')); +const { checkMojo } = await import(dist('mojo/check.js')); +const { runMojo } = await import(dist('mojo/interp.js')); +const { studyOf, runPseudo } = await import(dist('services/pseudo.js')); +const { pseudoToMojo } = await import(dist('sync/pseudo-to-mojo.js')); +const { formatAsMojo, sameOutput, outputDistance } = await import(dist('sync/compare.js')); +const { EXERCISES } = await import(dist('exercises/index.js')); +const { libraryModules } = await import(dist('library/registry.js')); +const { readHeader } = await import(pathToFileURL(path.join(ROOT, 'scripts', 'mojo', 'listings.mjs')).href); +// every program is built with the pin's arguments, then its own: on an x86-64 machine, the arguments godbolt is given +const { mojoBuildArgs } = await import(dist('services/godbolt.js')); +const HOST = { x86_64: process.arch === 'x64' }; + +/** + * Differences that are right, each with its reason. A listed program that stops differing fails the pass, so + * the list can only shrink. Keys are ` `. + */ +/** The largest relative difference between two printed numbers that still counts as rounding, not as another answer. */ +export const ROUNDING = 1e-9; + +export const EXPECTED = { + 'translator generated/18-repeat-input.pseudo': 'Mojo\'s input() opens its own buffered reader on every call, so on piped input the first call takes every line and the second raises EOF; on a terminal each call reads one line, as the notation does', + 'interpreter generated/18-repeat-input.pseudo': 'the interpreter reads one line per input(), as a terminal gives them; the built program was given a pipe (see the line above)', + 'interpreter probes/struct-field-loop-copy': 'a bug of Mojo 1.2.0, not of the interpreter: the built program frees a struct\'s List field twice and crashes on a later pass; the interpreter runs all three', + 'interpreter probes/input-lines': 'the interpreter reads one line per input(), as a terminal gives them; on piped input the second input() raises EOF', + 'translator generated/33-stops-div.pseudo': 'dividing by zero stops the notation; a Float64 division in Mojo gives inf or nan and goes on', + 'translator generated/44-parallel-race.pseudo': 'the notation detects a data race in PARALLEL FOR; Mojo runs the passes in order', + 'translator generated/45-parallel-neighbor-race.pseudo': 'the notation detects a data race in PARALLEL FOR; Mojo runs the passes in order', + 'translator generated/35-stops-unset.pseudo': 'the pseudo reads a variable that was never given a value; the compiler rejects the Mojo, as the checker does', + 'translator generated/36-stops-counter.pseudo': 'the pseudo reads a FOR counter after its loop; the compiler rejects the Mojo, as the checker does', + // the one program built with contraction on, as its header asks (# args: --fp-mode contract=fast): the built + // program fuses a multiply and a subtraction into one rounding; the interpreter rounds twice, as a program built + // with the pin's arguments does (probes/float-contract-runtime-off) + 'interpreter probes/float-contract-runtime': 'built with contraction on, as its own # args: ask, the program computes x * x - x from stdin with one rounding (a fused multiply-subtract); the interpreter rounds twice, as every program built with the pin\'s arguments does', +}; + +/* ------------------------------------------------------------------ */ +/* The programs */ +/* ------------------------------------------------------------------ */ + +function ownPrograms() { + const out = []; + for (const dir of ['probes', 'local', 'stress']) { + for (const f of fs.readdirSync(path.join(ROOT, 'library', 'mojo', dir)).filter((x) => x.endsWith('.mojo')).sort()) { + const source = fs.readFileSync(path.join(ROOT, 'library', 'mojo', dir, f), 'utf8'); + const h = readHeader(source); + const stdin = h.stdin ? h.stdin.replace(/\\n/g, '\n').replace(/\\t/g, '\t') : ''; + out.push({ id: `${dir}/${f.replace(/\.mojo$/, '')}`, kind: 'own', source, stdin, args: mojoBuildArgs(h.args ?? '', HOST), godboltArgs: mojoBuildArgs(h.args ?? ''), mode: h.mode ?? 'run' }); + } + } + return out; +} + +function generatedPrograms() { + const out = []; + const add = (id, pseudo, call, input) => { + const st = studyOf(pseudo); + if (!st.ok) return; + const tr = runPseudo(st, { call, input, maxSteps: 2000000 }); + const mj = pseudoToMojo(pseudo, { call }); + if ('unsupported' in mj) return; + const gaps = mj.notes.filter((n) => n.id === 'translate-gap'); + const want = call && tr.raw.result !== undefined ? [...tr.raw.output, formatAsMojo(tr.raw.result)].join('\n') : tr.raw.output.join('\n'); + out.push({ id: `generated/${id}`, kind: 'generated', source: mj.source, stdin: input.length ? `${input.join('\n')}\n` : '', args: mojoBuildArgs('', HOST), mode: 'run', + pseudo: { stops: tr.error ? `${tr.error.kind}: ${tr.error.message}` : null, want }, gaps: gaps.map((g) => g.message) }); + }; + for (const e of EXERCISES) if (e.sources.pseudo) add(`ex:${e.id}`, e.sources.pseudo, e.call ?? null, []); + const dir = path.join(ROOT, 'library', 'pseudo', 'programs'); + for (const f of fs.readdirSync(dir).filter((x) => x.endsWith('.pseudo')).sort()) { + const src = fs.readFileSync(path.join(dir, f), 'utf8'); + add(f, src, null, [...src.matchAll(/^\/\/ INPUT: (.*)$/gm)].map((m) => m[1])); + } + for (const m of libraryModules()) for (const r of m.routines) { + if (!r.cases.length) continue; + add(`lib:${r.id}`, `USE ${r.module}.${r.name}\n${r.cases.map((c) => `OUTPUT ${c.call}`).join('\n')}\n`, null, []); + } + return out; +} + +function referencePrograms() { + const p = spawnSync(process.execPath, [path.join(ROOT, 'scripts', 'library.mjs'), 'mojo-sources'], { encoding: 'utf8', maxBuffer: 1 << 26 }); + if (p.status !== 0) throw new Error(`scripts/library.mjs mojo-sources failed: ${p.stderr}`); + const wants = new Map(); + for (const m of libraryModules()) for (const r of m.routines) wants.set(r.id, r); + return JSON.parse(p.stdout).map((x) => { + const [rid, lvl] = x.id.split(' '); + const r = wants.get(rid); + const ref = r?.references.find((y) => `${y.lang}/${y.level}` === lvl); + // the cases arrive on stdin, as library-check and godbolt give them: no call has literal arguments the compiler could fold + return { id: `reference/${rid}/${lvl.replace('mojo/', '')}`, kind: 'reference', source: x.source, stdin: x.stdin, args: mojoBuildArgs('', HOST), mode: 'run', + cases: r?.cases ?? [], rel: ref?.rel ?? 0, abs: ref?.abs ?? 0 }; + }); +} + +/* ------------------------------------------------------------------ */ +/* The local compiler */ +/* ------------------------------------------------------------------ */ + +function compilerVersion() { + const p = spawnSync('mojo', ['--version'], { encoding: 'utf8' }); + if (p.status !== 0) return null; + return (/Mojo\s+(\S+)/.exec(p.stdout) || [])[1] ?? null; +} + +const exec = (cmd, args, { cwd, input = '', timeout = 60000 } = {}) => new Promise((resolve) => { + const c = spawn(cmd, args, { cwd, stdio: ['pipe', 'pipe', 'pipe'] }); + let stdout = ''; let stderr = ''; let timedOut = false; + const t = setTimeout(() => { timedOut = true; c.kill('SIGKILL'); }, timeout); + c.stdout.on('data', (d) => { if (stdout.length < 1 << 22) stdout += d; }); + c.stderr.on('data', (d) => { if (stderr.length < 1 << 22) stderr += d; }); + c.on('error', (e) => { clearTimeout(t); resolve({ status: null, signal: null, stdout, stderr: String(e), timedOut }); }); + c.on('close', (status, signal) => { clearTimeout(t); resolve({ status, signal, stdout, stderr, timedOut }); }); + c.stdin.on('error', () => { }); + c.stdin.end(input); +}); + +const SIGNALS = os.constants.signals; +const HAVE_PRLIMIT = spawnSync('prlimit', ['--version'], { encoding: 'utf8' }).status === 0; + +/** Build one program and run it: { compiled, buildStderr, executed, exitCode, stdout, stderr, timedOut }, in godbolt's shape. */ +async function buildAndRun(p, workdir) { + const dir = fs.mkdtempSync(path.join(workdir, 'p-')); + // Mojo keeps compiled objects in a cache of its own, and a trap's message carries the path the source had when it was + // first compiled, by whoever compiled it: any path to an example.mojo is read as godbolt's /app/example.mojo + const app = (s) => s.split(`${dir}/`).join('/app/').replace(/(?:\/[^\s:"']+)?\/example\.mojo/g, '/app/example.mojo'); + fs.writeFileSync(path.join(dir, 'example.mojo'), p.source); + const extra = p.args.split(/\s+/).filter(Boolean); + const b = await exec('mojo', ['build', ...extra, 'example.mojo', '-o', 'example'], { cwd: dir, timeout: 180000 }); + if (b.status !== 0 || !fs.existsSync(path.join(dir, 'example'))) { + return { compiled: false, buildStderr: app(b.stderr || b.stdout), executed: false, exitCode: null, stdout: '', stderr: '', timedOut: b.timedOut }; + } + // a trap must not become a core dump on someone's desktop: limit 1 tells the kernel not to pipe it + const r = HAVE_PRLIMIT ? await exec('prlimit', ['--core=1:1', './example'], { cwd: dir, input: p.stdin, timeout: 30000 }) + : await exec('./example', [], { cwd: dir, input: p.stdin, timeout: 30000 }); + const exitCode = r.timedOut ? null : r.status ?? (r.signal ? 128 + (SIGNALS[r.signal] ?? 0) : null); + return { compiled: true, buildStderr: app(b.stderr), executed: true, exitCode, stdout: app(r.stdout), stderr: app(r.stderr), timedOut: r.timedOut }; +} + +/* ------------------------------------------------------------------ */ +/* The questions */ +/* ------------------------------------------------------------------ */ + +const goldenFor = (() => { + let verdicts = null; + return (id) => { + const dir = path.join(ROOT, 'library', 'mojo', 'golden'); + if (!verdicts) { try { verdicts = new Map(readJson(path.join(dir, 'status.json')).items.map((r) => [r.id, r])); } catch { verdicts = new Map(); } } + const key = verdicts.get(id)?.key; + return key && fs.existsSync(path.join(dir, `${key}.json`)) ? readJson(path.join(dir, `${key}.json`)) : null; + }; +})(); + +const firstError = (stderr) => (stderr.split('\n').find((l) => /error:/.test(l)) ?? stderr.split('\n')[0] ?? '').trim().slice(0, 200); + +/** Does a printed value agree with the pseudo's result, within a reference's stated tolerance? */ +function agreesWithin(printed, want, rel, abs) { + const a = Number(String(printed).trim().replace(/^(-?)inf$/i, '$1Infinity')); const b = Number(String(want).trim().replace(/^(-?)inf$/i, '$1Infinity')); + if (Number.isNaN(a) || Number.isNaN(b)) return sameOutput(String(printed), String(want)); + return a === b || Math.abs(a - b) <= Math.max(rel * Math.max(Math.abs(a), Math.abs(b)), abs); +} + +function judge(p, r) { + const found = []; + const say = (question, detail) => found.push({ question, id: p.id, detail }); + const checked = checkMojo(p.source); + const errors = checked.diagnostics.filter((d) => d.severity === 'error'); + + // checker: never an error on a program the compiler accepts + if (r.compiled && errors.length) say('checker', `the compiler accepts it, and the checker reports: ${errors.map((d) => `L${d.line} ${d.message}`).join(' | ').slice(0, 300)}`); + + // interpreter: what the built program prints and how it ends, or it declines + if (r.compiled && r.executed && !r.timedOut && !errors.length) { + const run = runMojo(checked.module, p.source, { input: p.stdin ? p.stdin.replace(/\n$/, '').split('\n') : [], maxSteps: 400000, record: false }); + if (!run.refused && !run.limited && (norm(run.stdout) !== norm(r.stdout) || run.exitCode !== r.exitCode)) { + say('interpreter', `printed ${JSON.stringify(norm(run.stdout).slice(0, 160))} and exited ${run.exitCode}; the compiler's program printed ${JSON.stringify(norm(r.stdout).slice(0, 160))} and exited ${r.exitCode}`); + } + p.interp = run.refused ? 'declined' : run.limited ? 'limited' : 'ran'; + } + + // translator: generated Mojo compiles, and prints what the pseudo prints + if (p.kind === 'generated' && !p.gaps.length) { + if (!r.compiled) say('translator', `the compiler rejects the generated Mojo: ${firstError(r.buildStderr)}`); + else if (p.pseudo.stops) { if (r.exitCode === 0) say('translator', `the pseudo stops (${p.pseudo.stops}); the Mojo ran to the end and printed ${JSON.stringify(norm(r.stdout).slice(0, 120))}`); } + else if (r.exitCode !== 0) say('translator', `the Mojo stopped (exit ${r.exitCode}${r.timedOut ? ', timed out' : ''}): ${norm(r.stdout + r.stderr).split('\n').pop().slice(0, 160)}; the pseudo ran to the end`); + else { + const far = outputDistance(p.pseudo.want, r.stdout); + if (far > ROUNDING) say('translator', `the pseudo printed ${JSON.stringify(p.pseudo.want.slice(0, 160))}; the Mojo printed ${JSON.stringify(norm(r.stdout).slice(0, 160))}`); + else if (far > 0) say('rounding', `the last digits differ, by at most ${far.toExponential(1)} of the value`); + } + } + + // a library reference: the pseudo's recorded result on every case, within its tolerance + if (p.kind === 'reference') { + if (!r.compiled) say('translator', `the compiler rejects the reference: ${firstError(r.buildStderr)}`); + else if (r.exitCode !== 0) say('translator', `the reference stopped (exit ${r.exitCode})`); + else { + const got = norm(r.stdout).split('\n'); + if (got.length !== p.cases.length) say('translator', `printed ${got.length} line(s) for ${p.cases.length} case(s)`); + else p.cases.forEach((c, i) => { if (c.result !== undefined && !agreesWithin(got[i], c.result, p.rel, p.abs)) say('translator', `${c.call} printed ${got[i]}; the pseudo gives ${c.result}`); }); + } + } + + // godbolt: the stored answer for the same text, arguments and input, when there is one (godbolt gives the program + // its stdin as a pipe, as this pass does: probes/input-pipe-first-line, godbolt-stdin) + if (p.kind === 'own') { + const g = goldenFor(p.id); + if (g && g.compiler === GODBOLT_COMPILER && g.sourceSha256 === sha256(p.source) && (g.args ?? '') === p.args && p.args === p.godboltArgs && (g.stdin ?? '') === p.stdin) { + if (g.result.compiled !== r.compiled) say('godbolt', `godbolt ${g.result.compiled ? 'compiled' : 'rejected'} it; this compiler ${r.compiled ? 'compiled' : 'rejected'} it: ${firstError(r.buildStderr)}`); + else if (g.result.executed && r.executed && (norm(g.result.stdout) !== norm(r.stdout) || g.result.exitCode !== r.exitCode)) { + say('godbolt', `godbolt printed ${JSON.stringify(norm(g.result.stdout).slice(0, 160))} and exited ${g.result.exitCode}; here it printed ${JSON.stringify(norm(r.stdout).slice(0, 160))} and exited ${r.exitCode}`); + } + p.godbolt = 'compared'; + } + } + return found; +} + +/* ------------------------------------------------------------------ */ +/* Run */ +/* ------------------------------------------------------------------ */ + +const only = opt('only'); +const item = opt('item'); +let programs = [ + ...(!only || only === 'own' ? ownPrograms() : []), + ...(!only || only === 'generated' ? generatedPrograms() : []), + ...(!only || only === 'reference' ? referencePrograms() : []), +]; +if (item) programs = programs.filter((p) => p.id.includes(item)); +if (flag('list')) { for (const p of programs) console.log(p.id); process.exit(0); } + +const version = compilerVersion(); +if (!version) { console.error('mojo-local: no `mojo` on the PATH; run this as `pixi run mojo-local` (pixi brings the pinned compiler)'); process.exit(2); } +if (version !== MOJO_VERSION) { console.error(`mojo-local: the local compiler is Mojo ${version}, and the pin is ${MOJO_VERSION}; an answer from another version says nothing about the pin`); process.exit(2); } + +const cacheDir = path.join(ROOT, '.tools', 'mojo-local'); +fs.mkdirSync(cacheDir, { recursive: true }); +const keep = opt('keep'); +const workdir = keep ? (fs.mkdirSync(keep, { recursive: true }), path.resolve(keep)) : fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-mojo-local-')); +const jobs = Math.max(1, Number(opt('jobs', '2')) || 2); +const useCache = !flag('no-cache'); + +const findings = []; +const counts = { programs: programs.length, compiled: 0, rejected: 0, cached: 0, interpreterRan: 0, interpreterDeclined: 0, godboltCompared: 0 }; +let next = 0; +async function worker() { + while (next < programs.length) { + const p = programs[next++]; + const key = sha256(JSON.stringify([version, p.args, p.stdin, p.source])); + const file = path.join(cacheDir, `${key}.json`); + let r = useCache && fs.existsSync(file) ? readJson(file) : null; + if (r) { counts.cached++; for (const k of ['stdout', 'stderr', 'buildStderr']) r[k] = String(r[k] ?? '').replace(/(?:\/[^\s:"']+)?\/example\.mojo/g, '/app/example.mojo'); } + else { r = await buildAndRun(p, workdir); if (useCache && !r.timedOut) fs.writeFileSync(file, `${JSON.stringify(r)}\n`); } + if (r.compiled) counts.compiled++; else counts.rejected++; + findings.push(...judge(p, r)); + if (p.interp === 'ran') counts.interpreterRan++; + if (p.interp === 'declined') counts.interpreterDeclined++; + if (p.godbolt) counts.godboltCompared++; + if (!flag('json') && process.stderr.isTTY) process.stderr.write(`\r${next}/${programs.length} `); + } +} +await Promise.all(Array.from({ length: jobs }, worker)); +if (!keep) fs.rmSync(workdir, { recursive: true, force: true }); +if (!flag('json') && process.stderr.isTTY) process.stderr.write('\r'); + +const unexpected = []; const expected = []; const rounding = []; +for (const f of findings) (f.question === 'rounding' ? rounding : EXPECTED[`${f.question} ${f.id}`] ? expected : unexpected).push(f); +const ids = new Set(programs.map((p) => p.id)); +const stale = item || only ? [] : Object.keys(EXPECTED).filter((k) => ids.has(k.split(' ')[1]) && !findings.some((f) => `${f.question} ${f.id}` === k)); + +if (flag('json')) console.log(JSON.stringify({ compiler: `Mojo ${version}`, counts, unexpected, expected, rounding, stale }, null, 2)); +else { + console.log(`Mojo ${version} on this machine: ${counts.programs} programs (${counts.compiled} compiled, ${counts.rejected} rejected, ${counts.cached} from the cache)`); + console.log(`the interpreter ran ${counts.interpreterRan} of them and declined ${counts.interpreterDeclined}; ${counts.godboltCompared} compared with godbolt's stored answer`); + for (const q of ['checker', 'interpreter', 'translator', 'godbolt']) { + const fs_ = unexpected.filter((f) => f.question === q); + if (!fs_.length) continue; + console.log(`\n${q}: ${fs_.length}`); + for (const f of fs_) console.log(` ${f.id}: ${f.detail}`); + } + if (rounding.length) { console.log(`\nsame answer, other last digits (the compiler's rounding and std.math): ${rounding.length}`); for (const f of rounding) console.log(` ${f.id}: ${f.detail}`); } + if (expected.length) { console.log(`\nexpected differences: ${expected.length}`); for (const f of expected) console.log(` ${f.id}: ${EXPECTED[`${f.question} ${f.id}`]}`); } + for (const k of stale) console.log(`\nno longer differs; take it off EXPECTED in scripts/mojo/local.mjs: ${k}`); + if (!unexpected.length && !stale.length) console.log('\nevery answer agrees'); +} +process.exit(unexpected.length || stale.length ? 1 : 0); diff --git a/scripts/mojo/mojocorpus/__init__.py b/scripts/mojo/mojocorpus/__init__.py new file mode 100644 index 0000000..522eb94 --- /dev/null +++ b/scripts/mojo/mojocorpus/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Programming+ Mojo corpus pipeline. See docs/mojo-corpus.md.""" diff --git a/scripts/mojo/mojocorpus/common.py b/scripts/mojo/mojocorpus/common.py new file mode 100644 index 0000000..9a2ae4f --- /dev/null +++ b/scripts/mojo/mojocorpus/common.py @@ -0,0 +1,271 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Paths, the pin and the lock, and the small helpers every step shares. + +The pipeline has three files a person reads or edits: + + library/mojo/pin.json what we want: the Mojo version, the godbolt compiler, the + arguments every program is built with (`build.args`), and + for each source repository the ref (tag or commit) to use. + Edit this to move to a new Mojo version. + library/mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for + every source, written by `fetch`. Commit it; a harvest + from the same lock is reproducible. + library/mojo/corpus/ what we made: the processed files Programming+ reads. + Written by `harvest`; never edit by hand. + +Only the Python standard library is used, so the scripts run anywhere Python +3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a +TypeScript module), and nothing else. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable + +REPO = Path(__file__).resolve().parents[3] +MOJO_DIR = REPO / "library" / "mojo" # in pplus-library, the submodule library/ +PIN = MOJO_DIR / "pin.json" +LOCK = MOJO_DIR / "mojo.lock.json" +CORPUS = MOJO_DIR / "corpus" +GOLDEN = MOJO_DIR / "golden" +PROBES = MOJO_DIR / "probes" +LOCAL = MOJO_DIR / "local" +# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app +STRESS = MOJO_DIR / "stress" +GENERATED_TS = REPO / "src" / "mojo" / "generated" + +UNRESOLVED = "UNRESOLVED" + + +class PipelineError(Exception): + """A problem a person must fix (a bad pin, a missing source, a refused zip).""" + + +def log(msg: str) -> None: + print(msg, file=sys.stderr, flush=True) + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def read_json(path: Path) -> Any: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def write_json(path: Path, data: Any) -> None: + """Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small.""" + path.parent.mkdir(parents=True, exist_ok=True) + text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + if path.exists() and path.read_text(encoding="utf-8") == text: + return + path.write_text(text, encoding="utf-8") + + +def write_text(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and path.read_text(encoding="utf-8") == text: + return + path.write_text(text, encoding="utf-8") + + +def rel(path: Path, base: Path = REPO) -> str: + try: + return path.resolve().relative_to(base.resolve()).as_posix() + except ValueError: + return path.as_posix() + + +# -------------------------------------------------------------------------- +# The pin +# -------------------------------------------------------------------------- + + +@dataclass +class Source: + """One upstream repository and where its useful files live inside the zip.""" + + name: str + repo: str + ref: str + markers: list[str] + license: str + note: str = "" + repo_confirmed: bool = True + + @property + def resolved(self) -> bool: + return bool(self.ref) and self.ref != UNRESOLVED + + +@dataclass +class Pin: + version: str + compiler: str + base: str + concurrency: int + timeout: int + sources: dict[str, Source] + newer: list[dict[str, str]] = field(default_factory=list) + older: list[dict[str, str]] = field(default_factory=list) + raw: dict[str, Any] = field(default_factory=dict) + # how every program is built, here and on godbolt: the pin's `build.args`, before a program's own `# args:` + build: str = "" + + +def build_args(base: str, own: str = "", x86_64: bool = True) -> str: + """The arguments a program is built with: the pin's `build.args`, then the program's own (`# args:`). + + An option the program names replaces the base's option of that name, value and all (Mojo refuses a + second --target-cpu, and a probe about --fp-mode states its own). `x86_64` is false for a build on a + machine that is not x86-64 (an Apple silicon Mac): there the base's --target-cpu, an x86-64 level, is left + out, because Mojo refuses an x86-64 CPU for any other target. Godbolt's machines are x86-64. + src/services/godbolt.ts (`mojoBuildArgs`) does the same, and tests/build-args.test.mjs holds the two + to each other. + """ + mine = own.split() + named = {t.split("=", 1)[0] for t in mine if t.startswith("-")} + if not x86_64: + named.add("--target-cpu") + words = base.split() + kept: list[str] = [] + i = 0 + while i < len(words): + takes_value = "=" not in words[i] and i + 1 < len(words) and not words[i + 1].startswith("-") + group = words[i:i + 2] if takes_value else words[i:i + 1] + if group[0].split("=", 1)[0] not in named: + kept += group + i += len(group) + return " ".join(kept + mine) + + +def host_is_x86_64() -> bool: + """Is this machine x86-64, so a build here takes the pin's --target-cpu? (build_args)""" + import platform + return platform.machine().lower() in ("x86_64", "amd64", "x64") + + +def load_pin(path: Path = PIN) -> Pin: + if not path.exists(): + raise PipelineError(f"{rel(path)} is missing") + raw = read_json(path) + try: + sources = { + name: Source( + name=name, + repo=s["repo"], + ref=s.get("ref", UNRESOLVED), + markers=list(s["markers"]), + license=s.get("license", "unknown"), + note=s.get("note", ""), + repo_confirmed=bool(s.get("repoConfirmed", True)), + ) + for name, s in raw["sources"].items() + } + pin = Pin( + version=raw["mojo"]["version"], + compiler=raw["godbolt"]["compiler"], + base=raw["godbolt"]["base"].rstrip("/"), + concurrency=int(raw["godbolt"].get("concurrency", 2)), + timeout=int(raw["godbolt"].get("timeoutSeconds", 90)), + sources=sources, + newer=list(raw.get("dialect", {}).get("newer", [])), + older=list(raw.get("dialect", {}).get("older", [])), + raw=raw, + build=str(raw["build"]["args"]).strip(), + ) + except (KeyError, TypeError, ValueError) as e: + raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e + if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version): + raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z") + for d in pin.newer + pin.older: + re.compile(d["pattern"], re.M) # fail early on a bad marker + return pin + + +def load_lock(path: Path = LOCK) -> dict[str, Any]: + return read_json(path) if path.exists() else {"sources": {}} + + +# -------------------------------------------------------------------------- +# Dialect markers: a cheap first look at whether a program fits the pin +# -------------------------------------------------------------------------- + + +def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]: + """Which pinned markers a program matches. + + `newer`: uses syntax that arrived after the pinned version, so it cannot be + expected to compile there. `older`: uses syntax the pinned version removed, + so it is expected to fail. Only a run on the real compiler decides; these + flags just let the corpus say up front what it expects. + """ + out: dict[str, list[str]] = {} + for kind, markers in (("newer", pin.newer), ("older", pin.older)): + hits = [ + m["id"] + for m in markers + if re.search(m["pattern"], strip_mojo_comments(source), re.M) + ] + if hits: + out[kind] = hits + return out + + +_STRING = re.compile( + r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'' +) + + +def strip_mojo_comments(source: str) -> str: + """Blank out comments and string contents, keeping line structure, so markers + match code and not prose (a docstring that says `fn` is not a use of `fn`).""" + + def blank(m: re.Match[str]) -> str: + return re.sub(r"[^\n]", " ", m.group(0)) + + s = _STRING.sub(blank, source) + return re.sub(r"#[^\n]*", "", s) + + +def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]: + """Where a harvested item came from, recorded beside it.""" + return { + "source": source.name, + "repo": source.repo, + "commit": lock_entry.get("commit") + or f"local:{lock_entry.get('sha256', '')[:12]}", + **( + {"standsInFor": lock_entry["standsInFor"]} + if lock_entry.get("standsInFor") + else {} + ), + "zipSha256": lock_entry.get("sha256", ""), + "path": path, + "license": source.license, + } + + +def iter_files(root: Path, pattern: str) -> Iterable[Path]: + return sorted(p for p in root.glob(pattern) if p.is_file()) + + +def env_flag(name: str) -> bool: + return os.environ.get(name, "") not in ("", "0", "false", "no") diff --git a/scripts/mojo/mojocorpus/emit_ts.py b/scripts/mojo/mojocorpus/emit_ts.py new file mode 100644 index 0000000..1268ff4 --- /dev/null +++ b/scripts/mojo/mojocorpus/emit_ts.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Generated TypeScript: the parts of the corpus the app reads at run time. + +The app is TypeScript compiled by tsc with no bundler, so data it needs at +run time is emitted as .ts modules rather than imported JSON. Each file says +what generated it and from which commit; `npm test` checks they match the +pin and the corpus, so they cannot drift. + + src/mojo/generated/pin.ts the version, godbolt compiler and build arguments, from library/mojo/pin.json + src/mojo/generated/tokens.ts the words of the language, from corpus/tokens/tokens.json + src/mojo/generated/editor.ts comment and indentation rules, from corpus/editor/editor.json + src/mojo/generated/programs.ts our own programs (library/mojo/local/), each with its verdict from the goldens + library/syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py); pplus-web copies it into its Quarto extension + library/THIRD_PARTY.md every harvested file, its source and its license (notices.py) +""" +from __future__ import annotations + +import json +import re +from pathlib import Path + +from .common import CORPUS, GENERATED_TS, REPO, Pin, PipelineError, load_lock, read_json, write_text +from . import notices +from .kde import mojo_xml + +SYNTAX_DIR = REPO / "library" / "syntax" # in pplus-library; pplus-web copies it into its Quarto extension (its quarto-css task) + +BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change library/mojo/pin.json or re-harvest.\n" + + +def ts(value) -> str: + return json.dumps(value, ensure_ascii=False, indent=4).replace("\n", "\n") + + +def emit(pin: Pin, out_dir: Path = GENERATED_TS, corpus: Path = CORPUS, syntax_dir: Path = SYNTAX_DIR) -> list[Path]: + written = [] + lock = load_lock() + commits = {k: (v.get("commit") or f"local:{v.get('sha256', '')[:12]}") for k, v in lock.get("sources", {}).items()} + + pin_ts = ( + BANNER + + "/** The Mojo version Programming+ models, and the compiler that is its ground truth. */\n" + + f"export const MOJO_VERSION = {json.dumps(pin.version)};\n" + + f"export const GODBOLT_BASE = {json.dumps(pin.base)};\n" + + f"export const GODBOLT_COMPILER = {json.dumps(pin.compiler)};\n" + + "/** The arguments every program is built with, on godbolt and on this machine, before a program's own (library/mojo/pin.json, build.args). */\n" + + f"export const MOJO_BUILD_ARGS = {json.dumps(pin.build)};\n" + + f"/** The commit (or local zip hash) of each source the corpus was harvested from. */\n" + + f"export const MOJO_SOURCES: Record = {ts(commits)};\n" + ) + p = out_dir / "pin.ts"; write_text(p, pin_ts); written.append(p) + + tok_path = corpus / "tokens" / "tokens.json" + if not tok_path.exists(): + raise PipelineError("corpus/tokens/tokens.json is missing; run `harvest` first") + t = read_json(tok_path) + kws = t["keywords"] + public = sorted(k["word"] for k in kws if not k["internal"] and not k.get("removed")) + internal = sorted(k["word"] for k in kws if k["internal"]) + removed_kw = {k["word"]: k["replacement"] for k in kws if k.get("removed")} + removed_conv = {s["word"]: s["replacement"] for s in t["softKeywords"] if s.get("removed")} + removed = removed_kw | removed_conv + messages_path = corpus / "messages" / "messages.json" + said: dict[str, str] = {} + if messages_path.exists(): + for m in read_json(messages_path)["messages"]: + hit = re.match(r"^'(\w+)' (?:was|has been) removed; use '(\w+)'", m["message"]) + if hit and hit.group(1) in removed and m["severity"] == "error": + said.setdefault(hit.group(1), m["message"]) + soft = sorted(s["word"] for s in t["softKeywords"] if not s.get("removed")) + punct = sorted({x["spelling"] for x in t["punctuation"]}, key=lambda s: (-len(s), s)) + prov = t["provenance"]["tokenKinds"] + tokens_ts = ( + BANNER + + f"// From {prov['repo']}@{prov['commit'][:18]}{' (standing in for ' + prov['standsInFor'] + ')' if prov.get('standsInFor') else ''}: " + + "TokenKinds.def, lib/MojoParser, the decorator reference, and the stdlib prelude.\n\n" + + "/** Hard keywords: reserved words in every context. */\n" + + f"export const MOJO_KEYWORDS: readonly string[] = {ts(public)};\n\n" + + "/** Compiler-internal keywords (they start with __); valid, but not for students. */\n" + + f"export const MOJO_INTERNAL_KEYWORDS: readonly string[] = {ts(internal)};\n\n" + + "/** Words the parser accepts by spelling in one context, such as argument conventions. */\n" + + f"export const MOJO_SOFT_KEYWORDS: readonly string[] = {ts(soft)};\n\n" + + "/** Words the pinned compiler rejects as removed, and what to write instead. */\n" + + f"export const MOJO_REMOVED_WORDS: Readonly> = {ts(removed)};\n\n" + + "/** Removed words that are tokens of their own: an error wherever they appear. */\n" + + f"export const MOJO_REMOVED_KEYWORDS: readonly string[] = {ts(sorted(removed_kw))};\n\n" + + "/** Removed argument conventions: an error only in front of a parameter. */\n" + + f"export const MOJO_REMOVED_CONVENTIONS: readonly string[] = {ts(sorted(removed_conv))};\n\n" + + "/** The compiler's own words for each removed word, from its source. */\n" + + f"export const MOJO_REMOVED_MESSAGES: Readonly> = {ts(said)};\n\n" + + "/** Contextual words in a function signature. */\n" + + f"export const MOJO_EFFECTS: readonly string[] = {ts(t['effects'])};\n\n" + + "/** Decorators documented in the language reference. */\n" + + f"export const MOJO_DECORATORS: readonly string[] = {ts(t['decorators'])};\n\n" + + "/** Names every module sees without an import. */\n" + + f"export const MOJO_PRELUDE: readonly string[] = {ts(t['prelude'])};\n\n" + + "/** Operator and punctuation spellings, longest first, for a maximal-munch lexer. */\n" + + f"export const MOJO_PUNCTUATION: readonly string[] = {ts(punct)};\n" + ) + p = out_dir / "tokens.ts"; write_text(p, tokens_ts); written.append(p) + xml = mojo_xml(pin.version, prov["repo"], prov["commit"], public, [w for w in soft if w != "where"], sorted(removed), t["prelude"], t["effects"]) + p = syntax_dir / "mojo.xml"; write_text(p, xml); written.append(p) + + ed_path = corpus / "editor" / "editor.json" + if not ed_path.exists(): + raise PipelineError("corpus/editor/editor.json is missing; run `harvest` first") + ed = read_json(ed_path) + cfg = ed["languageConfiguration"] + rules = [r["beforeText"] for r in cfg.get("onEnterRules", []) if r.get("action", {}).get("indent") == "indent" and r.get("beforeText")] + ed_prov = ed["provenance"]["languageConfiguration"] + editor_ts = ( + BANNER + + f"// From {ed_prov['repo']}@{ed_prov['commit'][:12]}: language-configuration.json (VS Code extension {ed.get('extensionVersion')}).\n\n" + + "/** The line comment, for toggling comments. */\n" + + f"export const MOJO_LINE_COMMENT = {json.dumps(cfg.get('comments', {}).get('lineComment', '#'))};\n\n" + + "/** A line matching one of these opens a block: Enter indents the next line (the extension's onEnterRules). */\n" + + f"export const MOJO_INDENT_AFTER: readonly string[] = {ts(rules)};\n\n" + + "/** Bracket pairs. */\n" + + f"export const MOJO_BRACKETS: ReadonlyArray = {ts([list(b) for b in cfg.get('brackets', [])])};\n" + ) + p = out_dir / "editor.ts"; write_text(p, editor_ts); written.append(p) + + from . import verify # here, not at the top: verify imports the whole corpus machinery + doc = verify.status(pin, corpora=("local",), write=False) + verdicts = {r["id"]: r for r in doc["items"]} + programs = {} + for item in verify.collect(("local",), build=pin.build): + body = item.source.split("\n") + while body and verify.HEADER.match(body[0]): + body.pop(0) + r = verdicts[item.id] + programs[item.id.split("/", 1)[1]] = { + "source": "\n".join(body).lstrip("\n"), + "verdict": r["verdict"], + "verified": r["verdict"] == "pass", + "why": r["why"], + } + programs_ts = ( + BANNER + + "// Our own Mojo programs (library/mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n" + + "// `verified` is true only when godbolt's answer passed the program's own expectation.\n\n" + + "export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string }\n\n" + + f"export const MOJO_PROGRAMS: Readonly> = {ts(programs)};\n" + ) + p = out_dir / "programs.ts"; write_text(p, programs_ts); written.append(p) + + idx_path = corpus / "stdindex" / "stdindex.json" + idx = read_json(idx_path) if idx_path.exists() else None + stdlib_ts = ( + BANNER + + (f"// From {idx['provenance']['repo']}@{idx['provenance']['commit'][:18]}: stdlib/std, each package's __init__.mojo and each module's public definitions.\n" + if idx else "// The stdindex processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n") + + "\n/** Which standard-library package (or module) exports each public name; a name exported from two places is left out. */\n" + + f"export const MOJO_STD_EXPORTS: Readonly> = {ts(idx['exports'] if idx else {})};\n" + + "\n/** Prelude functions every overload of which raises (input, atol, ...): calling one needs raises or a try. */\n" + + f"export const MOJO_PRELUDE_RAISES: readonly string[] = {ts((idx or {}).get('raisingPrelude', []))};\n" + ) + p = out_dir / "stdlib.ts"; write_text(p, stdlib_ts); written.append(p) + + ty_path = corpus / "stdtypes" / "stdtypes.json" + ty = read_json(ty_path) if ty_path.exists() else None + traits = {n: {"bases": t["bases"], "defaults": t["methods"]} for n, t in sorted((ty or {}).get("traits", {}).items())} + structs = {n: {"bases": st["bases"], "implicitlyCopyable": n in (ty or {}).get("implicitlyCopyable", [])} for n, st in sorted((ty or {}).get("structs", {}).items())} + types_ts = ( + BANNER + + (f"// From {ty['provenance']['repo']}@{ty['provenance']['commit'][:18]}: the struct and trait declarations in stdlib/std.\n" + if ty else "// The stdtypes processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n") + + "\n/** Each standard trait: the traits it refines, and the methods it gives a conforming struct by default. */\n" + + f"export const MOJO_STD_TRAITS: Readonly> = {ts(traits)};\n\n" + + "/** Each standard struct: the traits it declares, and whether they reach ImplicitlyCopyable. */\n" + + f"export const MOJO_STD_STRUCTS: Readonly> = {ts(structs)};\n" + ) + p = out_dir / "types.ts"; write_text(p, types_ts); written.append(p) + + # library/THIRD_PARTY.md: every harvested file with its source and license (notices.py) + target = (notices.OUT if syntax_dir == SYNTAX_DIR else out_dir / "THIRD_PARTY.md") + p = notices.write(pin, target); written.append(p) + return written diff --git a/scripts/mojo/mojocorpus/fetch.py b/scripts/mojo/mojocorpus/fetch.py new file mode 100644 index 0000000..ade4d64 --- /dev/null +++ b/scripts/mojo/mojocorpus/fetch.py @@ -0,0 +1,252 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Get each source repository as a zip, check it, and unpack the part we use. + +For a GitHub source the pin gives a ref (a tag such as `modular/v26.1`, a +branch, or a commit). `fetch` turns it into a commit with the GitHub API, +downloads that commit's zip, and records the commit and the zip's sha256 in +the lock. Later runs with the same pin reuse the locked commit and refuse a +zip whose hash changed, so a harvest is reproducible. + +A local zip (`--local modular=/path/Mojo.zip`) stands in for the download, +for working offline or with a snapshot someone handed you. The lock records +it as local with its hash; a later harvest must be given the same file. + +Nothing is unpacked into the repository. Zips go to a temporary directory, +only the subtree holding the source's marker files is extracted, and the +directory is removed afterwards unless `--keep` is given. +""" +from __future__ import annotations + +import datetime as _dt +import os +import shutil +import stat +import tempfile +import time +import urllib.error +import urllib.request +import zipfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +from .common import LOCK, Pin, PipelineError, Source, load_lock, log, rel, sha256_file, write_json + +USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing)" +MAX_UNPACKED = 4 << 30 # refuse zips that would unpack past 4 GiB +SKIP_PARTS = {"__MACOSX", ".DS_Store", ".git"} + + +@dataclass +class Fetched: + source: Source + root: Path # the extracted directory that holds the markers + lock: dict[str, Any] # this source's lock entry + + +# -------------------------------------------------------------------------- +# GitHub +# -------------------------------------------------------------------------- + +def _request(url: str, accept: str) -> urllib.request.Request: + headers = {"User-Agent": USER_AGENT, "Accept": accept} + token = os.environ.get("GITHUB_TOKEN") + if token and url.startswith("https://api.github.com/"): + headers["Authorization"] = f"Bearer {token}" + return urllib.request.Request(url, headers=headers) + + +def resolve_ref(repo: str, ref: str) -> str: + """A tag, branch, or commit, as the full commit sha.""" + url = f"https://api.github.com/repos/{repo}/commits/{urllib.request.quote(ref, safe='')}" + try: + with urllib.request.urlopen(_request(url, "application/vnd.github.sha"), timeout=60) as r: + sha = r.read().decode().strip() + except urllib.error.HTTPError as e: + if e.code in (404, 422): + raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in library/mojo/pin.json.") from e + deny = e.headers.get("x-deny-reason") if e.headers else None + if deny: + raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e + if e.code in (403, 429): + raise PipelineError(f"{repo}: GitHub refused the request ({e.code}), usually its rate limit for anonymous use. " + f"Set GITHUB_TOKEN to any token with public read access and run again.") from e + raise PipelineError(f"{repo}: GitHub answered {e.code} while resolving {ref!r}") from e + except urllib.error.URLError as e: + raise PipelineError(f"{repo}: no network to resolve {ref!r} ({e.reason}). Use --local NAME=ZIP to work offline.") from e + if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha): + raise PipelineError(f"{repo}: GitHub answered {sha[:60]!r} for {ref!r}, not a commit sha") + return sha + + +def download(repo: str, commit: str, dest: Path) -> str: + url = f"https://codeload.github.com/{repo}/zip/{commit}" + for attempt in range(4): + try: + with urllib.request.urlopen(_request(url, "application/zip"), timeout=600) as r, open(dest, "wb") as f: + shutil.copyfileobj(r, f, 1 << 20) + return url + except (urllib.error.URLError, TimeoutError) as e: + if attempt == 3: + raise PipelineError(f"{repo}@{commit[:12]}: download failed: {e}") from e + time.sleep(2 ** attempt) + raise AssertionError("unreachable") + + +# -------------------------------------------------------------------------- +# Zips: find the root by its markers, refuse anything unsafe, extract a subtree +# -------------------------------------------------------------------------- + +def _clean_names(zf: zipfile.ZipFile) -> list[zipfile.ZipInfo]: + out = [] + total = 0 + for info in zf.infolist(): + p = PurePosixPath(info.filename) + if any(part in SKIP_PARTS or part.startswith("._") for part in p.parts): + continue + if p.is_absolute() or ".." in p.parts or "\\" in info.filename: + raise PipelineError(f"zip entry {info.filename!r} escapes the archive; refusing the zip") + mode = info.external_attr >> 16 + if stat.S_ISLNK(mode): + continue # never follow links out of the tree + total += info.file_size + out.append(info) + if total > MAX_UNPACKED: + raise PipelineError(f"zip would unpack to {total >> 20} MiB; refusing") + return out + + +def find_root(names: list[str], markers: list[str]) -> str: + """The shortest directory prefix under which every marker exists. + + Markers are paths relative to the root (files or directories), so the same + pin works for `Mojo/stdlib/...` in a hand-made zip and + `modular-/mojo/stdlib/...` from GitHub. + """ + files = set(names) + dirs = set() + for n in names: + parts = n.rstrip("/").split("/") + for i in range(1, len(parts) + (0 if not n.endswith("/") else 1)): + dirs.add("/".join(parts[:i])) + entries = files | {d + "/" for d in dirs} | dirs + + def has(prefix: str, marker: str) -> bool: + m = marker.strip("/") + return (prefix + m) in entries or (prefix + m + "/") in entries + + first = markers[0].strip("/") + candidates = set() + for e in entries: + e2 = e.rstrip("/") + if e2 == first: + candidates.add("") + elif e2.endswith("/" + first): + candidates.add(e2[: -len(first)]) + good = sorted((c for c in candidates if all(has(c, m) for m in markers)), key=lambda c: (c.count("/"), c)) + if not good: + raise PipelineError(f"no directory in the zip holds all of {markers}") + return good[0] + + +def extract_subtree(zip_path: Path, markers: list[str], dest: Path) -> tuple[Path, str]: + with zipfile.ZipFile(zip_path) as zf: + infos = _clean_names(zf) + prefix = find_root([i.filename for i in infos], markers) + for info in infos: + if not info.filename.startswith(prefix) or info.is_dir(): + continue + target = dest / info.filename + target.parent.mkdir(parents=True, exist_ok=True) + with zf.open(info) as src, open(target, "wb") as out: + shutil.copyfileobj(src, out, 1 << 20) + return (dest / prefix) if prefix else dest, prefix.rstrip("/") + + +# -------------------------------------------------------------------------- +# The step +# -------------------------------------------------------------------------- + +def _now() -> str: + return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat() + + +def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = False, + only: set[str] | None = None, allow_hash_change: bool = False, + lock_path: Path = LOCK) -> dict[str, Fetched]: + """Fetch every source the pin names (or `only` those), extract, and rewrite the lock.""" + lock = load_lock(lock_path) + entries: dict[str, Any] = dict(lock.get("sources", {})) + out: dict[str, Fetched] = {} + unknown = set(local) - set(pin.sources) + if unknown: + raise PipelineError(f"--local names unknown source(s): {', '.join(sorted(unknown))}; the pin has {', '.join(pin.sources)}") + + for name, src in pin.sources.items(): + if only and name not in only: + continue + old = entries.get(name, {}) + zip_path = workdir / f"{name}.zip" + + if name in local: + given = local[name] + if not given.exists(): + raise PipelineError(f"--local {name}={given}: no such file") + digest = sha256_file(given) + shutil.copyfile(given, zip_path) + entry = {"kind": "local", "file": given.name, "sha256": digest, "bytes": given.stat().st_size, + "repo": src.repo, "pinnedRef": src.ref, "fetchedAt": _now()} + if src.resolved: + # a zip standing in for a real ref (offline work): say so, and let the next + # online run fetch the ref itself rather than insist on this zip + entry["pinnedRef"] = "local-override" + entry["standsInFor"] = src.ref + if old.get("kind") == "local" and old.get("sha256") == digest: + entry["fetchedAt"] = old.get("fetchedAt", entry["fetchedAt"]) + log(f"[fetch] {name}: local {given.name} sha256 {digest[:12]}") + elif old.get("kind") == "local" and not update and (not src.resolved or old.get("pinnedRef") == src.ref): + # the lock came from a local zip and the pin has not moved: insist on the same zip + raise PipelineError( + f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). " + f"Pass --local {name}=PATH with that file, or set a ref in library/mojo/pin.json and run `fetch --update`.") + else: + if not src.resolved: + raise PipelineError( + f"{name}: library/mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there " + f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.") + reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update + commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref) + url = download(src.repo, commit, zip_path) + digest = sha256_file(zip_path) + if reuse and old.get("sha256") and old["sha256"] != digest and not allow_hash_change: + raise PipelineError( + f"{name}: GitHub's zip for {commit[:12]} hashes to {digest[:12]}, the lock says {old['sha256'][:12]}. " + f"Archives are usually stable; re-run with --allow-hash-change if you accept the new one.") + entry = {"kind": "github", "repo": src.repo, "pinnedRef": src.ref, "commit": commit, "url": url, + "sha256": digest, "bytes": zip_path.stat().st_size, + "fetchedAt": old.get("fetchedAt") if reuse and old.get("sha256") == digest else _now()} + log(f"[fetch] {name}: {src.repo}@{commit[:12]} ({src.ref}) sha256 {digest[:12]}") + + root, prefix = extract_subtree(zip_path, src.markers, workdir / name) + entry["root"] = prefix + entries[name] = entry + out[name] = Fetched(source=src, root=root, lock=entry) + zip_path.unlink(missing_ok=True) + + lock_doc = { + "comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change library/mojo/pin.json instead.", + "mojoVersion": pin.version, + "compiler": pin.compiler, + "sources": {k: entries[k] for k in sorted(entries) if k in pin.sources}, + } + write_json(lock_path, lock_doc) + log(f"[fetch] lock written: {rel(lock_path)}") + return out + + +def make_workdir(keep: bool, given: Path | None) -> tuple[Path, bool]: + if given: + given.mkdir(parents=True, exist_ok=True) + return given, True + return Path(tempfile.mkdtemp(prefix="pplus-mojo-")), keep diff --git a/scripts/mojo/mojocorpus/godbolt.py b/scripts/mojo/mojocorpus/godbolt.py new file mode 100644 index 0000000..944ddb3 --- /dev/null +++ b/scripts/mojo/mojocorpus/godbolt.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Compiler Explorer, the oracle every corpus item is checked against. + +Requests follow Mojo Quest's (src/lib/compile.ts), which its CI runs against +the same compiler id, so a result here means what it means there: + + run compile and execute in one request (an "executor request"); the + answer has buildResult.code, didExecute, code, stdout, stderr, timedOut + asm compile only and return the Intel-syntax listing with source lines + both compile with the listing AND execute; whether one request returns + both is itself a probe (probes/godbolt-asm-and-run.mojo) + +Every answer is normalized to one shape, `Result`, and kept beside the raw +JSON (minus a listing duplicated in `asm`) so a later reading can change. +""" +from __future__ import annotations + +import json +import time +import urllib.error +import urllib.request +from typing import Any + +from .common import PipelineError + +USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing; corpus verification, low rate)" + +ASM_FILTERS = {"binary": False, "execute": False, "intel": True, "demangle": True, "labels": True, + "libraryCode": False, "directives": True, "commentOnly": True, "trim": True} + + +def request_body(source: str, mode: str = "run", stdin: str = "", args: str = "") -> dict[str, Any]: + if mode == "run": + options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin}, + "compilerOptions": {"executorRequest": True, "skipAsm": True}, + "filters": {"execute": True}, "tools": [], "libraries": []} + elif mode == "asm": + options = {"userArguments": args, "compilerOptions": {"executorRequest": False, "skipAsm": False}, + "filters": dict(ASM_FILTERS), "tools": [], "libraries": []} + elif mode == "both": + options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin}, + "compilerOptions": {"executorRequest": False, "skipAsm": False}, + "filters": {**ASM_FILTERS, "execute": True}, "tools": [], "libraries": []} + else: + raise PipelineError(f"unknown godbolt mode {mode!r}") + return {"source": source, "options": options, "lang": "mojo", "allowStoreCodeDebug": False} + + +def _lines(arr: Any) -> str: + return "\n".join((x.get("text", "") if isinstance(x, dict) else str(x)) for x in arr) if isinstance(arr, list) else "" + + +def normalize(mode: str, data: dict[str, Any]) -> dict[str, Any]: + """One shape for every mode: compiled, buildStderr, executed, exitCode, stdout, stderr, timedOut, asm.""" + if mode == "run": + build = data.get("buildResult") or {} + return { + "compiled": build.get("code") == 0, + "buildStderr": _lines(build.get("stderr")), + "executed": bool(data.get("didExecute")), + "exitCode": data.get("code") if isinstance(data.get("code"), int) else None, + "stdout": _lines(data.get("stdout")), + "stderr": _lines(data.get("stderr")), + "timedOut": bool(data.get("timedOut")), + "asm": None, + } + exe = data.get("execResult") or {} + asm = [asm_line(a) for a in data.get("asm") or [] if isinstance(a, dict)] + return { + "compiled": data.get("code") == 0, + "buildStderr": _lines(data.get("stderr")), + "executed": bool(exe.get("didExecute")) if exe else False, + "exitCode": exe.get("code") if isinstance(exe.get("code"), int) else None, + "stdout": _lines(exe.get("stdout")), + "stderr": _lines(exe.get("stderr")), + "timedOut": bool(exe.get("timedOut")), + "asm": asm, + "returnedAsm": bool(asm), + "returnedExec": bool(exe), + } + + +def asm_line(a: dict[str, Any]) -> dict[str, Any]: + """One listing line, with where it came from. `file` and `main` keep godbolt's marking of the source + file as given (`source.file`, `source.mainsource`), so the instructions inlined from the standard + library, whose lines are lines of the library's files, can be told from the program's own.""" + src = a.get("source") or {} + out: dict[str, Any] = {"text": a.get("text", ""), "line": src.get("line")} + if src: + out["file"] = src.get("file") + if "mainsource" in src: + out["main"] = src.get("mainsource") + return out + + +def slim_raw(data: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in data.items() if k not in ("asm", "popularArguments")} + + +def post(base: str, compiler: str, body: dict[str, Any], timeout: int = 90, retries: int = 5, + attempts: list[int] | None = None) -> dict[str, Any]: + """POST and return the answer. `attempts`, if given, receives how many tries it took, so a slow + answer can be told apart from a slow compile (godbolt reports its own compile time separately).""" + url = f"{base}/api/compiler/{urllib.request.quote(compiler, safe='')}/compile" + data = json.dumps(body).encode() + delay = 2.0 + for attempt in range(retries + 1): + req = urllib.request.Request(url, data=data, method="POST", headers={ + "Content-Type": "application/json", "Accept": "application/json", "User-Agent": USER_AGENT}) + if attempts is not None: + attempts[:] = [attempt + 1] + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read().decode("utf-8")) + except urllib.error.HTTPError as e: + if e.code in (429, 500, 502, 503, 504) and attempt < retries: + wait = float(e.headers.get("Retry-After") or delay) + time.sleep(min(wait, 120)); delay *= 2 + continue + raise PipelineError(f"godbolt answered HTTP {e.code} for {compiler}") from e + except (urllib.error.URLError, TimeoutError) as e: + if attempt < retries: + time.sleep(delay); delay *= 2 + continue + raise PipelineError(f"godbolt unreachable: {e}") from e + raise AssertionError("unreachable") diff --git a/scripts/mojo/mojocorpus/harvest.py b/scripts/mojo/mojocorpus/harvest.py new file mode 100644 index 0000000..a53869d --- /dev/null +++ b/scripts/mojo/mojocorpus/harvest.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Fetch the sources, run the processors, and write the provenance report. + +`harvest` is the one command to run after changing library/mojo/pin.json: + + fetch -> a temporary directory holding each source's extracted subtree + processors -> library/mojo/corpus// (each owns its directory) + report -> library/mojo/corpus/SOURCES.md and report.json: what came from where, + under which license, what each processor found, every warning + emit -> src/mojo/generated/*.ts + +The temporary directory is removed afterwards unless --keep is given. +""" +from __future__ import annotations + +import shutil +import zipfile +from pathlib import Path +from typing import Any + +from . import emit_ts +from .common import CORPUS, Pin, PipelineError, log, rel, write_json, write_text +from .fetch import _clean_names, fetch_all, find_root, make_workdir +from .processors import Context, all_processors + + +def detect_local_zips(pin: Pin, directory: Path, include_resolved: bool = False) -> dict[str, Path]: + """Assign each zip in a directory to the one source whose markers it contains. + + A source whose pin names a real ref is fetched from GitHub, so a zip that + matches it is ignored here (pass --local NAME=ZIP to override on purpose). + """ + found: dict[str, Path] = {} + for z in sorted(directory.glob("*.zip")): + try: + with zipfile.ZipFile(z) as zf: + names = [i.filename for i in _clean_names(zf)] + except (zipfile.BadZipFile, PipelineError) as e: + log(f"[harvest] {z.name}: not usable ({e})") + continue + matches = [] + for name, src in pin.sources.items(): + try: + find_root(names, src.markers) + matches.append(name) + except PipelineError: + pass + if len(matches) == 1 and pin.sources[matches[0]].resolved and not include_resolved: + log(f"[harvest] {z.name} looks like {matches[0]}, but the pin fetches {matches[0]} at {pin.sources[matches[0]].ref}; ignored") + elif len(matches) == 1: + if matches[0] in found: + raise PipelineError(f"two zips look like {matches[0]}: {found[matches[0]].name} and {z.name}; pass --local explicitly") + found[matches[0]] = z + log(f"[harvest] {z.name} -> {matches[0]}") + elif len(matches) > 1: + raise PipelineError(f"{z.name} matches several sources ({', '.join(matches)}); pass --local explicitly") + else: + log(f"[harvest] {z.name}: matches no source in the pin; ignored") + return found + + +def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors: set[str] | None = None, + keep: bool = False, workdir: Path | None = None, allow_hash_change: bool = False) -> dict[str, Any]: + procs = [p for p in all_processors() if not processors or p.NAME in processors] + if processors and len(procs) != len(processors): + known = {p.NAME for p in all_processors()} + raise PipelineError(f"unknown processor(s): {', '.join(sorted(processors - known))}; known: {', '.join(sorted(known))}") + needed = {p.SOURCE for p in procs} + wd, keep_it = make_workdir(keep, workdir) + log(f"[harvest] working in {wd}") + try: + fetched = fetch_all(pin, wd, local, update=update, only=needed, allow_hash_change=allow_hash_change) + results: dict[str, Any] = {} + warnings: list[str] = [] + for proc in procs: + f = fetched.get(proc.SOURCE) + if f is None: + raise PipelineError(f"processor {proc.NAME} needs source {proc.SOURCE}, which was not fetched") + missing = [i for i in proc.INPUTS if not (f.root / i).exists()] + if missing: + kept = "its previous output is kept" if (CORPUS / proc.NAME).exists() else "it has no previous output" + w = f"{proc.NAME}: skipped, {proc.SOURCE} has no {', '.join(missing)} at this ref; {kept}" + warnings.append(w); results[proc.NAME] = {"skipped": w} + continue + ctx = Context(pin=pin, fetched=f, out=CORPUS / proc.NAME) + log(f"[harvest] {proc.NAME} <- {proc.SOURCE}") + results[proc.NAME] = proc.run(ctx) + warnings += ctx.warnings + report = {"mojoVersion": pin.version, "compiler": pin.compiler, + "sources": {k: {**v.lock, "license": v.source.license, "repo": v.source.repo, + "repoConfirmed": v.source.repo_confirmed, "note": v.source.note} + for k, v in fetched.items()}, + "processors": results, "warnings": warnings} + if not processors: + write_json(CORPUS / "report.json", report) + write_text(CORPUS / "SOURCES.md", sources_markdown(report)) + written = emit_ts.emit(pin) if (not processors or "tokens" in processors) else [] + for w in warnings: + log(f"[harvest] warning: {w}") + log(f"[harvest] done; generated {', '.join(rel(p) for p in written) or 'no TypeScript'}") + return report + finally: + if not keep_it: + shutil.rmtree(wd, ignore_errors=True) + + +def sources_markdown(r: dict[str, Any]) -> str: + L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "", + "Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `library/mojo/pin.json` and re-harvest.", + "Every item in the corpus records the same provenance per file in its manifest.", "", + "| source | repository | from | zip sha256 | license |", "|---|---|---|---|---|"] + for name, s in sorted(r["sources"].items()): + if s.get("kind") == "github": + frm = f"commit `{s['commit'][:12]}` ({s.get('pinnedRef')})" + elif s.get("standsInFor"): + frm = f"local zip `{s.get('file')}`, standing in for `{s['standsInFor']}`" + else: + frm = f"local zip `{s.get('file')}` (no ref pinned yet)" + repo = s["repo"] + ("" if s.get("repoConfirmed", True) else " (unconfirmed)") + L.append(f"| {name} | {repo} | {frm} | `{s.get('sha256', '')[:16]}` | {s['license']} |") + L += ["", "## What each processor found", ""] + for name, res in r["processors"].items(): + L.append(f"- **{name}**: " + ", ".join(f"{k} {v}" for k, v in res.items())) + if r["warnings"]: + L += ["", "## Warnings", ""] + [f"- {w}" for w in r["warnings"]] + notes = [f"- **{n}**: {s['note']}" for n, s in sorted(r["sources"].items()) if s.get("note")] + if notes: + L += ["", "## Notes from the pin", ""] + notes + return "\n".join(L) + "\n" diff --git a/scripts/mojo/mojocorpus/kde.py b/scripts/mojo/mojocorpus/kde.py new file mode 100644 index 0000000..6f3f3a7 --- /dev/null +++ b/scripts/mojo/mojocorpus/kde.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""A KDE syntax definition for Mojo, for Pandoc and Quarto listings. + +Generated from the same harvested words as the editor's highlighter +(corpus/tokens/tokens.json), so a printed listing and the Mojo pane color +the same words the same way: keywords, the ownership conventions, the +postfix transfer ^, words the pin removed, decorators, and prelude names. +Strings, including triple-quoted docstrings across lines, are their own +contexts. +""" +from __future__ import annotations + +from xml.sax.saxutils import escape + +TRIPLE_DOUBLE = """"" +TRIPLE_SINGLE = "'" * 3 + + +def _items(words) -> str: + return "\n".join(f" {escape(w)}" for w in words) + + +def mojo_xml(version: str, repo: str, commit: str, keywords: list[str], conventions: list[str], removed: list[str], + prelude: list[str], effects: list[str]) -> str: + kw = sorted((set(keywords) - {"var", "ref"}) | set(effects)) # var and ref are colored as conventions + conv = sorted(set(conventions) | {"var", "ref"}) + q = """ + return f""" + + + + + +{_items(kw)} + + +{_items(conv)} + + +{_items(removed)} + + +{_items(prelude)} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" diff --git a/scripts/mojo/mojocorpus/notices.py b/scripts/mojo/mojocorpus/notices.py new file mode 100644 index 0000000..badbd98 --- /dev/null +++ b/scripts/mojo/mojocorpus/notices.py @@ -0,0 +1,200 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""The library's third-party inventory: every file under library/mojo/corpus/, where it came from, and its license. + +The corpus is Modular's material, harvested at the commits in library/mojo/mojo.lock.json. This module reads the +pin, the lock, every processor's manifest and every harvested file, and writes library/THIRD_PARTY.md: for each +source the license its repository states and what was checked at which commit (`licenseCheck` in the pin, with +the upstream license text copied into library/mojo/licenses/), how many files carry a license header of their own, +and every file that carries none. `emit` writes it, so a harvest at a new pin rewrites it, and core's +tests/third-party.test.mjs fails while the file on disk differs from what this module would write, or while a +source's license was checked at a commit other than the one the lock holds. + + python3 scripts/mojo/corpus.py emit # writes it with everything else emit writes +""" +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path +from typing import Any + +from .common import CORPUS, MOJO_DIR, Pin, load_lock, write_text + +OUT = MOJO_DIR.parent / "THIRD_PARTY.md" +LICENSES = MOJO_DIR / "licenses" + +MODULAR_HEADER = re.compile(r"Licensed under the Apache License v2\.0 with LLVM Exceptions") +MODULAR_COPYRIGHT = re.compile(r"Copyright \(c\) \d{4}, Modular Inc\.") +SPDX = re.compile(r"SPDX-License-Identifier:\s*(\S+)") +OTHER = re.compile(r"(?i)\bcopyright\b|\blicen[cs]ed?\b") + +APACHE_LLVM = "Apache-2.0 WITH LLVM-exception" + +# A data file that is an upstream file itself rather than something a processor derived from several. +VERBATIM = {"syntax/mojo.syntax.json": ("syntaxes/mojo.syntax.json", "The TextMate grammar itself, parsed and written back as JSON")} + + +def header_of(text: str) -> str: + """The license a file states in its first 40 lines: Modular's header, an SPDX line, something else, or none.""" + head = "\n".join(text.splitlines()[:40]) + if MODULAR_HEADER.search(head): + return APACHE_LLVM if MODULAR_COPYRIGHT.search(head) else f"{APACHE_LLVM} (no copyright line)" + m = SPDX.search(head) + if m: + return m.group(1) + if OTHER.search(head): + return "other: " + next(l.strip() for l in head.splitlines() if OTHER.search(l))[:80] + return "none" + + +def _processors() -> dict[str, Any]: + from .processors import all_processors + return {p.NAME: p for p in all_processors()} + + +def _first_line(doc: str | None) -> str: + return (doc or "").strip().split("\n")[0].rstrip(".") + + +def inventory(pin: Pin, corpus: Path = CORPUS) -> dict[str, Any]: + """Every file under the corpus with its source, upstream path and header, plus the data files.""" + procs = _processors() + items: dict[str, dict[str, Any]] = {} + for manifest in sorted(corpus.glob("*/manifest.json")): + proc = manifest.parent.name + for it in json.loads(manifest.read_text(encoding="utf-8"))["items"]: + items[f"{proc}/{it['file']}"] = it["provenance"] + programs, data = [], [] + for f in sorted(p for p in corpus.rglob("*") if p.is_file()): + rel = f.relative_to(corpus).as_posix() + proc = rel.split("/")[0] if "/" in rel else None + if rel in items: + prov = items[rel] + programs.append({"file": rel, "processor": proc, "source": prov["source"], "path": prov["path"], + "header": header_of(f.read_text(encoding="utf-8", errors="replace"))}) + elif proc in procs: + p = procs[proc] + src_path, what = VERBATIM.get(rel, (", ".join(p.INPUTS), _first_line(p.__doc__))) + data.append({"file": rel, "processor": proc, "source": p.SOURCE, "from": src_path, "what": what}) + else: + data.append({"file": rel, "processor": None, "source": None, "from": "the harvest itself", + "what": "what came from where, and what each processor found"}) + return {"programs": programs, "data": data} + + +def license_checks(pin: Pin, lock: dict[str, Any]) -> list[dict[str, Any]]: + """For each source: what the pin says was checked, whether it was checked at the locked commit, and the copy.""" + out = [] + for name in sorted(pin.sources): + src = pin.sources[name] + raw = pin.raw["sources"][name] + check = raw.get("licenseCheck") or {} + commit = lock.get("sources", {}).get(name, {}).get("commit", "") + row = {"source": name, "repo": src.repo, "commit": commit, "stated": src.license, + "checkedAt": check.get("commit", ""), "on": check.get("on", ""), "file": check.get("file"), + "says": check.get("says", ""), "copy": check.get("copy"), "problems": []} + if not check: + row["problems"].append("no licenseCheck in the pin") + elif check.get("commit") != commit: + row["problems"].append(f"license checked at {check.get('commit', '')[:12]}, but the lock holds {commit[:12]}") + if check.get("copy"): + path = MOJO_DIR / check["copy"] + if not path.exists(): + row["problems"].append(f"{check['copy']} is missing") + elif hashlib.sha256(path.read_bytes()).hexdigest() != check.get("sha256"): + row["problems"].append(f"{check['copy']} does not have the sha256 the pin records") + out.append(row) + return out + + +def render(pin: Pin, corpus: Path = CORPUS) -> str: + lock = load_lock() + inv = inventory(pin, corpus) + checks = license_checks(pin, lock) + L = ["# Third-party material in pplus-library", "", + "Generated by `python3 scripts/mojo/corpus.py emit` in pplus-core (`pixi run mojo-emit`) from `mojo/pin.json`, " + "`mojo/mojo.lock.json` and the files under `mojo/corpus/`. Do not edit; pplus-core's `tests/third-party.test.mjs` " + "fails while this file is out of date.", "", + f"Everything in this repository is ours, under the Apache License 2.0 (`LICENSE.md`), except `mojo/corpus/`, " + f"which is Modular's material harvested for Mojo {pin.version} and listed below; `mojo/licenses/`, the upstream " + "license texts; and `mojo/golden/`, the answers Compiler Explorer (godbolt.org) returned for each program, which " + "are the Mojo compiler's output for our programs and for the corpus. A file under `mojo/corpus/` is a test " + "fixture: pplus-core's tests read it and the app does not ship it.", "", + "## Sources", "", + "| source | repository | commit | license the pin states | license file at that commit | copy here |", + "|---|---|---|---|---|---|"] + for c in checks: + found = f"`{c['file']}`, checked {c['on']}" if c["file"] else (f"none, checked {c['on']}" if c["on"] else "not checked") + if c["problems"]: + found += " **(" + "; ".join(c["problems"]) + ")**" + copy = f"`mojo/{c['copy']}`" if c["copy"] else "none" + L.append(f"| {c['source']} | [{c['repo']}](https://github.com/{c['repo']}) | `{c['commit'][:12]}` | {c['stated']} | {found} | {copy} |") + L += ["", "What each check found:", ""] + L += [f"- **{c['source']}**: {c['says']}" for c in checks if c["says"]] + + L += ["", "## Harvested programs, by license header", "", + "Each `.mojo` file the processors wrote, with the header it carries in its first 40 lines. " + f"\"{APACHE_LLVM}\" is Modular's header (\"Copyright (c) , Modular Inc. ... Licensed under the Apache " + "License v2.0 with LLVM Exceptions\").", "", + "| source | processor | files | with Modular's header | with another header | with no header |", + "|---|---|---|---|---|---|"] + groups: dict[tuple[str, str], list[dict[str, Any]]] = {} + for p in inv["programs"]: + groups.setdefault((p["source"], p["processor"]), []).append(p) + totals = [0, 0, 0, 0] + for (source, proc), ps in sorted(groups.items()): + n = len(ps) + modular = sum(1 for p in ps if p["header"] == APACHE_LLVM) + none = sum(1 for p in ps if p["header"] == "none") + other = n - modular - none + totals = [totals[0] + n, totals[1] + modular, totals[2] + other, totals[3] + none] + L.append(f"| {source} | {proc} | {n} | {modular} | {other} | {none} |") + L.append(f"| **all** | | **{totals[0]}** | **{totals[1]}** | **{totals[2]}** | **{totals[3]}** |") + + others = [p for p in inv["programs"] if p["header"] not in (APACHE_LLVM, "none")] + L += ["", "### Files with a header other than Modular's", ""] + L += [f"- `mojo/corpus/{p['file']}` ({p['source']}: `{p['path']}`): {p['header']}" for p in others] or ["None."] + + bare = [p for p in inv["programs"] if p["header"] == "none"] + L += ["", f"### Files with no license header ({len(bare)})", "", + "These carry no header because the processor took them out of a file that has none: a manual page's " + "program and its output (`docs/site/manual/*.mdx`), or a Mojo Quest starter (`src/data/issues.ts`). " + "They fall under their repository's license, where it has one (see Sources).", ""] + by_upstream: dict[str, list[dict[str, Any]]] = {} + for p in bare: + by_upstream.setdefault(f"{p['source']}: `{p['path'].split('#')[0]}`", []).append(p) + for upstream, ps in sorted(by_upstream.items()): + L.append(f"- {upstream}: " + ", ".join(f"`{p['file']}`" for p in ps)) + + L += ["", "## Data files", "", + "Files a processor derived from upstream files: lists of words, names, messages and test cases, in JSON. " + "None can carry a header; each comes from the source and inputs named here.", "", + "| file | source | made from | what it holds |", "|---|---|---|---|"] + for d in inv["data"]: + L.append(f"| `mojo/corpus/{d['file']}` | {d['source'] or 'ours'} | {d['from']} | {d['what']} |") + L += ["", "## Notices", "", + "- The files from modular/modular and modular/skills are Copyright (c) Modular Inc. and are distributed under " + "the Apache License v2.0 with LLVM Exceptions (`mojo/licenses/modular-LICENSE.txt`, " + "`mojo/licenses/skills-LICENSE.txt`); those from modular/vscode-mojo under the Apache License 2.0 " + "(`mojo/licenses/vscode-mojo-LICENSE.txt`). Nothing was changed, except that a manual snippet was cut out of " + "the page that holds it and a data file was derived as the table above says.", + "- The files from modular/mojo-quest are Modular's. Its solutions carry Modular's Apache-2.0 WITH " + "LLVM-exception header; the repository has no license file, and its starters and ticket text come from a " + "module with no header (see Sources).", + "- `mojo/corpus/syntax/mojo.syntax.json` is the TextMate grammar of modular/mojo-syntax, under the MIT license " + "whose notice is `mojo/licenses/mojo-syntax-LICENSE.txt`.", + "- \"Mojo\" and \"Modular\" are trademarks of Modular Inc. This project is not affiliated with or endorsed by " + "Modular.", ""] + return "\n".join(L) + + +def problems(pin: Pin) -> list[str]: + """What keeps the inventory from being trusted: a license checked at another commit, a copy that changed.""" + return [f"{c['source']}: {p}" for c in license_checks(pin, load_lock()) for p in c["problems"]] + + +def write(pin: Pin, out: Path = OUT) -> Path: + write_text(out, render(pin)) + return out diff --git a/scripts/mojo/mojocorpus/processors/__init__.py b/scripts/mojo/mojocorpus/processors/__init__.py new file mode 100644 index 0000000..66b6825 --- /dev/null +++ b/scripts/mojo/mojocorpus/processors/__init__.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Processors turn an extracted source tree into corpus files. + +Each processor module defines: + + NAME the corpus subdirectory it owns (library/mojo/corpus//) + SOURCE the pin source it reads + INPUTS paths under the source root it cannot work without + run(ctx) -> dict writes its files and returns a summary for the report + +If an INPUT is missing at the pinned ref (a tag that predates the compiler +being published, say), the processor is skipped with a warning and its +previous output is left as it was, so one missing directory upstream does not +wipe good data. + +A processor owns its directory outright: it is emptied before the processor +runs, so a file that upstream deleted disappears from the corpus too. Every +manifest records, per item, where it came from (repo, commit, path, license). +""" +from __future__ import annotations + +import shutil +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from ..common import CORPUS, Pin, provenance +from ..fetch import Fetched + + +@dataclass +class Context: + pin: Pin + fetched: Fetched + out: Path + warnings: list[str] = field(default_factory=list) + + @property + def root(self) -> Path: + return self.fetched.root + + def prov(self, path: str) -> dict[str, str]: + return provenance(self.fetched.source, self.fetched.lock, path) + + def warn(self, msg: str) -> None: + self.warnings.append(msg) + + +def fresh_dir(name: str, base: Path = CORPUS) -> Path: + d = base / name + if d.exists(): + shutil.rmtree(d) + d.mkdir(parents=True) + return d + + +def all_processors() -> list[Any]: + from . import quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes + return [quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes] diff --git a/scripts/mojo/mojocorpus/processors/diagnostics.py b/scripts/mojo/mojocorpus/processors/diagnostics.py new file mode 100644 index 0000000..e30e4ac --- /dev/null +++ b/scripts/mojo/mojocorpus/processors/diagnostics.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""The compiler's own diagnostic tests: what it rejects, where, and in what words. + +Files under test/ that run with `-verify-diagnostics` carry annotations in +MLIR's style: + + x = y # expected-error {{use of unknown declaration 'y'}} this line + # expected-error @below {{...}} the next code line + # expected-error @+2 {{...}} two lines down + # expected-note-re @above {{...{{[0-9]+}}...}} a regex message + +`@above` and `@below` name the nearest line above or below that is not itself +an annotation. Each file is tagged with the pass that reports its diagnostics, +read from its RUN lines: `lifetimes` (the borrow checker, -check-lifetimes), +`parser` (parse and type check), or `other`. + +These are the conformance tests for Programming+'s own checker (Stage 2) and +the source of the compiler's exact wording. Many use internal modules or the +isolated parser and cannot be reproduced on godbolt; `reproducible` says +which look runnable, and verify (opt-in, `--only diagnostics`) decides. + +Writes: + diagnostics/manifest.json one item per file, with its annotations + diagnostics/files/.mojo the test files, verbatim +""" +from __future__ import annotations + +import re + +from ..common import dialect_flags, write_json, write_text +from . import Context, fresh_dir + +NAME = "diagnostics" +SOURCE = "modular" +INPUTS = ["test"] + +ANN = re.compile(r"expected-(?Perror|warning|note|remark)(?P-re)?\s*(?P@\s*(?:above|below|[+-]\s*\d+))?\s*\{\{") +RUN = re.compile(r"^\s*#\s*RUN:\s*(.*)$") +PRIVATE = re.compile(r"(?m)^\s*(?:from|import)\s+std(?:\.\w+)*\._\w+|__mlir_(?:op|type|attr)\b|__extension\b") + + +def parse_annotations(text: str) -> list[dict]: + lines = text.split("\n") + is_ann = [bool(ANN.search(l)) and l.lstrip().startswith("#") for l in lines] + out = [] + for idx, line in enumerate(lines): + starts = [m for m in ANN.finditer(line)] + for k, m in enumerate(starts): + end_limit = starts[k + 1].start() if k + 1 < len(starts) else len(line) + close = line.rfind("}}", m.end(), end_limit) + if close < 0: + continue + message = line[m.end():close] + at = (m.group("at") or "").replace(" ", "") + n = idx + 1 # 1-based line of the annotation + if not at: + target = n + elif at == "@below": + j = idx + 1 + while j < len(lines) and (is_ann[j] or not lines[j].strip()): + j += 1 + target = j + 1 + elif at == "@above": + j = idx - 1 + while j >= 0 and (is_ann[j] or not lines[j].strip()): + j -= 1 + target = j + 1 + else: + target = n + int(at[1:]) + out.append({"kind": m.group("kind"), "regex": bool(m.group("re")), "line": target, + "annotationLine": n, "message": message}) + return out + + +def pass_of(runs: list[str]) -> str: + verify = [r for r in runs if "-verify-diagnostics" in r] or runs + joined = " ".join(verify) + if "-check-lifetimes" in joined: + return "lifetimes" + if "parse-mojo" in joined or "-import-mojo" in joined: + return "parser" + return "other" + + +def run(ctx: Context) -> dict: + out = fresh_dir(NAME) + items = [] + by_pass: dict[str, int] = {} + by_kind: dict[str, int] = {} + for f in sorted((ctx.root / "test").rglob("*.mojo")): + text = f.read_text(encoding="utf-8", errors="replace") + if "expected-" not in text: + continue + runs = [m.group(1).strip() for m in map(RUN.match, text.split("\n")) if m] + if runs and not any("-verify-diagnostics" in r for r in runs): + continue + anns = parse_annotations(text) + if not anns: + continue + r = f.relative_to(ctx.root / "test").as_posix() + write_text(out / "files" / r, text) + p = pass_of(runs) + by_pass[p] = by_pass.get(p, 0) + 1 + for a in anns: + by_kind[a["kind"]] = by_kind.get(a["kind"], 0) + 1 + private = bool(PRIVATE.search(text)) + items.append({ + "id": f"diagnostics/{r.removesuffix('.mojo')}", "file": f"files/{r}", "mode": "run", + "pass": p, "run": runs, "annotations": anns, + "reproducible": "unlikely" if private else "maybe", + "expect": {"kind": "compile-error", "messages": [a for a in anns if a["kind"] == "error"]}, + "dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"test/{r}"), + }) + write_json(out / "manifest.json", {"processor": NAME, "items": items}) + return {"files": len(items), "annotations": sum(len(i["annotations"]) for i in items), + "byPass": by_pass, "byKind": by_kind, "maybeReproducible": sum(1 for i in items if i["reproducible"] == "maybe")} diff --git a/scripts/mojo/mojocorpus/processors/editor.py b/scripts/mojo/mojocorpus/processors/editor.py new file mode 100644 index 0000000..c381c0d --- /dev/null +++ b/scripts/mojo/mojocorpus/processors/editor.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""How an editor should treat Mojo, from the official VS Code extension. + +language-configuration.json gives comments, brackets, auto-closing pairs, +and indentation rules; package.json's `contributes.languages` gives the file +extensions and names. Stage 1's editor reads these for Tab/Enter behavior, +comment toggling, and bracket pairs, instead of guessing. + +VS Code allows comments and trailing commas in these files; both are +tolerated here. + +Writes editor/editor.json. +""" +from __future__ import annotations + +import json +import re + +from ..common import PipelineError, write_json +from . import Context, fresh_dir + +NAME = "editor" +SOURCE = "vscode-mojo" +INPUTS = ["language-configuration.json", "package.json"] + + +def loads_jsonc(text: str): + out, i, n = [], 0, len(text) + while i < n: + c = text[i] + if c == '"': + j = i + 1 + while j < n and text[j] != '"': + j += 2 if text[j] == "\\" else 1 + out.append(text[i:j + 1]); i = j + 1 + elif text.startswith("//", i): + i = text.find("\n", i); i = n if i < 0 else i + elif text.startswith("/*", i): + i = text.find("*/", i) + 2 + else: + out.append(c); i += 1 + cleaned = re.sub(r",(\s*[}\]])", r"\1", "".join(out)) + return json.loads(cleaned) + + +def run(ctx: Context) -> dict: + out = fresh_dir(NAME) + cfg_path = ctx.root / "language-configuration.json" + pkg_path = ctx.root / "package.json" + try: + cfg = loads_jsonc(cfg_path.read_text(encoding="utf-8")) + pkg = loads_jsonc(pkg_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + raise PipelineError(f"editor: cannot read the VS Code language files: {e}") from e + langs = [l for l in pkg.get("contributes", {}).get("languages", []) if l.get("id") == "mojo"] + doc = { + "provenance": {"languageConfiguration": ctx.prov("language-configuration.json"), + "languages": ctx.prov("package.json#contributes.languages")}, + "extensionVersion": pkg.get("version"), + "languages": langs, + "languageConfiguration": cfg, + } + write_json(out / "editor.json", doc) + return {"extensionVersion": pkg.get("version"), "extensions": sorted({e for l in langs for e in l.get("extensions", [])}), + "hasIndentationRules": "indentationRules" in cfg or "onEnterRules" in cfg} diff --git a/scripts/mojo/mojocorpus/processors/manual.py b/scripts/mojo/mojocorpus/processors/manual.py new file mode 100644 index 0000000..8219fd6 --- /dev/null +++ b/scripts/mojo/mojocorpus/processors/manual.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""The Mojo Manual: its tested code samples, and its program/output pairs. + +Two kinds of item: + + code/ docs/site/code/**/*.mojo. Each file carries the tests for one + manual page (most run a std.testing TestSuite), so the expectation + is "compiles, runs, exits 0". A file without `def main` is a module + for another sample (a Python extension, say) and is recorded but + not run. + snippets/ a ```mojo block that is a whole program (it defines `main`) + followed directly by an ```output block on a manual or reference + page. The expectation is that exact output. + +The manual in a snapshot can be newer than the pin. Items whose code matches +a `newer` dialect marker are kept but expected to be skipped, and verify is the +judge for the rest. +""" +from __future__ import annotations + +import re + +from ..common import dialect_flags, write_json, write_text +from . import Context, fresh_dir + +NAME = "manual" +SOURCE = "modular" +INPUTS = ["docs/site/code", "docs/site/manual"] + +PAIR = re.compile( + r"(?ms)^(?P[ \t]*)```mojo[^\n]*\n(?P.*?)^(?P=ind)```[ \t]*\n\s*" + r"^(?P=ind)```output[^\n]*\n(?P.*?)^(?P=ind)```" +) + + +def _dedent(text: str, indent: str) -> str: + if not indent: + return text + return "".join(l[len(indent):] if l.startswith(indent) else l for l in text.splitlines(keepends=True)) + + +def _slug(path: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", path.lower().removesuffix(".mdx")).strip("-") + + +def run(ctx: Context) -> dict: + out = fresh_dir(NAME) + site = ctx.root / "docs" / "site" + items = [] + + for f in sorted((site / "code").rglob("*.mojo")): + r = f.relative_to(site / "code").as_posix() + text = f.read_text(encoding="utf-8") + write_text(out / "code" / r, text) + runnable = bool(re.search(r"(?m)^def main\b", text)) + items.append({ + "id": f"manual/code/{r.removesuffix('.mojo')}", "file": f"code/{r}", "mode": "run", + "expect": {"kind": "pass", "check": {}} if runnable else {"kind": "skip", "why": "no main: a module used by another sample"}, + "dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"docs/site/code/{r}"), + }) + + pages = sorted(list((site / "manual").rglob("*.mdx")) + list((site / "reference").rglob("*.mdx"))) + snippets = 0 + for page in pages: + r = page.relative_to(site).as_posix() + s = page.read_text(encoding="utf-8") + for n, m in enumerate(PAIR.finditer(s), 1): + code = _dedent(m.group("code"), m.group("ind")) + if not re.search(r"(?m)^def main\b", code): + continue + expected = _dedent(m.group("out"), m.group("ind")) + line = s.count("\n", 0, m.start()) + 1 + name = f"{_slug(r)}-{n}.mojo" + write_text(out / "snippets" / name, code) + items.append({ + "id": f"manual/snippet/{_slug(r)}-{n}", "file": f"snippets/{name}", "mode": "run", + "expect": {"kind": "pass", "check": {"stdout": expected}}, + "dialect": dialect_flags(code, ctx.pin), "provenance": ctx.prov(f"docs/site/{r}#L{line}"), + }) + snippets += 1 + + for it in items: + if it["dialect"].get("newer") and it["expect"]["kind"] != "skip": + it["expect"] = {"kind": "skip", "why": "uses syntax newer than the pin: " + ", ".join(it["dialect"]["newer"]), "was": it["expect"]} + write_json(out / "manifest.json", {"processor": NAME, "items": items}) + return {"codeSamples": sum(1 for i in items if i["id"].startswith("manual/code/")), "snippets": snippets, + "skippedNewer": sum(1 for i in items if i["expect"]["kind"] == "skip" and "newer" in i["expect"].get("why", ""))} diff --git a/scripts/mojo/mojocorpus/processors/messages.py b/scripts/mojo/mojocorpus/processors/messages.py new file mode 100644 index 0000000..bea9249 --- /dev/null +++ b/scripts/mojo/mojocorpus/processors/messages.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""The compiler's diagnostic wording, and the fixes it offers, from its source. + +Programming+'s checker (Stage 2) should say what the real compiler says, in +the same words, and offer the same fixes. This scans the parser +(lib/MojoParser) and the borrow checker (lib/LowerLIT) for diagnostic calls +(emitError, emitWarning, emitNote, emitRemark, emitTokenError, attachNote, +emitErrorIfNotDiagnosed) and records, per statement: + + message the string literals in order; a streamed value (`<< name`) is {} + fixits FixIt::replaceToken / insert / remove ... and their literal text + severity error | warning | note | remark + pass parser | lifetimes + +This is a scan of C++ text, not a C++ parser: a message assembled in a helper +function shows up as {} pieces, and that is reported, not hidden. + +Writes messages/messages.json. +""" +from __future__ import annotations + +import re + +from ..common import write_json +from . import Context, fresh_dir + +NAME = "messages" +SOURCE = "modular" +INPUTS = ["lib/MojoParser"] + +CALL = re.compile(r"\b(?PemitError|emitWarning|emitNote|emitRemark|emitTokenError|emitErrorIfNotDiagnosed|attachNote|emitOpError)\s*\(") +FIXIT = re.compile(r"FixIt::(?P\w+)\s*\(") +DIRS = {"MojoParser": "parser", "LowerLIT": "lifetimes"} + + +def statement_end(text: str, i: int) -> int: + """Index just past the `;` that ends the statement containing position i.""" + depth = 0 + n = len(text) + while i < n: + c = text[i] + if c == '"': + i += 1 + while i < n and text[i] != '"': + i += 2 if text[i] == "\\" else 1 + elif c == "'": + i += 1 + while i < n and text[i] != "'": + i += 2 if text[i] == "\\" else 1 + elif text.startswith("//", i): + i = text.find("\n", i) + if i < 0: + return n + continue + elif c in "([{": + depth += 1 + elif c in ")]}": + depth -= 1 + if depth < 0: + return i + elif c == ";" and depth == 0: + return i + 1 + i += 1 + return n + + +LIT = re.compile(r'"((?:\\.|[^"\\])*)"') + + +def unescape(s: str) -> str: + return s.encode("utf-8").decode("unicode_escape").encode("latin-1").decode("utf-8", errors="replace") if "\\" in s else s + + +def literals(fragment: str) -> str: + """Adjacent C++ literals concatenate.""" + return "".join(unescape(m.group(1)) for m in LIT.finditer(fragment)) + + +def split_stream(stmt: str) -> list[str]: + """Split a statement on top-level `<<`.""" + parts, depth, cur, i = [], 0, [], 0 + while i < len(stmt): + c = stmt[i] + if c == '"': + j = i + 1 + while j < len(stmt) and stmt[j] != '"': + j += 2 if stmt[j] == "\\" else 1 + cur.append(stmt[i:j + 1]); i = j + 1; continue + if c in "([{": + depth += 1 + elif c in ")]}": + depth -= 1 + if depth == 0 and stmt.startswith("<<", i): + parts.append("".join(cur)); cur = []; i += 2; continue + cur.append(c); i += 1 + parts.append("".join(cur)) + return parts + + +def message_of(stmt: str, call_end: int) -> tuple[str, list[dict]]: + head_close = statement_end(stmt, call_end) # the `)` that closes the call's arguments + head_args = stmt[call_end:head_close] + msg = literals(head_args) + fixits = [] + for piece in split_stream(stmt[head_close + 1:]): + p = piece.strip() + if m := FIXIT.search(p): + fixits.append({"kind": m.group("kind"), "text": literals(p[m.end():]) or None}) + elif p.startswith('"'): + msg += literals(p) + elif p: + msg += "{}" + return msg, fixits + + +def scan(text: str) -> list[dict]: + out = [] + for m in CALL.finditer(text): + start = m.start() + close = statement_end(text, m.end()) # the `)` closing the call's arguments + end = statement_end(text, close + 1) # then on to the `;` that ends the statement + stmt = text[start:end] + msg, fixits = message_of(stmt, m.end() - start) + if not msg.strip("{} "): + continue + fn = m.group("fn") + sev = "warning" if "Warning" in fn else "note" if "Note" in fn else "remark" if "Remark" in fn else "error" + out.append({"line": text.count("\n", 0, start) + 1, "call": fn, "severity": sev, + "message": msg, "fixits": fixits}) + return out + + +def run(ctx: Context) -> dict: + out = fresh_dir(NAME) + entries, seen = [], set() + for d, p in DIRS.items(): + for f in sorted((ctx.root / "lib" / d).rglob("*.cpp")): + text = f.read_text(encoding="utf-8", errors="replace") + r = f.relative_to(ctx.root).as_posix() + for e in scan(text): + key = (r, e["message"]) + if key in seen: + continue + seen.add(key) + entries.append({**e, "file": r, "pass": p}) + write_json(out / "messages.json", {"processor": NAME, "provenance": ctx.prov("lib/MojoParser, lib/LowerLIT"), + "messages": entries}) + return {"messages": len(entries), "withFixits": sum(1 for e in entries if e["fixits"]), + "byPass": {p: sum(1 for e in entries if e["pass"] == p) for p in DIRS.values()}} diff --git a/scripts/mojo/mojocorpus/processors/quest.py b/scripts/mojo/mojocorpus/processors/quest.py new file mode 100644 index 0000000..eb36b5b --- /dev/null +++ b/scripts/mojo/mojocorpus/processors/quest.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Mojo Quest: 85 tickets, each a completed solution and a broken starter. + +Mojo Quest checks every solution and starter against the real compiler in its +own CI, so this is the most trustworthy behavioral corpus there is: each +solution must compile, run, exit 0 and print exactly its expected output (or, +for `source` tickets, contain an idiom), and each starter must fail that check. + +The ticket data lives in a TypeScript module (src/data/issues.ts). Node reads +it with type stripping (Node 22.6 or newer); nothing is re-implemented here. + +Writes: + quest/manifest.json one item per solution and per starter + quest/solutions/.mojo verbatim from exercises/, license header included + quest/starters/.mojo the starter text from issues.ts +""" +from __future__ import annotations + +import json +import shutil +import subprocess + +from ..common import PipelineError, dialect_flags, write_json, write_text +from . import Context, fresh_dir + +NAME = "quest" +SOURCE = "mojo-quest" +INPUTS = ["src/data/issues.ts", "exercises"] + +_DUMP = """ +const m = await import(process.argv[1]); +process.stdout.write(JSON.stringify(m.issues)); +""" + + +def load_issues(root) -> list[dict]: + node = shutil.which("node") + if not node: + raise PipelineError("the quest processor needs Node 22.6+ on PATH (Mojo Quest keeps its tickets in TypeScript)") + issues_ts = root / "src" / "data" / "issues.ts" + proc = subprocess.run( + [node, "--experimental-strip-types", "--no-warnings", "--input-type=module", "-e", _DUMP, issues_ts.as_uri()], + capture_output=True, text=True, timeout=120, + ) + if proc.returncode != 0: + raise PipelineError(f"node could not load {issues_ts.name}: {proc.stderr.strip()[:400]}") + data = json.loads(proc.stdout) + if not isinstance(data, list) or not data: + raise PipelineError("issues.ts exported no issues") + return data + + +def check_of(validation: dict) -> dict: + if validation.get("kind") == "run": + return {"stdout": validation["expectedStdout"]} + if validation.get("kind") == "source": + return {"patterns": list(validation["patterns"]), "flags": validation.get("flags", ""), "message": validation.get("message", "")} + raise PipelineError(f"unknown Mojo Quest validation kind {validation.get('kind')!r}") + + +def run(ctx: Context) -> dict: + out = fresh_dir(NAME) + issues = load_issues(ctx.root) + items = [] + missing = [] + for issue in issues: + iid = issue["id"] + stem = iid.replace("-", "_") + sol = ctx.root / "exercises" / f"{stem}.mojo" + check = check_of(issue["validation"]) + meta = {k: issue.get(k) for k in ("concept", "title", "topic", "docUrl", "file", "hint")} + if sol.exists(): + text = sol.read_text(encoding="utf-8") + write_text(out / "solutions" / f"{stem}.mojo", text) + items.append({ + "id": f"quest/{iid}/solution", "file": f"solutions/{stem}.mojo", "mode": "run", + "expect": {"kind": "pass", "check": check}, "dialect": dialect_flags(text, ctx.pin), + "meta": meta, "provenance": ctx.prov(f"exercises/{stem}.mojo"), + }) + else: + missing.append(iid) + starter = issue.get("starter") + if starter: + write_text(out / "starters" / f"{stem}.mojo", starter) + items.append({ + "id": f"quest/{iid}/starter", "file": f"starters/{stem}.mojo", "mode": "run", + "expect": {"kind": "fail", "check": check}, "dialect": dialect_flags(starter, ctx.pin), + "meta": meta, "provenance": ctx.prov(f"src/data/issues.ts#{iid}"), + }) + if missing: + ctx.warn(f"quest: {len(missing)} ticket(s) have no solution file: {', '.join(missing[:8])}") + write_json(out / "manifest.json", {"processor": NAME, "items": items}) + kinds = {} + for i in items: + kinds[i["expect"]["kind"]] = kinds.get(i["expect"]["kind"], 0) + 1 + return {"tickets": len(issues), "items": len(items), "byExpectation": kinds, + "runChecks": sum(1 for i in issues if i["validation"]["kind"] == "run")} diff --git a/scripts/mojo/mojocorpus/processors/release.py b/scripts/mojo/mojocorpus/processors/release.py new file mode 100644 index 0000000..8cb073c --- /dev/null +++ b/scripts/mojo/mojocorpus/processors/release.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Release notes for the pinned version, and whether the snapshot matches the pin. + +Two jobs: + +1. From docs/site/releases/v.md, record each section's bullets (the + first sentence of each, with its anchor). "Removed" and "Language changes" + are where Stage 2's quick fixes come from. +2. Say how the snapshot relates to the pin. The newest release file and the + nightly changelog tell us: a snapshot whose newest release is the pin and + whose nightly changelog is empty IS the pin; one with nightly entries is + newer, and those entries are exactly what the pin does not have. + +A release can be tagged before its notes leave the nightly changelog: at +mojo/v1.2.0 there is no v1.2.0.md, the newest release file is v1.1.0.md, and +the changelog holds exactly what 1.2.0 added. When the pinned ref is the +pin's own release tag and the newest release file is older than the pin, the +changelog is therefore read as the pin's notes (`notesFrom: +"nightly-changelog"`), and nothing is beyond the pin. + +Writes release/release.json. A newer snapshot is a warning in the report, not +an error: the dialect markers and verify keep its newer material out. +""" +from __future__ import annotations + +import re + +from ..common import write_json +from . import Context, fresh_dir + +NAME = "release" +SOURCE = "modular" +INPUTS = ["docs/site/releases"] + +VERSION = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") + + +def sections(md: str) -> list[dict]: + out: list[dict] = [] + cur: dict | None = None + bullet: list[str] | None = None + + def close_bullet(): + nonlocal bullet + if cur is not None and bullet: + text = " ".join(x.strip() for x in bullet) + first = re.split(r"(?<=[.!?])\s+(?=[A-Z`])", text, maxsplit=1)[0] + cur["bullets"].append(first[:300]) + bullet = None + + for line in md.split("\n"): + h = re.match(r"^(#{2,3})\s+(.*?)\s*(?:\{#([\w-]+)\})?\s*$", line) + if h: + close_bullet() + cur = {"level": len(h.group(1)), "title": h.group(2), "anchor": h.group(3) or "", "bullets": []} + out.append(cur) + elif line.startswith("- "): + close_bullet() + bullet = [line[2:]] + elif bullet is not None and line.startswith(" ") and line.strip() and not line.strip().startswith("```"): + bullet.append(line) + elif not line.strip(): + close_bullet() + close_bullet() + return out + + +def relate(pin: str, versions: list[tuple[int, int, int]], has_notes: bool, nightly_entries: int, ref: str) -> tuple[str, str]: + """How the snapshot relates to the pin, and where the pin's notes are. + + Returns (relation, notes_from): relation is `same`, `newer`, `older` or + `unknown`; notes_from is `release-file`, `nightly-changelog` or `none`. + """ + if not versions: + return "unknown", "release-file" if has_notes else "none" + pinned = tuple(map(int, pin.split("."))) + if pinned > versions[-1]: + # tagged before the notes were moved out of the changelog: the changelog is the release + if ref == f"mojo/v{pin}" and nightly_entries and not has_notes: + return "same", "nightly-changelog" + return "older", "release-file" if has_notes else "none" + if pinned != versions[-1]: + return "newer", "release-file" if has_notes else "none" + return ("newer" if nightly_entries else "same"), "release-file" if has_notes else "none" + + +def run(ctx: Context) -> dict: + out = fresh_dir(NAME) + rel_dir = ctx.root / "docs" / "site" / "releases" + versions = sorted(tuple(map(int, m.groups())) for p in rel_dir.glob("v*.md") if (m := VERSION.match(p.stem))) + newest = ".".join(map(str, versions[-1])) if versions else None + pin = ctx.pin.version + notes = rel_dir / f"v{pin}.md" + nightly = ctx.root / "docs" / "site" / "nightly-changelog.md" + nightly_secs = sections(nightly.read_text(encoding="utf-8")) if nightly.exists() else [] + nightly_bullets = [{"section": s["title"], "bullet": b} for s in nightly_secs for b in s["bullets"]] + relation, notes_from = relate(pin, versions, notes.exists(), len(nightly_bullets), ctx.fetched.source.ref) + + if relation == "newer": + ctx.warn(f"release: the modular snapshot is newer than Mojo {pin} (newest release notes v{newest}, " + f"{len(nightly_bullets)} nightly changelog entries). Dialect markers and verify keep newer material out; " + f"pin a release tag to remove this warning.") + elif relation == "older": + ctx.warn(f"release: the modular snapshot (newest release v{newest}) is older than the pin {pin}") + if notes_from == "none": + ctx.warn(f"release: no release notes for v{pin} in the snapshot") + + if notes_from == "nightly-changelog": + notes_path, note_secs, beyond = "docs/site/nightly-changelog.md", nightly_secs, [] + else: + notes_path = f"docs/site/releases/v{pin}.md" + note_secs = sections(notes.read_text(encoding="utf-8")) if notes.exists() else [] + beyond = nightly_bullets + doc = { + "pin": pin, "newestReleaseInSnapshot": newest, "snapshot": relation, "notesFrom": notes_from, + "notes": {"provenance": ctx.prov(notes_path), "sections": note_secs}, + "nightlyBeyondPin": {"provenance": ctx.prov("docs/site/nightly-changelog.md"), "entries": beyond}, + } + write_json(out / "release.json", doc) + return {"snapshot": relation, "newestRelease": newest, "notesFrom": notes_from, + "sections": len(doc["notes"]["sections"]), "nightlyEntries": len(beyond)} diff --git a/scripts/mojo/mojocorpus/processors/skill.py b/scripts/mojo/mojocorpus/processors/skill.py new file mode 100644 index 0000000..8689341 --- /dev/null +++ b/scripts/mojo/mojocorpus/processors/skill.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Modular's mojo-syntax skill: its table of removed syntax and replacements. + +The skill is Modular's correction layer for code written from old tutorials, +which is exactly what students paste. Its "Removed syntax" table becomes the +candidate list for Stage 2's quick fixes; each one is checked against the +compiler's own fix-its (messages/) and against the pinned release notes +(release/) before Programming+ offers it. + +The skill targets "latest" Mojo, which can be ahead of the pin; the dialect +markers in library/mojo/pin.json and verify are what decide, not this table. + +Writes skill/removed-syntax.json. +""" +from __future__ import annotations + +import re + +from ..common import PipelineError, write_json +from . import Context, fresh_dir + +NAME = "skill" +SOURCE = "skills" +INPUTS = ["mojo-syntax/SKILL.md"] + + +def table_after(md: str, heading: str) -> list[list[str]]: + m = re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", md) + if not m: + raise PipelineError(f"skill: no '## {heading}' section") + rows = [] + for line in md[m.end():].split("\n"): + if line.startswith("## "): + break + if not line.startswith("|"): + if rows: + break + continue + cells = [c.strip() for c in line.strip().strip("|").split("|")] + if all(re.fullmatch(r":?-+:?", c) for c in cells): + continue + rows.append(cells) + return rows + + +def code_spans(cell: str) -> list[str]: + return re.findall(r"`([^`]+)`", cell) + + +def run(ctx: Context) -> dict: + out = fresh_dir(NAME) + md = (ctx.root / "mojo-syntax" / "SKILL.md").read_text(encoding="utf-8") + rows = table_after(md, "Removed syntax — DO NOT generate these") + header, body = rows[0], rows[1:] + entries = [{"removed": r[0], "replacement": r[1], "removedCode": code_spans(r[0]), "replacementCode": code_spans(r[1])} + for r in body if len(r) >= 2] + write_json(out / "removed-syntax.json", {"processor": NAME, "provenance": ctx.prov("mojo-syntax/SKILL.md"), + "columns": header, "entries": entries}) + return {"entries": len(entries)} diff --git a/scripts/mojo/mojocorpus/processors/stdindex.py b/scripts/mojo/mojocorpus/processors/stdindex.py new file mode 100644 index 0000000..22ab0c5 --- /dev/null +++ b/scripts/mojo/mojocorpus/processors/stdindex.py @@ -0,0 +1,151 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Which standard-library package exports each public name. + +The compiler's "use of unknown declaration 'sqrt'; did you mean to import it +from 'std.math'?" needs to know where a name lives. A package says what it +exports in its __init__.mojo (`from .math import (sqrt, ...)`, and its own +top-level definitions); a module file's public top-level definitions are +exported by that module. A name a package exports is attributed to the +package; otherwise to its module. A name exported from more than one place is +ambiguous and gets no hint, rather than a guess. + +The prelude package is not an import hint (its names are visible everywhere), but +its functions are read for whether they raise: `input` and `atol` raise, so calling +them from a function that does not is the compiler's "cannot call function that +may raise". A prelude function counts as raising only if every overload says +`raises` in its signature. + +Writes stdindex/stdindex.json: {name: module}, the ambiguous names, and the raising +prelude functions. +""" +from __future__ import annotations + +import re + +from ..common import write_json +from . import Context, fresh_dir + +NAME = "stdindex" +SOURCE = "modular" +INPUTS = ["stdlib/std"] + +IMPORT = re.compile(r"(?ms)^from\s+([\w.]+)\s+import\s+(\((?P.*?)\)|(?P[^\n]+))") +TOPLEVEL = re.compile(r"(?m)^(?:def|struct|trait)\s+([A-Za-z]\w*)|^comptime\s+([A-Za-z]\w*)\s*(?:[:=\[])") + + +def _names(body: str) -> list[str]: + body = re.sub(r"#[^\n]*", "", body) + out = [] + for part in body.split(","): + part = part.strip() + if part: + name = part.split(" as ")[-1].strip() + if re.fullmatch(r"[A-Za-z]\w*", name): + out.append(name) + return out + + +def defined(text: str) -> set[str]: + return {m.group(1) or m.group(2) for m in TOPLEVEL.finditer(text)} - {None} + + +def run(ctx: Context) -> dict: + out = fresh_dir(NAME) + std = ctx.root / "stdlib" / "std" + places: dict[str, set[str]] = {} + + def add(name: str, module: str) -> None: + if not name.startswith("_"): + places.setdefault(name, set()).add(module) + + package_exports: dict[str, set[str]] = {} + for init in sorted(std.rglob("__init__.mojo")): + rel = init.parent.relative_to(std.parent) + if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",): + continue + module = ".".join(rel.parts) + text = init.read_text(encoding="utf-8") + names = defined(text) + for m in IMPORT.finditer(text): + names.update(_names(m.group("group") or m.group("line") or "")) + package_exports[module] = names + for n in names: + add(n, module) + exported_anywhere = set().union(*package_exports.values()) if package_exports else set() + for f in sorted(std.rglob("*.mojo")): + if f.name == "__init__.mojo": + continue + rel = f.relative_to(std.parent).with_suffix("") + if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",): + continue + for n in defined(f.read_text(encoding="utf-8")): + if n not in exported_anywhere: + add(n, ".".join(rel.parts)) + unique = {n: next(iter(ms)) for n, ms in sorted(places.items()) if len(ms) == 1} + raising = raising_prelude(std) + ambiguous = sorted(n for n, ms in places.items() if len(ms) > 1) + write_json(out / "stdindex.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "exports": unique, "ambiguous": ambiguous, + "raisingPrelude": raising}) + return {"names": len(unique), "ambiguous": len(ambiguous), "packages": len(package_exports), "raisingPrelude": len(raising)} + + +def signatures(text: str, name: str) -> list[str]: + """Every top-level `def name...:` signature, up to the colon that opens its body.""" + out = [] + for m in re.finditer(rf"(?m)^def\s+{re.escape(name)}\b", text): + depth, j = 0, m.end() + while j < len(text): + c = text[j] + if c in "([{": + depth += 1 + elif c in ")]}": + depth -= 1 + elif c == ":" and depth == 0: + break + j += 1 + out.append(text[m.start():j]) + return out + + +def module_file(std, mod: str): + base = std.parent.joinpath(*mod.split(".")) + if base.with_suffix(".mojo").exists(): + return base.with_suffix(".mojo") + return base / "__init__.mojo" if (base / "__init__.mojo").exists() else None + + +def signatures_of(std, mod: str, name: str, depth: int = 0) -> list[str]: + """`name`'s signatures where it is defined, following re-exports (`from .string import atol`).""" + src = module_file(std, mod) + if src is None or depth > 4: + return [] + text = src.read_text(encoding="utf-8") + sigs = signatures(text, name) + if sigs: + return sigs + package = mod if src.name == "__init__.mojo" else mod.rsplit(".", 1)[0] + for m in IMPORT.finditer(text): + if name in _names(m.group("group") or m.group("line") or ""): + target = m.group(1) + if target.startswith("."): + target = package + "." + target.lstrip(".") if target.strip(".") else package + return signatures_of(std, target, name, depth + 1) + return [] + + +def raising_prelude(std) -> list[str]: + init = std / "prelude" / "__init__.mojo" + if not init.exists(): + return [] + text = init.read_text(encoding="utf-8") + found = [] + for m in IMPORT.finditer(text): + mod = m.group(1) + if not mod.startswith("std."): + continue + for name in _names(m.group("group") or m.group("line") or ""): + sigs = signatures_of(std, mod, name) + if sigs and all(re.search(r"\)\s*(?:raises\b|[^\n]*\braises\b)", sig.split("(", 1)[1] if "(" in sig else sig) for sig in sigs): + found.append(name) + return sorted(set(found)) diff --git a/scripts/mojo/mojocorpus/processors/stdtypes.py b/scripts/mojo/mojocorpus/processors/stdtypes.py new file mode 100644 index 0000000..823f16e --- /dev/null +++ b/scripts/mojo/mojocorpus/processors/stdtypes.py @@ -0,0 +1,188 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""The standard library's structs and traits, as far as a checker needs them. + +For each top-level `struct` and `trait` in stdlib/std: + + bases the traits it names in its declaration (`struct List[...](Copyable, Sized, ...)`) + methods the methods it defines (for a trait: only those with a body of their own, + the defaults a conforming struct gets without writing them; a bare + requirement, `def f(self): ...`, is not one) + +From these, Programming+ knows which types are implicitly copyable (a struct +whose traits reach ImplicitlyCopyable), and which methods a student's struct +has because of a trait it conforms to (Copyable gives `copy`, Equatable gives +`__ne__`), so "has no attribute" is never said of a method a trait provides. + +Writes stdtypes/stdtypes.json. +""" +from __future__ import annotations + +import re + +from ..common import write_json +from . import Context, fresh_dir + +NAME = "stdtypes" +SOURCE = "modular" +INPUTS = ["stdlib/std"] + +HEAD = re.compile(r"^(struct|trait)\s+([A-Za-z]\w*)") + + +def _bases(header: str) -> list[str]: + """The names in the declaration's parenthesised list, skipping parameter brackets and where-clauses.""" + depth, start, i = 0, None, 0 + # skip a leading [parameters] + m = re.match(r"(?:struct|trait)\s+\w+\s*", header) + i = m.end() if m else 0 + if i < len(header) and header[i] == "[": + d = 0 + while i < len(header): + if header[i] == "[": + d += 1 + elif header[i] == "]": + d -= 1 + if d == 0: + i += 1 + break + i += 1 + while i < len(header) and header[i] in " \t\n": + i += 1 + if i >= len(header) or header[i] != "(": + return [] + start = i + 1 + for j in range(i, len(header)): + c = header[j] + if c in "([": + depth += 1 + elif c in ")]": + depth -= 1 + if depth == 0: + inner = header[start:j] + break + else: + return [] + out, cur, d = [], "", 0 + for c in inner + ",": + if c in "([": + d += 1 + elif c in ")]": + d -= 1 + if c == "," and d == 0: + name = re.split(r"\s+where\s+|\[", cur.strip())[0].strip() + if re.fullmatch(r"[A-Za-z_][\w.]*", name): + out.append(name.split(".")[-1]) + cur = "" + else: + cur += c + return out + + +def declarations(text: str) -> list[dict]: + lines = text.split("\n") + decls = [] + i = 0 + while i < len(lines): + m = HEAD.match(lines[i]) + if not m: + i += 1 + continue + kind, name = m.group(1), m.group(2) + # the header runs to the colon that opens the body + header, j = lines[i], i + while not re.search(r":\s*(#.*)?$", header) and j + 1 < len(lines): + j += 1 + header += "\n" + lines[j] + body_start = j + 1 + k = body_start + while k < len(lines) and (not lines[k].strip() or lines[k].startswith((" ", "\t"))): + k += 1 + body = lines[body_start:k] + # lines inside a triple-quoted string (a docstring's code examples) are not definitions + in_string = [] + q = None + for line in body: + in_string.append(q is not None) + for m in re.finditer(r'"""|\'\'\'', line): + if q is None: + q = m.group(0) + elif m.group(0) == q: + q = None + methods = set() + for b, line in enumerate(body): + if in_string[b]: + continue + dm = re.match(r"^ {4}def\s+(\w+)", line) + if not dm: + continue + if kind == "struct": + methods.add(dm.group(1)) + continue + # a trait method is a default only if it has a body other than `...` (and a docstring) + sig = line + e = b + while not re.search(r":\s*(\.\.\.)?\s*(#.*)?$", sig) and e + 1 < len(body): + e += 1 + sig += body[e] + if re.search(r":\s*\.\.\.\s*(#.*)?$", sig): + continue + rest = [] + for line2 in body[e + 1:]: + if line2.strip() and not line2.startswith(" " * 8): + break + if line2.strip(): + rest.append(line2.strip()) + code = _strip_docstring(rest) + if code and code != ["..."]: + methods.add(dm.group(1)) + decls.append({"kind": kind, "name": name, "bases": _bases(header), "methods": sorted(methods)}) + i = k + return decls + + +def _strip_docstring(stmts: list[str]) -> list[str]: + if stmts and stmts[0].startswith(('"""', "'''")): + q = stmts[0][:3] + if stmts[0].count(q) >= 2 and len(stmts[0]) > 3: + return stmts[1:] + for n, s in enumerate(stmts[1:], 1): + if q in s: + return stmts[n + 1:] + return [] + return stmts + + +def run(ctx: Context) -> dict: + out = fresh_dir(NAME) + std = ctx.root / "stdlib" / "std" + structs: dict[str, dict] = {} + traits: dict[str, dict] = {} + dupes = set() + for f in sorted(std.rglob("*.mojo")): + rel = f.relative_to(std.parent) + if any(p.startswith("_") for p in rel.parts[:-1]): + continue + for d in declarations(f.read_text(encoding="utf-8")): + table = structs if d["kind"] == "struct" else traits + if d["name"] in table: + dupes.add(d["name"]) + table[d["name"]] = {"bases": d["bases"], "methods": d["methods"], "module": ".".join(rel.with_suffix("").parts)} + for n in dupes: + structs.pop(n, None) + traits.pop(n, None) + + def reaches(name: str, goal: str, seen=None) -> bool: + seen = seen or set() + if name == goal: + return True + if name in seen: + return False + seen.add(name) + t = traits.get(name) + return bool(t) and any(reaches(b, goal, seen) for b in t["bases"]) + + implicitly = sorted(n for n, s in structs.items() if any(reaches(b, "ImplicitlyCopyable") for b in s["bases"])) + write_json(out / "stdtypes.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "structs": structs, "traits": traits, + "implicitlyCopyable": implicitly, "ambiguous": sorted(dupes)}) + return {"structs": len(structs), "traits": len(traits), "implicitlyCopyable": len(implicitly), "ambiguous": len(dupes)} diff --git a/scripts/mojo/mojocorpus/processors/syntax.py b/scripts/mojo/mojocorpus/processors/syntax.py new file mode 100644 index 0000000..e48a2d4 --- /dev/null +++ b/scripts/mojo/mojocorpus/processors/syntax.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""The Mojo TextMate grammar's tests, as plain highlighting cases. + +mojo-syntax tests use vscode-tmgrammar-test assertions under each source line: + + var foo = 1 + # <- storage.modifier.declaration.python column 0 (one '-' = one char; '~' shifts right) + # ^^^ variable.other.python the columns under the carets + # ^^^ variable.other.python - keyword '- x' means x must NOT apply + +Each assertion becomes {line, text, start, end, scopes, notScopes}. Stage 1 +maps TextMate scopes onto the editor's token classes and runs these cases +against the Programming+ tokenizer. Cases for syntax the pin removed (the +grammar still tests `alias`) are flagged with the dialect markers so the +tokenizer can treat them as removed words instead. + +Writes syntax/cases.json, plus the grammar itself (syntax/mojo.syntax.json) +as a reference for keyword groups. +""" +from __future__ import annotations + +import json +import re + +from ..common import dialect_flags, write_json +from . import Context, fresh_dir + +NAME = "syntax" +SOURCE = "mojo-syntax" +INPUTS = ["tests", "syntaxes/mojo.syntax.json"] + +LEFT = re.compile(r"^(?P\s*#\s*)<(?P~*)(?P-+)\s+(?P.+)$") +CARET = re.compile(r"^\s*#(?P\s*)(?P\^+)\s+(?P.+)$") + + +def is_assertion(line: str) -> bool: + s = line.lstrip() + if not s.startswith("#"): + return False + t = s[1:].lstrip() + return t.startswith("<-") or t.startswith("<~") or t.startswith("^") + + +def split_scopes(rest: str) -> tuple[list[str], list[str]]: + want, _, never = rest.partition(" - ") + return want.split(), never.split() + + +def parse_cases(text: str, file: str) -> list[dict]: + lines = text.split("\n") + cases = [] + source_line = None + for i, line in enumerate(lines): + if i == 0 and line.startswith("# SYNTAX TEST"): + continue + if not is_assertion(line): + source_line = i + continue + if source_line is None: + continue + src = lines[source_line] + if m := LEFT.match(line): + start = len(m.group("shift")) + end = start + len(m.group("len")) + rest = m.group("rest") + elif m := CARET.match(line): + start = line.index("^") + end = start + len(m.group("carets")) + rest = m.group("rest") + else: + continue + want, never = split_scopes(rest) + cases.append({"file": file, "line": source_line + 1, "text": src, "start": start, "end": end, + "slice": src[start:end], "scopes": want, "notScopes": never}) + return cases + + +def run(ctx: Context) -> dict: + out = fresh_dir(NAME) + cases = [] + for f in sorted((ctx.root / "tests").glob("*.mojo")): + cases += parse_cases(f.read_text(encoding="utf-8"), f.name) + for c in cases: + flags = dialect_flags(c["text"], ctx.pin) + if flags: + c["dialect"] = flags + grammar = json.loads((ctx.root / "syntaxes" / "mojo.syntax.json").read_text(encoding="utf-8")) + write_json(out / "cases.json", {"processor": NAME, "provenance": ctx.prov("tests/*.mojo"), "cases": cases}) + write_json(out / "mojo.syntax.json", grammar) + return {"cases": len(cases), "files": len({c["file"] for c in cases}), + "flaggedOlder": sum(1 for c in cases if c.get("dialect", {}).get("older"))} diff --git a/scripts/mojo/mojocorpus/processors/tokens.py b/scripts/mojo/mojocorpus/processors/tokens.py new file mode 100644 index 0000000..a879f52 --- /dev/null +++ b/scripts/mojo/mojocorpus/processors/tokens.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""The words of the language, from the compiler and the stdlib themselves. + + hard keywords include/Mojo/MojoParser/TokenKinds.def, TOK_KEYWORD(...). A + trailing comment such as "Mojo extension" is kept; names that + start with "__" are internal and flagged so. + punctuation TokenKinds.def, TOK_PUNCTUATION(name, "spelling"). + soft keywords words the parser accepts by spelling in a context, found as + consumeIfSoftIdentifier("...") / isSoftIdentifier("...") in + lib/MojoParser. + removed any keyword or soft word the parser rejects with "'x' was + removed; use 'y'" (or "has been removed") is marked removed, + with its replacement: `fn` is still a token, but an error. + effects contextual words in a signature (raises, capturing, abi, ...), + the ones the parser compares by spelling and the grammar shows. + decorators the reference's decorator pages (docs/site/reference/decorators). + prelude names imported into every module, from stdlib/std/prelude. + +Writes tokens/tokens.json; the TypeScript emitter turns it into +src/mojo/generated/tokens.ts for the Stage 1 highlighter. +""" +from __future__ import annotations + +import re + +from ..common import PipelineError, write_json +from . import Context, fresh_dir + +NAME = "tokens" +SOURCE = "modular" +INPUTS = ["include/Mojo/MojoParser/TokenKinds.def", "lib/MojoParser"] + +KW = re.compile(r"^TOK_KEYWORD\((?P[^)]+?)\s*\)\s*(?://\s*(?P.*))?$") +PUNCT = re.compile(r'^TOK_PUNCTUATION\((?P\w+),\s*"(?P[^"]+)"\)') +LIT = re.compile(r"^TOK_LITERAL\((?P\w+)\)\s*(?://\s*(?P.*))?$") +SOFT = re.compile(r'(?:consumeIfSoftIdentifier|isSoftIdentifier|isSoftKeyword)\("(?P[A-Za-z_]+)"\)') +REMOVED = re.compile(r"""emit\w*Error\([^;]*?"'(?P\w+)' (?:was|has been) removed; use '(?P\w+)'""", re.S) +EFFECTS = ("raises", "capturing", "abi", "thin", "where", "escaping") + + +def parse_token_kinds(text: str) -> dict: + keywords, punctuation, literals = [], [], [] + for raw in text.split("\n"): + line = raw.strip() + if m := KW.match(line): + w = m.group("w").strip() + keywords.append({"word": w, "note": (m.group("c") or "").strip(), "internal": w.startswith("__")}) + elif m := PUNCT.match(line): + punctuation.append({"name": m.group("n"), "spelling": m.group("s")}) + elif m := LIT.match(line): + literals.append({"name": m.group("n"), "example": (m.group("c") or "").strip()}) + if not keywords: + raise PipelineError("TokenKinds.def has no TOK_KEYWORD entries; the format changed") + return {"keywords": keywords, "punctuation": punctuation, "literals": literals} + + +def parse_prelude(text: str) -> list[str]: + names = set() + for m in re.finditer(r"(?ms)^from\s+[\w.]+\s+import\s+(\((?P.*?)\)|(?P[^\n]+))", text): + body = m.group("group") or m.group("line") or "" + body = re.sub(r"#[^\n]*", "", body) + for part in body.split(","): + part = part.strip() + if not part: + continue + name = part.split(" as ")[-1].strip() + if re.fullmatch(r"[A-Za-z_]\w*", name): + names.add(name) + return sorted(names) + + +def run(ctx: Context) -> dict: + out = fresh_dir(NAME) + root = ctx.root + tk = root / "include" / "Mojo" / "MojoParser" / "TokenKinds.def" + data = parse_token_kinds(tk.read_text(encoding="utf-8")) + + soft: dict[str, dict] = {} + removed: dict[str, str] = {} + for f in sorted((root / "lib" / "MojoParser").rglob("*.cpp")): + text = f.read_text(encoding="utf-8", errors="replace") + for m in SOFT.finditer(text): + soft.setdefault(m.group("w"), {"word": m.group("w"), "seenIn": []})["seenIn"].append(f.name) + for m in REMOVED.finditer(text): + removed[m.group("w")] = m.group("r") + soft_list = [] + for w, s in sorted(soft.items()): + entry = {"word": w, "seenIn": sorted(set(s["seenIn"]))} + if w in removed: + entry["removed"] = True + entry["replacement"] = removed[w] + soft_list.append(entry) + + for k in data["keywords"]: + if k["word"] in removed: + k["removed"] = True + k["replacement"] = removed[k["word"]] + + deco_dir = root / "docs" / "site" / "reference" / "decorators" + decorators = sorted(p.stem.replace("-", "_") for p in deco_dir.glob("*.mdx") if p.stem != "index") if deco_dir.exists() else [] + prelude_file = root / "stdlib" / "std" / "prelude" / "__init__.mojo" + prelude = parse_prelude(prelude_file.read_text(encoding="utf-8")) if prelude_file.exists() else [] + if not prelude: + ctx.warn("tokens: no prelude names found; highlighting of built-in names will be empty") + + doc = { + "provenance": {"tokenKinds": ctx.prov("include/Mojo/MojoParser/TokenKinds.def"), + "softKeywords": ctx.prov("lib/MojoParser/*.cpp"), + "decorators": ctx.prov("docs/site/reference/decorators/*.mdx"), + "prelude": ctx.prov("stdlib/std/prelude/__init__.mojo")}, + **data, "softKeywords": soft_list, "effects": list(EFFECTS), + "decorators": decorators, "prelude": prelude, + } + write_json(out / "tokens.json", doc) + return {"keywords": len(data["keywords"]), "publicKeywords": sum(1 for k in data["keywords"] if not k["internal"] and not k.get("removed")), + "punctuation": len(data["punctuation"]), "softKeywords": len(soft_list), + "removedSoft": sorted(removed), "decorators": len(decorators), "prelude": len(prelude)} diff --git a/scripts/mojo/mojocorpus/verify.py b/scripts/mojo/mojocorpus/verify.py new file mode 100644 index 0000000..204e595 --- /dev/null +++ b/scripts/mojo/mojocorpus/verify.py @@ -0,0 +1,378 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Check every corpus item, probe, and local program against the pinned compiler. + +Two halves, kept apart on purpose: + + run (network) for each item without a golden, ask godbolt and store the + answer as library/mojo/golden/.json, where key hashes the + compiler id, mode, arguments (the pin's build.args, then the + program's own), stdin and source. Changing any of those (a new + pin, new build arguments, an edited probe) makes a new key, so + an old answer can never be mistaken for a new one. + judge (offline) read the goldens and decide each item's verdict. `status` + and `npm test` only ever do this, so they need no network. + +Verdicts: + pass did what its expectation says + fail did not (the reason says how) + observed a probe: no expectation, the golden IS the answer + skipped the item says why (newer dialect, no main, not reproducible) + unverified no golden yet for the pinned compiler + +Expectations (in manifests, or in a probe's header): + {"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out, + and stdout/patterns match if given + {"kind": "fail", "check": {...}} the same check must NOT hold (a starter) + {"kind": "compile-error", "messages": [...]} must not compile, and each + expected error appears at its line + {"kind": "observe"} / {"kind": "skip", "why": ...} + +Probe and local program headers (comment lines at the top of the file): + # probe: the question this program answers + # stdin: text with \\n escapes # args: compiler arguments, after the pin's build.args + # mode: run | asm | both # expect-stdout: a line (repeatable) + # expect: pass | fail | compile-error | observe +""" +from __future__ import annotations + +import concurrent.futures as cf +import datetime as _dt +import json +import re +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable + +from . import godbolt +from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, build_args, load_pin, log, read_json, rel, sha256_bytes, write_json, write_text + +CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress") +CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",) +# The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested +# corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading +# in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures". +CORPORA_OURS = ("probes", "local", "stress") + + +@dataclass +class Item: + id: str + corpus: str + path: Path + source: str + expect: dict[str, Any] + mode: str = "run" + stdin: str = "" + args: str = "" + question: str = "" + extra: dict[str, Any] = field(default_factory=dict) + build: str = "" # the pin's build.args, which come before the program's own `args` + + @property + def command_args(self) -> str: + """Every argument the program is built with: the pin's, then its own (common.build_args).""" + return build_args(self.build, self.args) + + def key(self, compiler: str) -> str: + return sha256_bytes(json.dumps([compiler, self.mode, self.command_args, self.stdin, self.source], ensure_ascii=False).encode()) + + +# -------------------------------------------------------------------------- +# Collecting items +# -------------------------------------------------------------------------- + +HEADER = re.compile(r"^#\s*(?Pprobe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P.*)$") + + +def parse_header(text: str) -> dict[str, Any]: + h: dict[str, Any] = {"expect-stdout": []} + for line in text.split("\n"): + if not line.startswith("#"): + break + m = HEADER.match(line) + if not m: + continue + k, v = m.group("k"), m.group("v").rstrip() + if k == "expect-stdout": + h[k].append(v) + elif k == "stdin": + h[k] = v.encode("utf-8").decode("unicode_escape") + else: + h[k] = v.strip() + return h + + +def header_item(path: Path, corpus: str, base: Path) -> Item: + text = path.read_text(encoding="utf-8") + h = parse_header(text) + want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe") + if want == "pass": + expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}} + elif want == "fail": + expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}} + elif want == "compile-error": + expect = {"kind": "compile-error", "messages": []} + elif want == "observe": + expect = {"kind": "observe"} + else: + raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'") + mode = h.get("mode", "run") + if mode not in ("run", "asm", "both"): + raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'") + return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path, + source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""), + question=h.get("probe", "")) + + +def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]: + mf = corpus_dir / corpus / "manifest.json" + if not mf.exists(): + return [] + out = [] + for it in read_json(mf)["items"]: + p = corpus_dir / corpus / it["file"] + expect = it["expect"] + if corpus == "diagnostics" and it.get("reproducible") == "unlikely": + expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect} + out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect, + mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""), + extra={"dialect": it.get("dialect", {})})) + return out + + +def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL, + build: str | None = None) -> list[Item]: + """Every item of the given corpora. `build` is the pin's build.args (read from the pin when not given): + each item is built with them before its own arguments, and its stored answer is keyed by both.""" + if build is None: + build = load_pin().build + items: list[Item] = [] + for c in corpora: + if c == "probes": + items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))] + elif c == "local": + items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))] + elif c == "stress": + items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))] + elif c in ("quest", "manual", "diagnostics"): + items += manifest_items(c, corpus_dir) + else: + raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}") + ids = [i.id for i in items] + dupes = {x for x in ids if ids.count(x) > 1} + if dupes: + raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}") + for it in items: + it.build = build + return items + + +# -------------------------------------------------------------------------- +# Judging (offline) +# -------------------------------------------------------------------------- + +def normalize_output(s: str) -> str: + """Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace.""" + s = s.replace("\r\n", "\n") + s = "\n".join(l.rstrip() for l in s.split("\n")) + return s.rstrip("\n").strip() + + +def _js_flags(flags: str) -> int: + f = 0 + for c in flags or "": + f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0) + return f + + +def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]: + if not result.get("compiled"): + return False, "did not compile" + if result.get("timedOut"): + return False, "timed out" + if not result.get("executed") or result.get("exitCode") != 0: + return False, f"exited abnormally (exit code {result.get('exitCode')})" + if "stdout" in check: + want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", "")) + if want != got: + return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}" + for p in check.get("patterns", []): + if not re.search(p, source, _js_flags(check.get("flags", ""))): + return False, f"source lacks the required idiom ({check.get('message') or p})" + return True, "ok" + + +def _message_regex(m: dict[str, Any]) -> str: + text = m["message"] + if not m.get("regex"): + return re.escape(text) + parts = re.split(r"(\{\{.*?\}\})", text) + return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts) + + +def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]: + kind = item.expect.get("kind") + if kind == "skip": + return "skipped", item.expect.get("why", "") + if golden is None: + return "unverified", "no answer from the pinned compiler yet" + r = golden["result"] + if kind == "observe": + return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run") + if kind == "pass": + ok, why = check_holds(item.expect.get("check", {}), r, item.source) + return ("pass", why) if ok else ("fail", why) + if kind == "fail": + ok, why = check_holds(item.expect.get("check", {}), r, item.source) + return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}") + if kind == "compile-error": + if r.get("compiled"): + return "fail", "compiled, but errors were expected" + stderr = r.get("buildStderr", "") + missing = [m for m in item.expect.get("messages", []) + if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)] + if missing: + m = missing[0] + return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}" + return "pass", "rejected with the expected errors" + return "fail", f"unknown expectation {kind!r}" + + +def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path: + return golden_dir / f"{key}.json" + + +def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None: + p = golden_path(item.key(compiler), golden_dir) + return read_json(p) if p.exists() else None + + +def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True, + corpus_dir: Path = CORPUS) -> dict[str, Any]: + items = collect(corpora, corpus_dir, build=pin.build) + rows = [] + for it in items: + g = load_golden(it, pin.compiler, golden_dir) + verdict, why = judge(it, g) + rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why, + "key": it.key(pin.compiler) if g else None, "question": it.question or None}) + summary: dict[str, dict[str, int]] = {} + for r in rows: + summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0) + summary[r["corpus"]][r["verdict"]] += 1 + doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows} + if write: + write_json(golden_dir / "status.json", doc) + write_text(golden_dir / "STATUS.md", status_markdown(doc)) + return doc + + +def prune(pin: Pin, golden_dir: Path = GOLDEN, corpus_dir: Path = CORPUS, dry_run: bool = False, + keys: set[str] | None = None) -> dict[str, int]: + """Remove the stored answers nothing reads any more. + + An answer is keyed by compiler, mode, arguments, stdin and source, so two + kinds are dead: those of a compiler other than the pin (after a version + bump, every one of the old pin's), and those of the pin for a program that + has since been edited or removed. Git history keeps both. `keys` stands in + for the live keys in tests. + """ + live = keys if keys is not None else {it.key(pin.compiler) for it in collect(CORPORA_ALL, corpus_dir, build=pin.build)} + other = orphaned = kept = 0 + for p in sorted(golden_dir.glob("*.json")): + if p.name == "status.json": + continue + g = read_json(p) + if g.get("compiler") != pin.compiler: + other += 1 + elif p.stem not in live: + orphaned += 1 + else: + kept += 1 + continue + if not dry_run: + p.unlink() + return {"otherCompiler": other, "orphaned": orphaned, "kept": kept} + + +def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]: + """The failing items in the given corpora: what `status --strict` refuses.""" + wanted = set(corpora) + return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted] + + +def status_markdown(doc: dict[str, Any]) -> str: + verdicts = ["pass", "fail", "observed", "skipped", "unverified"] + lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "", + "Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "", + "| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)] + for c, counts in sorted(doc["summary"].items()): + lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |") + fails = [r for r in doc["items"] if r["verdict"] == "fail"] + if fails: + lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails] + probes = [r for r in doc["items"] if r["corpus"] == "probes"] + if probes: + lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"] + lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes] + return "\n".join(lines) + "\n" + + +# -------------------------------------------------------------------------- +# Running (network) +# -------------------------------------------------------------------------- + +def _now() -> str: + return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat() + + +def ask(pin: Pin, item: Item) -> dict[str, Any]: + body = godbolt.request_body(item.source, item.mode, item.stdin, item.command_args) + t0 = time.monotonic() + tries: list[int] = [] + data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries) + return { + "attempts": tries[0] if tries else 1, + "key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.command_args, "stdin": item.stdin, + "sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(), + "seconds": round(time.monotonic() - t0, 2), + "result": godbolt.normalize(item.mode, data), "raw": godbolt.slim_raw(data), + } + + +def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None, + dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]: + items = [i for i in collect(corpora, build=pin.build) if i.expect.get("kind") != "skip"] + if only_ids: + items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))] + seen: dict[str, Item] = {} + for i in items: # one request per distinct key + seen.setdefault(i.key(pin.compiler), i) + todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()] + if limit is not None: + todo = todo[:limit] + log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})") + if dry_run: + if todo: + log("[verify] dry run; the first request would be:") + print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].command_args), indent=2)[:3000]) + return {"asked": 0, "failed": 0, "todo": len(todo)} + asked = failed = 0 + workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server + with cf.ThreadPoolExecutor(max_workers=workers) as pool: + futs = {pool.submit(ask, pin, i): i for i in todo} + for f in cf.as_completed(futs): + it = futs[f] + try: + g = f.result() + except PipelineError as e: + failed += 1 + log(f"[verify] {it.id}: {e}") + continue + write_json(golden_path(g["key"], golden_dir), g) + asked += 1 + if asked % 10 == 0: + log(f"[verify] {asked}/{len(todo)}") + return {"asked": asked, "failed": failed, "todo": len(todo)} diff --git a/scripts/mojo/new-program.mjs b/scripts/mojo/new-program.mjs new file mode 100644 index 0000000..3d1c8d9 --- /dev/null +++ b/scripts/mojo/new-program.mjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Start a Mojo program of our own in the corpus: a probe or a stress program. + * (An exercise's program in library/mojo/local/ is made by `pixi run new-exercise `, + * which also writes the exercise that shows it.) + * + * pixi run new-probe int-floordiv-negative "does // floor toward negative infinity?" + * pixi run new-stress ring-buffer "a fixed-size queue with wraparound" + * + * probe library/mojo/probes/.mojo one question about the pinned compiler; godbolt's + * answer is the point (expectation: observe) + * stress library/mojo/stress/.mojo realistic Mojo for the interpreter; its + * `# expect-stdout:` lines are the prediction that + * godbolt confirms or refutes (expectation: pass) + * + * Then: edit the file, `pixi run mojo-verify-item '/'` (network), and + * `pixi run test`. docs/mojo-corpus.md, "Mojo of our own", has the header keys. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = fileURLToPath(new URL('../..', import.meta.url)); +const KINDS = { + probe: { dir: 'library/mojo/probes', corpus: 'probes' }, + stress: { dir: 'library/mojo/stress', corpus: 'stress' }, +}; + +function die(msg) { console.error(`new-program: ${msg}`); process.exit(1); } + +const [kind, name, ...rest] = process.argv.slice(2); +const about = rest.join(' ').trim(); +if (!KINDS[kind]) die('usage: new-program.mjs probe|stress ""'); +if (!name || !/^[a-z0-9]+(-[a-z0-9]+)*$/.test(name)) die(`"${name ?? ''}" is not a valid name: lowercase letters and digits, joined by single hyphens`); +if (!about) die(`say in one line what the ${kind} is about, after its name`); + +const file = path.join(ROOT, KINDS[kind].dir, `${name}.mojo`); +if (fs.existsSync(file)) die(`${path.relative(ROOT, file)} already exists; nothing was changed`); + +const text = kind === 'probe' + ? `# probe: ${about} +def main(): + print("replace this with the smallest program that answers the question") +` + : `# probe: ${about} +# status: the expected output is Programming+'s interpreter's prediction; corpus.py verify confirms or refutes it on godbolt +# expect-stdout: replace with the first line the program should print +def main(): + print("replace with the first line the program should print") +`; + +fs.mkdirSync(path.dirname(file), { recursive: true }); +fs.writeFileSync(file, text); +console.log(`created ${path.relative(ROOT, file)} + +next: + edit it (the header block is read by corpus.py verify; the program starts at the first other line) + pixi run mojo-verify-item '${KINDS[kind].corpus}/${name}' # ask godbolt (network) + pixi run test`); diff --git a/scripts/mojo/tests/test_pipeline.py b/scripts/mojo/tests/test_pipeline.py new file mode 100644 index 0000000..362d128 --- /dev/null +++ b/scripts/mojo/tests/test_pipeline.py @@ -0,0 +1,439 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the Mojo corpus pipeline. Offline: no test touches the network. + + python3 -m unittest discover -s scripts/mojo/tests (or: pixi run mojo-test) +""" +from __future__ import annotations + +import io +import json +import stat +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from mojocorpus import common, emit_ts, godbolt, verify # noqa: E402 +from mojocorpus.common import Pin, PipelineError, Source, dialect_flags, load_pin, read_json # noqa: E402 +from mojocorpus.fetch import _clean_names, extract_subtree, fetch_all, find_root # noqa: E402 +from mojocorpus.harvest import detect_local_zips # noqa: E402 +from mojocorpus.processors import diagnostics, editor, messages, release, skill, syntax, tokens # noqa: E402 + +REPO = common.REPO + + +def make_zip(path: Path, files: dict[str, str], links: dict[str, str] | None = None) -> Path: + with zipfile.ZipFile(path, "w") as zf: + for name, text in files.items(): + zf.writestr(name, text) + for name, target in (links or {}).items(): + info = zipfile.ZipInfo(name) + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + zf.writestr(info, target) + return path + + +def tiny_pin(**refs) -> Pin: + return Pin(version="1.1.0", compiler="mojo_1_1_0", base="https://godbolt.org", concurrency=2, timeout=30, + sources={ + "modular": Source("modular", "modular/modular", refs.get("modular", common.UNRESOLVED), + ["stdlib/std/prelude/__init__.mojo", "docs/site/releases"], "Apache-2.0"), + "mojo-syntax": Source("mojo-syntax", "modular/mojo-syntax", refs.get("mojo-syntax", common.UNRESOLVED), + ["syntaxes/mojo.syntax.json"], "MIT"), + }, + newer=[{"id": "match", "pattern": r"^\s*(?:comptime\s+)?__match\b", "why": "nightly"}], + older=[{"id": "fn", "pattern": r"^\s*fn\s+\w", "why": "removed"}]) + + +MODULAR_FILES = {"stdlib/std/prelude/__init__.mojo": "from std.builtin import Int\n", + "docs/site/releases/v1.1.0.md": "## Removed\n\n- `fn` is gone.\n"} + + +# -------------------------------------------------------------------------- +class FetchTests(unittest.TestCase): + def test_root_is_found_by_markers_whatever_the_wrapping_directory(self): + markers = ["stdlib/std/prelude/__init__.mojo", "docs/site/releases"] + for prefix in ("Mojo/", "modular-0123abc/mojo/", ""): + names = [prefix + n for n in MODULAR_FILES] + ["modular-0123abc/README.md"] + self.assertEqual(find_root(names, markers), prefix, prefix) + + def test_a_directory_marker_needs_no_directory_entry_in_the_zip(self): + self.assertEqual(find_root(["a/docs/site/releases/v1.md", "a/stdlib/std/prelude/__init__.mojo"], + ["stdlib/std/prelude/__init__.mojo", "docs/site/releases"]), "a/") + + def test_missing_markers_name_the_markers(self): + with self.assertRaisesRegex(PipelineError, "prelude"): + find_root(["x/README.md"], ["stdlib/std/prelude/__init__.mojo"]) + + def test_path_traversal_is_refused_and_junk_and_links_are_skipped(self): + with tempfile.TemporaryDirectory() as d: + bad = make_zip(Path(d) / "bad.zip", {"../evil.txt": "x"}) + with zipfile.ZipFile(bad) as zf, self.assertRaisesRegex(PipelineError, "escapes"): + _clean_names(zf) + ok = make_zip(Path(d) / "ok.zip", {"a/f.txt": "x", "__MACOSX/a/._f.txt": "y", "a/.DS_Store": "z"}, + links={"a/link": "/etc/passwd"}) + with zipfile.ZipFile(ok) as zf: + self.assertEqual([i.filename for i in _clean_names(zf)], ["a/f.txt"]) + + def test_only_the_marked_subtree_is_extracted(self): + with tempfile.TemporaryDirectory() as d: + z = make_zip(Path(d) / "m.zip", {**{"top/mojo/" + k: v for k, v in MODULAR_FILES.items()}, "top/max/big.bin": "x" * 100}) + root, prefix = extract_subtree(z, ["stdlib/std/prelude/__init__.mojo", "docs/site/releases"], Path(d) / "out") + self.assertEqual(prefix, "top/mojo") + self.assertTrue((root / "docs/site/releases/v1.1.0.md").exists()) + self.assertFalse((Path(d) / "out/top/max").exists()) + + def test_local_zip_lock_records_hash_and_stands_in_for_a_resolved_ref(self): + with tempfile.TemporaryDirectory() as d: + d = Path(d) + z = make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()}) + lock = d / "lock.json" + pin = tiny_pin(modular="mojo/v1.1.0") + wd = d / "wd"; wd.mkdir() + fetch_all(pin, wd, {"modular": z}, only={"modular"}, lock_path=lock) + entry = read_json(lock)["sources"]["modular"] + self.assertEqual(entry["kind"], "local") + self.assertEqual(entry["sha256"], common.sha256_file(z)) + self.assertEqual(entry["standsInFor"], "mojo/v1.1.0") + self.assertEqual(entry["pinnedRef"], "local-override") + self.assertEqual(entry["root"], "Mojo") + + def test_an_unresolved_ref_with_a_local_lock_insists_on_the_same_zip(self): + with tempfile.TemporaryDirectory() as d: + d = Path(d) + z = make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()}) + lock = d / "lock.json" + pin = tiny_pin() + (d / "wd1").mkdir(); (d / "wd2").mkdir() + fetch_all(pin, d / "wd1", {"modular": z}, only={"modular"}, lock_path=lock) + with self.assertRaisesRegex(PipelineError, "local zip"): + fetch_all(pin, d / "wd2", {}, only={"modular"}, lock_path=lock) + + def test_an_unresolved_ref_without_a_local_zip_explains_what_to_do(self): + with tempfile.TemporaryDirectory() as d: + with self.assertRaisesRegex(PipelineError, "UNRESOLVED"): + fetch_all(tiny_pin(), Path(d), {}, only={"modular"}, lock_path=Path(d) / "lock.json") + + def test_local_dir_matches_zips_by_markers_and_leaves_resolved_sources_to_github(self): + with tempfile.TemporaryDirectory() as d: + d = Path(d) + make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()}) + make_zip(d / "syntax.zip", {"mojo-syntax-main/syntaxes/mojo.syntax.json": "{}"}) + make_zip(d / "other.zip", {"x/readme.txt": "?"}) + self.assertEqual(sorted(detect_local_zips(tiny_pin(), d)), ["modular", "mojo-syntax"]) + resolved = tiny_pin(modular="mojo/v1.1.0") + self.assertEqual(sorted(detect_local_zips(resolved, d)), ["mojo-syntax"]) + self.assertEqual(sorted(detect_local_zips(resolved, d, include_resolved=True)), ["modular", "mojo-syntax"]) + + +# -------------------------------------------------------------------------- +class PinTests(unittest.TestCase): + def test_the_committed_pin_loads_and_names_one_release_three_ways(self): + # the version, godbolt's compiler id and the modular ref are one release, whichever it is: a bump edits all three + pin = load_pin() + self.assertRegex(pin.version, r"^\d+\.\d+\.\d+$") + self.assertEqual(pin.compiler, "mojo_" + pin.version.replace(".", "_")) + self.assertEqual(pin.sources["modular"].ref, "mojo/v" + pin.version) + self.assertTrue(pin.older) + # a release tag is the release itself, so nothing in it is newer than the pin + self.assertEqual(pin.newer, []) + + def test_dialect_markers_match_code_not_comments_or_strings(self): + pin = tiny_pin() + self.assertEqual(dialect_flags("fn main():\n pass\n", pin), {"older": ["fn"]}) + self.assertEqual(dialect_flags('# fn main() was the old way\ndef main():\n print("fn x")\n', pin), {}) + self.assertEqual(dialect_flags('def f():\n """\nfn g(): docstring\n"""\n', pin), {}) + self.assertEqual(dialect_flags("def f(x: Int):\n comptime __match x:\n", pin), {"newer": ["match"]}) + + +# -------------------------------------------------------------------------- +class ProcessorTests(unittest.TestCase): + def test_annotations_resolve_to_the_lines_they_name(self): + src = "\n".join([ + "def f():", # 1 + " x = y # expected-error {{use of unknown declaration 'y'}}", # 2 + " # expected-error @below {{first}}", # 3 + " # expected-note @below {{second, same line}}", # 4 + "", # 5 + " z()", # 6 + " # expected-error @+2 {{two down}}", # 7 + " a()", # 8 + " b()", # 9 + " c()", # 10 + " # expected-warning @above {{c is odd}}", # 11 + " d() # expected-error-re {{value of type '{{.*}}' cannot}}", # 12 + ]) + got = [(a["kind"], a["line"], a["message"], a["regex"]) for a in diagnostics.parse_annotations(src)] + self.assertEqual(got, [ + ("error", 2, "use of unknown declaration 'y'", False), + ("error", 6, "first", False), + ("note", 6, "second, same line", False), + ("error", 9, "two down", False), + ("warning", 10, "c is odd", False), + ("error", 12, "value of type '{{.*}}' cannot", True), + ]) + + def test_the_pass_comes_from_the_verify_run_line(self): + self.assertEqual(diagnostics.pass_of(["%parse-mojo %s | FileCheck %s", "%parse-mojo %s | kgen-opt -check-lifetimes -verify-diagnostics"]), "lifetimes") + self.assertEqual(diagnostics.pass_of(["%parse-mojo-isolated %s -verify-diagnostics"]), "parser") + + def test_messages_keep_wording_placeholders_and_fixits(self): + cpp = ''' + } else if (p.consumeIfSoftIdentifier("read")) { + p.emitError(loc, "'read' was removed; use 'imm'") + << FixIt::replaceToken(loc, "imm"); + } + emitError(loc) << "use of uninitialized value '" << name << "'"; + diag.attachNote(other) << "declared here"; +''' + got = [(m["severity"], m["message"], m["fixits"]) for m in messages.scan(cpp)] + self.assertEqual(got, [ + ("error", "'read' was removed; use 'imm'", [{"kind": "replaceToken", "text": "imm"}]), + ("error", "use of uninitialized value '{}'", []), + ("note", "declared here", []), + ]) + + def test_token_kinds_and_prelude_parse(self): + tk = tokens.parse_token_kinds('TOK_KEYWORD(def)\nTOK_KEYWORD(fn) // Mojo extension.\nTOK_KEYWORD(__mlir_region)\n' + 'TOK_PUNCTUATION(minus_greater, "->")\nTOK_LITERAL(integer) // 42\n') + self.assertEqual([k["word"] for k in tk["keywords"]], ["def", "fn", "__mlir_region"]) + self.assertEqual(tk["keywords"][1]["note"], "Mojo extension.") + self.assertTrue(tk["keywords"][2]["internal"]) + self.assertEqual(tk["punctuation"], [{"name": "minus_greater", "spelling": "->"}]) + pre = tokens.parse_prelude("from std.a import (\n Int,\n String, # text\n)\nfrom std.b import input, print\nfrom std.c import x as y\n") + self.assertEqual(pre, ["Int", "String", "input", "print", "y"]) + self.assertIsNotNone(tokens.REMOVED.search('''emitError(getToken().getLoc(), "'fn' has been removed; use 'def' instead")''')) + + def test_syntax_assertions_become_cases(self): + text = '# SYNTAX TEST "source.mojo" "x"\nvar foo = 1\n# <- storage.modifier\n# ^^^ variable.other - keyword\n' + cases = syntax.parse_cases(text, "t.mojo") + self.assertEqual([(c["slice"], c["scopes"], c["notScopes"]) for c in cases], + [("v", ["storage.modifier"], []), ("foo", ["variable.other"], ["keyword"])]) + self.assertEqual(syntax.parse_cases("x = 1\n# <~~- a\n", "t")[0]["slice"], "=") # two '~' shift to column 2 + + def test_release_sections_and_skill_table(self): + md = "## Removed {#removed}\n\n- Removed `fn`. Use `def`.\n- Removed\n `alias`, too.\n\n## Fixed\n\n- A bug.\n" + secs = release.sections(md) + self.assertEqual([(s["title"], s["anchor"], s["bullets"]) for s in secs], + [("Removed", "removed", ["Removed `fn`.", "Removed `alias`, too."]), ("Fixed", "", ["A bug."])]) + rows = skill.table_after("## Removed syntax\n\n| Old | New |\n|---|---|\n| `fn` | `def` |\n\n## Next\n", "Removed syntax") + self.assertEqual(rows, [["Old", "New"], ["`fn`", "`def`"]]) + + def test_release_relation_reads_the_changelog_at_a_tag_cut_before_its_notes(self): + r = release.relate + self.assertEqual(r("1.1.0", [(1, 0, 0), (1, 1, 0)], True, 0, "mojo/v1.1.0"), ("same", "release-file")) + self.assertEqual(r("1.1.0", [(1, 0, 0), (1, 1, 0)], True, 9, "main"), ("newer", "release-file")) + self.assertEqual(r("1.1.0", [(1, 1, 0), (1, 2, 0)], True, 0, "mojo/v1.2.0"), ("newer", "release-file")) + # mojo/v1.2.0 was tagged with its notes still in the nightly changelog + self.assertEqual(r("1.2.0", [(1, 0, 0), (1, 1, 0)], False, 66, "mojo/v1.2.0"), ("same", "nightly-changelog")) + # the same snapshot reached by any other ref is only known to be older than the pin + self.assertEqual(r("1.2.0", [(1, 0, 0), (1, 1, 0)], False, 66, "527d069ee11a"), ("older", "none")) + self.assertEqual(r("1.2.0", [(1, 0, 0), (1, 1, 0)], False, 0, "mojo/v1.2.0"), ("older", "none")) + self.assertEqual(r("1.2.0", [], False, 0, "mojo/v1.2.0"), ("unknown", "none")) + + def test_vscode_json_with_comments_and_trailing_commas(self): + self.assertEqual(editor.loads_jsonc('{\n // c\n "a": "//not a comment", /* b */ "x": [1, 2,],\n}'), + {"a": "//not a comment", "x": [1, 2]}) + + +# -------------------------------------------------------------------------- +class GodboltTests(unittest.TestCase): + def test_run_requests_match_mojo_quest(self): + b = godbolt.request_body("def main(): pass", "run", stdin="1\n") + self.assertEqual(b["options"]["compilerOptions"], {"executorRequest": True, "skipAsm": True}) + self.assertEqual(b["options"]["executeParameters"], {"args": [], "stdin": "1\n"}) + self.assertEqual(b["lang"], "mojo") + self.assertTrue(godbolt.request_body("x", "both")["options"]["filters"]["execute"]) + with self.assertRaises(PipelineError): + godbolt.request_body("x", "jit") + + def test_answers_normalize_to_one_shape(self): + run = godbolt.normalize("run", {"buildResult": {"code": 0, "stderr": [{"text": "warning: w"}]}, "didExecute": True, + "code": 0, "stdout": [{"text": "hi"}, {"text": "there"}], "stderr": [], "timedOut": False}) + self.assertEqual((run["compiled"], run["executed"], run["exitCode"], run["stdout"], run["buildStderr"]), + (True, True, 0, "hi\nthere", "warning: w")) + both = godbolt.normalize("both", {"code": 0, "stderr": [], "asm": [{"text": "add:", "source": None}, {"text": " ret", "source": {"line": 2}}], + "execResult": {"didExecute": True, "code": 0, "stdout": [{"text": "5"}], "stderr": []}}) + self.assertEqual(both["asm"], [{"text": "add:", "line": None}, {"text": " ret", "line": 2, "file": None}]) + marked = godbolt.normalize("asm", {"code": 0, "asm": [{"text": " lea", "source": {"file": "/lib/io.mojo", "line": 218, "mainsource": False}}]}) + self.assertEqual(marked["asm"], [{"text": " lea", "line": 218, "file": "/lib/io.mojo", "main": False}]) + self.assertEqual((both["stdout"], both["returnedAsm"], both["returnedExec"]), ("5", True, True)) + + +# -------------------------------------------------------------------------- +class VerifyTests(unittest.TestCase): + def item(self, **kw): + base = dict(id="t/x", corpus="t", path=Path("x.mojo"), source="def main():\n pass\n", expect={"kind": "pass", "check": {}}) + base.update(kw) + return verify.Item(**base) + + @staticmethod + def golden(**r): + res = {"compiled": True, "executed": True, "exitCode": 0, "stdout": "", "stderr": "", "timedOut": False, "buildStderr": ""} + res.update(r) + return {"result": res} + + def test_output_is_normalized_like_mojo_quest(self): + self.assertEqual(verify.normalize_output("a \r\nb\t\n\n\n"), "a\nb") + + def test_pass_and_fail_expectations(self): + it = self.item(expect={"kind": "pass", "check": {"stdout": "3"}}) + self.assertEqual(verify.judge(it, self.golden(stdout="3\n"))[0], "pass") + self.assertEqual(verify.judge(it, self.golden(stdout="4"))[0], "fail") + self.assertEqual(verify.judge(it, self.golden(compiled=False))[1], "did not compile") + self.assertIn("exit code 1", verify.judge(it, self.golden(exitCode=1))[1]) + self.assertEqual(verify.judge(it, None)[0], "unverified") + starter = self.item(expect={"kind": "fail", "check": {"stdout": "3"}}) + self.assertEqual(verify.judge(starter, self.golden(stdout="2"))[0], "pass") + self.assertEqual(verify.judge(starter, self.golden(stdout="3"))[0], "fail") + + def test_source_patterns_use_javascript_flags(self): + it = self.item(source='def f():\n """Doc."""\n', expect={"kind": "pass", "check": {"patterns": ['"""[\\s\\S]+?"""'], "flags": ""}}) + self.assertEqual(verify.judge(it, self.golden())[0], "pass") + it2 = self.item(source="DEF f", expect={"kind": "pass", "check": {"patterns": ["def"], "flags": "i"}}) + self.assertEqual(verify.judge(it2, self.golden())[0], "pass") + + def test_compile_errors_must_appear_at_their_lines(self): + it = self.item(expect={"kind": "compile-error", "messages": [ + {"line": 3, "message": "use of uninitialized value 'x'", "regex": False}, + {"line": 5, "message": "value of type '{{[A-Za-z]+}}' cannot be", "regex": True}]}) + stderr = ("/app/example.mojo:3:11: error: use of uninitialized value 'x'\n" + "/app/example.mojo:5:1: error: value of type 'Loud' cannot be implicitly copied\n") + self.assertEqual(verify.judge(it, self.golden(compiled=False, buildStderr=stderr))[0], "pass") + self.assertEqual(verify.judge(it, self.golden(compiled=False, buildStderr=stderr.replace(":3:", ":4:")))[0], "fail") + self.assertEqual(verify.judge(it, self.golden(compiled=True))[0], "fail") + + def test_golden_keys_change_with_anything_that_changes_the_answer(self): + a = self.item() + k = a.key("mojo_1_1_0") + self.assertEqual(k, self.item().key("mojo_1_1_0")) + self.assertNotEqual(k, a.key("mojo_1_2_0")) + self.assertNotEqual(k, self.item(stdin="1").key("mojo_1_1_0")) + self.assertNotEqual(k, self.item(args="-O0").key("mojo_1_1_0")) + self.assertNotEqual(k, self.item(mode="asm").key("mojo_1_1_0")) + self.assertNotEqual(k, self.item(source="def main():\n pass \n").key("mojo_1_1_0")) + # the pin's build arguments are part of the question + self.assertNotEqual(k, self.item(build="--fp-mode contract=off").key("mojo_1_1_0")) + self.assertEqual(self.item(build="--fp-mode contract=off", args="-O0").command_args, "--fp-mode contract=off -O0") + + def test_build_arguments_are_the_pins_then_the_programs_own(self): + base = "--target-cpu x86-64-v3 --fp-mode contract=off" + self.assertEqual(common.build_args(base), base) + self.assertEqual(common.build_args(base, "-O0 --debug-level=line-tables"), base + " -O0 --debug-level=line-tables") + self.assertEqual(common.build_args(base, "--fp-mode contract=fast"), "--target-cpu x86-64-v3 --fp-mode contract=fast") + self.assertEqual(common.build_args(base, "--target-cpu x86-64-v4"), "--fp-mode contract=off --target-cpu x86-64-v4") + self.assertEqual(common.build_args(base, "-O0", x86_64=False), "--fp-mode contract=off -O0") + self.assertEqual(load_pin().build, read_json(common.PIN)["build"]["args"]) + + def test_items_are_built_with_the_pins_arguments(self): + pin = load_pin() + built = verify.collect(["probes"], build=pin.build) + bare = verify.collect(["probes"], build="") + self.assertEqual([i.id for i in verify.collect(["probes"])], [i.id for i in built]) + for it, plain in zip(built, bare): + self.assertEqual(it.command_args, common.build_args(pin.build, it.args), it.id) + self.assertNotEqual(it.key(pin.compiler), plain.key(pin.compiler), f"{it.id}: an answer without the pin's arguments is another question") + + def test_headers_set_stdin_mode_args_and_expectations(self): + h = verify.parse_header("# probe: q\n# stdin: 1\\n2\\n\n# mode: both\n# args: -O0\n# expect-stdout: a\n# expect-stdout: b\ndef main(): pass\n# probe: not a header\n") + self.assertEqual((h["probe"], h["stdin"], h["mode"], h["args"], h["expect-stdout"]), ("q", "1\n2\n", "both", "-O0", ["a", "b"])) + with tempfile.TemporaryDirectory() as d: + p = Path(d) / "p.mojo" + p.write_text("# expect-stdout: 3\ndef main():\n print(3)\n") + self.assertEqual(verify.header_item(p, "local", Path(d)).expect, {"kind": "pass", "check": {"stdout": "3"}}) + p.write_text("# expect: sometimes\n") + with self.assertRaises(PipelineError): + verify.header_item(p, "local", Path(d)) + + + def test_prune_removes_other_compilers_and_orphans_only(self): + with tempfile.TemporaryDirectory() as d: + g = Path(d) + write = lambda key, compiler: (g / f"{key}.json").write_text(json.dumps({"compiler": compiler, "key": key})) + write("live", "mojo_9_9_9"); write("edited", "mojo_9_9_9"); write("old", "mojo_9_9_8") + (g / "status.json").write_text("{}") + pin = tiny_pin() + pin.compiler = "mojo_9_9_9" + self.assertEqual(verify.prune(pin, golden_dir=g, dry_run=True, keys={"live"}), {"otherCompiler": 1, "orphaned": 1, "kept": 1}) + self.assertEqual(sorted(p.name for p in g.iterdir()), ["edited.json", "live.json", "old.json", "status.json"], "a dry run removes nothing") + verify.prune(pin, golden_dir=g, keys={"live"}) + self.assertEqual(sorted(p.name for p in g.iterdir()), ["live.json", "status.json"]) + + +# -------------------------------------------------------------------------- +class CommittedCorpusTests(unittest.TestCase): + """The corpus in the repository is whole: every file a manifest names exists, ids are unique.""" + + KINDS = {"pass", "fail", "compile-error", "observe", "skip"} + + def test_manifests_are_consistent(self): + corpus = common.CORPUS + if not (corpus / "report.json").exists(): + self.skipTest("no harvested corpus") + for name in ("quest", "manual", "diagnostics"): + mf = read_json(corpus / name / "manifest.json") + ids = [i["id"] for i in mf["items"]] + self.assertEqual(len(ids), len(set(ids)), name) + for it in mf["items"]: + self.assertTrue((corpus / name / it["file"]).exists(), it["file"]) + self.assertIn(it["expect"]["kind"], self.KINDS, it["id"]) + self.assertIn("path", it["provenance"], it["id"]) + + def test_every_probe_and_local_program_parses_and_asks_a_question(self): + items = verify.collect(["probes", "local"]) + self.assertGreaterEqual(len(items), 60) + for it in items: + self.assertTrue(it.question, f"{it.id} has no '# probe:' line") + self.assertRegex(it.source, r"(?m)^(def|fn) main\(", it.id) # the fn probe keeps fn on purpose + + def test_status_judges_every_item_without_the_network(self): + doc = verify.status(load_pin(), write=False) + verdicts = {"pass", "fail", "observed", "skipped", "unverified"} + self.assertTrue(all(r["verdict"] in verdicts for r in doc["items"])) + + def test_strict_status_holds_our_corpora_and_reports_upstream(self): + doc = {"items": [ + {"id": "manual/a", "corpus": "manual", "verdict": "fail", "why": "newer syntax"}, + {"id": "local/b", "corpus": "local", "verdict": "pass", "why": ""}, + {"id": "stress/c", "corpus": "stress", "verdict": "fail", "why": "printed 3"}, + ]} + self.assertEqual([r["id"] for r in verify.strict_failures(doc)], ["stress/c"]) + self.assertEqual([r["id"] for r in verify.strict_failures(doc, verify.CORPORA_ALL)], ["manual/a", "stress/c"]) + # the committed goldens: everything we wrote passes or was observed + committed = verify.status(load_pin(), write=False) + self.assertEqual(verify.strict_failures(committed), []) + + def test_generated_typescript_matches_the_pin_and_corpus(self): + with tempfile.TemporaryDirectory() as d: + emit_ts.emit(load_pin(), Path(d), syntax_dir=Path(d)) + for f in ("pin.ts", "tokens.ts", "editor.ts", "programs.ts", "stdlib.ts", "types.ts"): + committed = (common.GENERATED_TS / f).read_text(encoding="utf-8") + self.assertEqual((Path(d) / f).read_text(encoding="utf-8"), committed, + f"src/mojo/generated/{f} is stale; run `python3 scripts/mojo/corpus.py emit`") + self.assertEqual((Path(d) / "mojo.xml").read_text(encoding="utf-8"), (emit_ts.SYNTAX_DIR / "mojo.xml").read_text(encoding="utf-8"), + "library/syntax/mojo.xml is stale; run `python3 scripts/mojo/corpus.py emit`") + + def test_the_quarto_syntax_is_well_formed_with_the_pinned_words(self): + import xml.etree.ElementTree as ET + root = ET.parse(emit_ts.SYNTAX_DIR / "mojo.xml").getroot() + lists = {l.get("name"): [i.text for i in l] for l in root.iter("list")} + self.assertIn("def", lists["keywords"]) + self.assertNotIn("fn", lists["keywords"]) + self.assertEqual(sorted(lists["removed"]), ["fn", "read"]) + self.assertTrue({"mut", "out", "deinit", "imm", "var", "ref"} <= set(lists["conventions"])) + names = {c.get("name") for c in root.iter("context")} + for c in root.iter("context"): + for rule in c: + ctx = rule.get("context") + if ctx and not ctx.startswith("#"): + self.assertIn(ctx, names, f"a rule in {c.get('name')} names a missing context {ctx}") + for kw in root.iter("keyword"): + self.assertIn(kw.get("String"), lists, f"keyword rule names a missing list {kw.get('String')}") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/new-exercise.mjs b/scripts/new-exercise.mjs new file mode 100644 index 0000000..4c9ef6a --- /dev/null +++ b/scripts/new-exercise.mjs @@ -0,0 +1,77 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Start a new exercise whose Mojo you write in a plain .mojo file. + * + * pixi run new-exercise my-thing + * node scripts/new-exercise.mjs my-thing --title "My thing" --group "Loop forms" # with a title and group + * + * If library/mojo/local/my-thing.mojo already exists it is kept as written and only the + * exercise is made, so you can start from a .mojo file. Otherwise it creates + * library/mojo/local/my-thing.mojo a runnable skeleton with the header verify.py reads + * src/exercises/my-thing.ts an exercise that names it (`mojoProgram`) + * and registers the exercise in src/exercises/index.ts (before the demo playground, + * which stays last). Then `pixi run play my-thing` runs it, checks it on godbolt, + * builds it into the app, and tests everything. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = fileURLToPath(new URL('..', import.meta.url)); +const argv = process.argv.slice(2); +const flag = (name) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : undefined; }; +const id = argv.find((a, i) => !a.startsWith('--') && !(i > 0 && argv[i - 1].startsWith('--'))); + +function die(msg) { console.error(`new: ${msg}`); process.exit(1); } + +if (!id) die('usage: pixi run new-exercise \n or: node scripts/new-exercise.mjs [--title "Title"] [--group "Home-page section"]\n is lowercase words joined by hyphens, e.g. rolling-sum'); +if (!/^[a-z0-9]+(-[a-z0-9]+)*$/.test(id)) die(`"${id}" is not a valid id: lowercase letters and digits, joined by single hyphens`); + +const mojoPath = path.join(ROOT, 'library/mojo/local', `${id}.mojo`); +const tsPath = path.join(ROOT, 'src/exercises', `${id}.ts`); +const indexPath = path.join(ROOT, 'src/exercises/index.ts'); +// A .mojo file you already wrote is adopted as it is: only the exercise is made. +const adopt = fs.existsSync(mojoPath); +if (fs.existsSync(tsPath)) die(`${path.relative(ROOT, tsPath)} already exists; nothing was changed`); + +const title = flag('title') ?? id.charAt(0).toUpperCase() + id.slice(1).replace(/-/g, ' '); +const group = flag('group') ?? 'Your own'; +const ident = (/^\d/.test(id) ? 'x-' : '') + id; +const camel = ident.replace(/-([a-z0-9])/g, (_, c) => c.toUpperCase()); + +let index = fs.readFileSync(indexPath, 'utf8'); +if (new RegExp(`\\bimport ${camel} from`).test(index)) die(`src/exercises/index.ts already imports something called ${camel}; pick another id`); +const anchorImport = "import blank from './demo.js';"; +const anchorList = ' blank,\n];'; +if (!index.includes(anchorImport) || !index.includes(anchorList)) die('src/exercises/index.ts no longer has the demo playground as its last entry, so I do not know where to register the exercise. Add the import and list entry by hand.'); + +if (!adopt) fs.writeFileSync(mojoPath, `# origin: Programming+ exercise ${id} +# status: unverified; \`corpus.py verify\` checks it on the pinned compiler +# probe: what this program shows, in one line (edit me) +# mode: both +# args: -O0 --debug-level=line-tables +# expect-stdout: Hello from ${id} +def main(): + print("Hello from ${id}") +`); + +fs.writeFileSync(tsPath, `import { exercise } from './types.js'; + +export default exercise({ + id: '${id}', + mojoProgram: '${id}', // library/mojo/local/${id}.mojo; \`pixi run play ${id}\` runs, verifies and builds it + title: ${JSON.stringify(title)}, + group: ${JSON.stringify(group)}, + blurb: 'One or two sentences: what the student does here and what they will notice.', + meta: { level: 'core' }, + sources: {}, + lang: 'mojo', +}); +`); + +index = index.replace(anchorImport, `import ${camel} from './${id}.js';\n${anchorImport}`).replace(anchorList, ` ${camel},\n blank,\n];`); +fs.writeFileSync(indexPath, index); + +console.log(`${adopt ? 'adopted ' : 'created '} library/mojo/local/${id}.mojo\ncreated src/exercises/${id}.ts\nupdated src/exercises/index.ts\n\nnext: pixi run play ${id}`); diff --git a/scripts/package.py b/scripts/package.py new file mode 100644 index 0000000..82dfadc --- /dev/null +++ b/scripts/package.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""Zip the repository for sharing: the files git tracks, plus new files it does not ignore. + + pixi run package # writes package/pplus-core-.zip + +Nothing git ignores goes in: no .DS_Store, __pycache__, scratch/, dist/, site/, node_modules/ or +.pixi/. Files are stored under a top folder named pplus-core/. +Standard library only, like the corpus pipeline. +""" +import pathlib +import subprocess +import sys +import zipfile + +ROOT = pathlib.Path(__file__).resolve().parent.parent + + +def git(*args: str) -> str: + return subprocess.run(["git", *args], cwd=ROOT, check=True, capture_output=True, text=True).stdout + + +def main() -> int: + try: + commit = git("rev-parse", "--short", "HEAD").strip() + except (subprocess.CalledProcessError, FileNotFoundError): + print("package: this needs a git checkout (git init and commit first)", file=sys.stderr) + return 1 + dirty = bool(git("status", "--porcelain").strip()) + names = sorted(set(git("ls-files", "-z", "--cached", "--others", "--exclude-standard").split("\0")) - {""}) + # pplus-library (library/) is a submodule: take the files its checkout tracks, so the zip tests on its own + if (ROOT / "library" / ".git").exists(): + tracked = subprocess.run(["git", "ls-files", "-z"], cwd=ROOT / "library", check=True, capture_output=True, text=True).stdout + names += [f"library/{n}" for n in tracked.split("\0") if n] + names = [n for n in names if (ROOT / n).is_file()] + out_dir = ROOT / "package" + out_dir.mkdir(exist_ok=True) + out = out_dir / f"pplus-core-{commit}{'-dirty' if dirty else ''}.zip" + with zipfile.ZipFile(out, "w", compression=zipfile.ZIP_DEFLATED) as z: + for n in names: + z.write(ROOT / n, f"pplus-core/{n}") + size = out.stat().st_size / 1e6 + print(f"wrote {out.relative_to(ROOT)}: {len(names)} files, {size:.1f} MB{' (with uncommitted changes)' if dirty else ''}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/play.mjs b/scripts/play.mjs new file mode 100644 index 0000000..05a8236 --- /dev/null +++ b/scripts/play.mjs @@ -0,0 +1,187 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The fast loop for a Mojo program in library/mojo/local/: write it, run one command, + * and have it run, be checked on the pinned compiler, be built into the app, + * and be tested. + * + * pixi run play demo # one pass over library/mojo/local/demo.mojo + * pixi run play-watch demo # again on every save (tests off unless --test) + * node scripts/play.mjs demo --no-verify # skip godbolt (offline, or a quick edit) + * node scripts/play.mjs demo --no-test # skip the tests + * + * One pass is: + * 1. run the file on your own compiler (`mojo`, or `pixi run mojo-run`), if you have one + * 2. verify ask godbolt (the pinned Mojo) about it: its verdict, output, and x86-64 listing [network] + * 3. status judge it from the goldens and rewrite src/mojo/generated/programs.ts [offline] + * 4. build tsc + * 5. listing turn godbolt's assembly into the x86-64 pane (scripts/mojo/listings.mjs), rebuild if it changed + * 6. test the whole test suite (npm test) + * + * A step that cannot run (no local compiler, godbolt unreachable) is reported as + * skipped and the pass goes on; a program that fails to compile locally, a + * build error, or a failing test stops it or fails the pass. + * Exercises that name the program in `mojoProgram` pick up the result; to make + * one, `pixi run new-exercise `. + */ +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const ROOT = fileURLToPath(new URL('..', import.meta.url)); +const LOCAL = path.join(ROOT, 'library/mojo/local'); +const PY = process.env.PYTHON || 'python3'; + +const argv = process.argv.slice(2); +const flags = new Set(argv.filter((a) => a.startsWith('--'))); +const names = argv.filter((a) => !a.startsWith('--')).map((a) => path.basename(a).replace(/\.mojo$/, '')); +const watch = flags.has('--watch'); +const doVerify = !flags.has('--no-verify'); +const doTest = !flags.has('--no-test') && !(watch && !flags.has('--test')); + +function sh(cmd, args, { quiet = false } = {}) { + const r = spawnSync(cmd, args, { cwd: ROOT, stdio: quiet ? ['ignore', 'pipe', 'pipe'] : ['ignore', 'inherit', 'inherit'], encoding: 'utf8' }); + return { ok: r.status === 0, missing: r.error?.code === 'ENOENT', status: r.status, out: (r.stdout || '') + (r.stderr || '') }; +} + +const available = (cmd) => sh(cmd, ['--version'], { quiet: true }).ok; + +/** Exercises whose `mojoProgram` has no file in library/mojo/local/. */ +function danglingPrograms() { + const dir = path.join(ROOT, 'src/exercises'); + const out = []; + for (const f of fs.readdirSync(dir).filter((x) => x.endsWith('.ts') && !x.startsWith('_'))) { + const m = /mojoProgram:\s*['"]([^'"]+)['"]/.exec(fs.readFileSync(path.join(dir, f), 'utf8')); + if (m && !fs.existsSync(path.join(LOCAL, `${m[1]}.mojo`))) out.push({ exercise: f.replace(/\.ts$/, ''), program: m[1] }); + } + return out; +} +/** Is there an exercise whose `mojoProgram` is this program? */ +function exerciseNaming(program) { + const dir = path.join(ROOT, 'src/exercises'); + return fs.readdirSync(dir).some((f) => f.endsWith('.ts') && new RegExp(`mojoProgram:\\s*['"]${program}['"]`).test(fs.readFileSync(path.join(dir, f), 'utf8'))); +} +const listingsFile = path.join(ROOT, 'src/mojo/generated/listings.ts'); +const read = (p) => (fs.existsSync(p) ? fs.readFileSync(p, 'utf8') : ''); + +async function pass(name) { + const file = path.join(LOCAL, `${name}.mojo`); + const rel = `library/mojo/local/${name}.mojo`; + const report = []; + const step = (what, status, note = '') => { report.push({ what, status, note }); }; + const heading = (s) => console.log(`\n▸ ${s}`); + + if (!fs.existsSync(file)) { + console.error(`play: ${rel} does not exist. Programs in library/mojo/local/: ${fs.readdirSync(LOCAL).filter((f) => f.endsWith('.mojo')).map((f) => f.replace(/\.mojo$/, '')).join(', ')}`); + return false; + } + + // 1. your own compiler + heading(`run ${rel} locally`); + // built with the pin's arguments and then the file's own, as godbolt is asked (corpus.py mojo-run) + const runner = available('mojo') ? [PY, ['scripts/mojo/corpus.py', 'mojo-run', rel]] : available('pixi') ? ['pixi', ['run', 'mojo-run', rel]] : null; + if (!runner) step('run locally', 'skipped', 'no `mojo` or `pixi` on the PATH; install pixi (https://pixi.sh) and the pinned compiler comes with `pixi install`'); + else { + const r = sh(runner[0], runner[1]); + if (!r.ok) { step('run locally', 'FAILED', `exit ${r.status}; fix the program first, nothing downstream ran`); return summarize(name, report); } + step('run locally', 'ok', `${runner[0]} ${runner[1].join(' ')}`); + } + + // a .mojo file with no exercise yet gets one (src/exercises/.ts, registered in index.ts) + if (!fs.existsSync(path.join(ROOT, 'src/exercises', `${name}.ts`)) && !exerciseNaming(name)) { + heading(`no exercise uses ${rel} yet: creating src/exercises/${name}.ts`); + const r = sh('node', ['scripts/new-exercise.mjs', name]); + if (!r.ok) { step('create exercise', 'FAILED', 'scripts/new-exercise.mjs refused; see above'); return summarize(name, report); } + step('create exercise', 'ok', `src/exercises/${name}.ts`); + } + + // verify and status both rewrite src/mojo/generated/programs.ts from library/mojo/local/. An exercise that names a + // program with no file there would then make the whole catalog throw at import (src/core/catalog.ts), so stop first. + const dangling = danglingPrograms(); + if (dangling.length) { + step('exercises name real programs', 'FAILED', dangling.map((d) => `${d.exercise} names library/mojo/local/${d.program}.mojo, which does not exist`).join('; ') + '. Create the file, or remove `mojoProgram` from the exercise, then run this again. Nothing was regenerated.'); + return summarize(name, report); + } + + // 2. godbolt + if (!doVerify) step('verify on godbolt', 'skipped', '--no-verify'); + else { + heading(`verify on godbolt (Mojo ${JSON.parse(fs.readFileSync(path.join(ROOT, 'library', 'mojo', 'pin.json'), 'utf8')).mojo.version})`); + const r = sh(PY, ['scripts/mojo/corpus.py', 'verify', '--item', `local/${name}`]); + step('verify on godbolt', r.ok ? 'ok' : 'skipped', r.ok ? '' : 'godbolt did not answer (offline, blocked, or rate-limited); the verdict and listing below are from the last answer on file, if any'); + } + + // 3. judge offline, refresh programs.ts + heading('status (offline)'); + const st = sh(PY, ['scripts/mojo/corpus.py', 'status']); + if (!st.ok) { step('status', 'FAILED', 'corpus.py status failed'); return summarize(name, report); } + step('status', 'ok'); + + // 4. build + heading('build'); + if (!sh('npx', ['tsc', '-p', 'tsconfig.json']).ok) { step('build', 'FAILED', 'tsc reported errors'); return summarize(name, report); } + step('build', 'ok'); + + // 5. the x86-64 listing + heading('x86-64 listing'); + const before = read(listingsFile); + const li = sh('node', ['scripts/mojo/listings.mjs', name]); + if (!li.ok) { step('x86-64 listing', 'FAILED', 'listings.mjs failed'); return summarize(name, report); } + if (read(listingsFile) !== before) { + if (!sh('npx', ['tsc', '-p', 'tsconfig.json']).ok) { step('x86-64 listing', 'FAILED', 'rebuild after the listing changed failed'); return summarize(name, report); } + step('x86-64 listing', 'ok', 'updated'); + } else step('x86-64 listing', 'ok', 'unchanged (see above for whether one exists)'); + + // what the app now says about the program + try { + const mod = await import(pathToFileURL(path.join(ROOT, 'dist/mojo/generated/programs.js')).href + `?t=${Date.now()}`); + const p = mod.MOJO_PROGRAMS[name]; + step('verdict in the app', p?.verified ? 'ok' : 'note', p ? `${p.verdict}: ${p.why}` : 'the program is not in programs.ts'); + } catch (e) { step('verdict in the app', 'note', String(e.message || e)); } + + // 6. tests + if (!doTest) step('tests', 'skipped', watch ? 'watch mode (add --test)' : '--no-test'); + else { + heading('npm test'); + const t = sh('npm', ['test']); + step('tests', t.ok ? 'ok' : 'FAILED', t.ok ? '' : 'see the failures above'); + } + return summarize(name, report); +} + +function summarize(name, report) { + console.log(`\n── ${name} ──`); + for (const r of report) console.log(` ${r.status.padEnd(7)} ${r.what}${r.note ? ` — ${r.note}` : ''}`); + return !report.some((r) => r.status === 'FAILED'); +} + +if (!names.length && !watch) { + console.error('usage: pixi run play (or node scripts/play.mjs [--watch] [--no-verify] [--no-test] [--test])\n is a program in library/mojo/local/ (without .mojo). To make a new one: pixi run new-exercise '); + process.exit(2); +} + +let failed = false; +for (const n of names) if (!(await pass(n))) failed = true; + +if (watch) { + const only = new Set(names); + console.log(`\nwatching library/mojo/local/${only.size ? ` for ${[...only].join(', ')}` : ''} (Ctrl+C to stop)`); + const queue = new Set(); + let busy = false, timer = null; + const drain = async () => { + if (busy) return; + busy = true; + while (queue.size) { const n = [...queue][0]; queue.delete(n); await pass(n); console.log(`\nwatching… (${n} done)`); } + busy = false; + }; + fs.watch(LOCAL, (_event, f) => { + if (!f || !f.endsWith('.mojo')) return; + const n = f.replace(/\.mojo$/, ''); + if (only.size && !only.has(n)) return; + queue.add(n); + clearTimeout(timer); + timer = setTimeout(drain, 500); // editors write a file in several events + }); +} else process.exit(failed ? 1 : 0); diff --git a/scripts/prove-fuzz.mjs b/scripts/prove-fuzz.mjs new file mode 100644 index 0000000..dc7ee2e --- /dev/null +++ b/scripts/prove-fuzz.mjs @@ -0,0 +1,467 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Fuzz the prover with programs nobody wrote: "proved" must be a proof (AGENTS.md, rule 5). + * + * Each program is a PROCEDURE P(A, B, L) made at random from the notation's loops, branches, calls and contracts, + * with helpers it may call. The prover is asked about every obligation; then P is run on random arguments with the + * contracts checked. A finding is an obligation the prover marked proved that a run breaks: a run that stops with + * the same kind of failure on the same line (the same column for an ASSERT, the same half of an INVARIANT or a + * DECREASES). Units the prover could not finish (partial) are not counted. A finding is a soundness bug in + * src/engines/pseudo/: write its program into library/pseudo/prover/ with a `// MUST-NOT-PROVE:` header first + * (docs/engines.md#soundness), then fix the rule. + * + * What the programs use (each a "feature" that --without can switch off, to see which one a finding needs): + * + * core INTEGER arithmetic (DIV, MOD, MIN, MAX, ABS), lists (APPEND, REMOVE, indexing, IN), IF, FOR with a + * literal STEP, WHILE, FOR EACH, REPEAT, BREAK and CONTINUE, INVARIANTs on FOR and WHILE, ASSERTs, + * calls with REQUIRE, ENSURE, BYREF and OLD, calls on the right of AND and OR + * foreach an INVARIANT in FOR EACH repeat an INVARIANT in REPEAT + * step a FOR whose STEP is an expression case CASE OF with values, ranges and OTHERWISE + * float FLOAT and FLOAT32 variables, conversions and comparisons + * sized INT8, UINT8 and INT16 variables and their arithmetic (a result out of range stops the run) + * impure FUNCTIONs that draw a random number (RANDINT, RAND) or read INPUT, called more than once + * + * node scripts/prove-fuzz.mjs [--seed N] [--programs N] [--runs N] [--budget-ms N] [--time S] + * [--without f1,f2] [--only-program N] [--keep DIR] [--no-reduce] [--dist DIR] [--json] + * + * --seed the programs come from this seed and the program's number, so a finding is reproduced by + * --seed S --only-program N (default 1) + * --programs how many programs (default 300) --runs runs of each program (default 24) + * --budget-ms the prover's time for each routine, as tests/prover-soundness.test.mjs gives it a budget + * (default 1500); a slower machine may prove less, never more + * --time stop making programs after this many seconds and report what was done (default: no limit) + * --keep DIR write each finding, reduced, as a regression-ready .pseudo file in DIR + * --no-reduce report each finding as generated instead of the smallest program that still shows it + * --dist DIR test another build of core than dist/ (an older one, to see that the fuzzer finds its known bugs) + * + * Exit status: 0 when nothing was found, 1 when a proved obligation was broken, 2 on a usage error or a missing + * build (dist/). The same seed and options give the same programs and the same runs. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const ROOT = fileURLToPath(new URL('..', import.meta.url)); +const argv = process.argv.slice(2); +const opt = (name, dflt = null) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : dflt; }; +const flag = (name) => argv.includes(`--${name}`); +const DIST = path.resolve(opt('dist', path.join(ROOT, 'dist'))); +const dist = (p) => pathToFileURL(path.join(DIST, p)).href; +const num = (name, dflt) => { + const v = opt(name); + if (v === null) return dflt; + const n = Number(v); + if (!Number.isInteger(n) || n < 0) { console.error(`prove-fuzz: --${name} wants a whole number, and got ${v}`); process.exit(2); } + return n; +}; +if (!fs.existsSync(path.join(DIST, 'engines', 'pseudo', 'vcgen.js'))) { console.error(`prove-fuzz: no build at ${DIST}; run \`pixi run build\` first`); process.exit(2); } + +const { studyOf } = await import(dist('services/pseudo.js')); +const { verify } = await import(dist('engines/pseudo/vcgen.js')); +const { run, evaluateConstant, valueFits } = await import(dist('engines/pseudo/interp.js')); +const { parseExpression } = await import(dist('engines/pseudo/parse.js')); + +export const FEATURES = ['core', 'foreach', 'repeat', 'step', 'case', 'float', 'sized', 'impure']; +const SEED = num('seed', 1); +const PROGRAMS = Math.min(num('programs', 300), 100000); +const RUNS = Math.min(num('runs', 24), 1000); +const BUDGET_MS = num('budget-ms', 1500); +const TIME_S = num('time', 0); +const ONLY = opt('only-program') === null ? null : num('only-program', 0); +const KEEP = opt('keep'); +const without = new Set((opt('without') || '').split(',').filter(Boolean)); +for (const f of without) if (!FEATURES.includes(f) || f === 'core') { console.error(`prove-fuzz: --without takes ${FEATURES.slice(1).join(', ')}; got ${f}`); process.exit(2); } +const on = (f) => !without.has(f); + +/** The verifier's obligation kinds against the runtime error kinds a run reports (as tests/prover-soundness.test.mjs). */ +const RUNTIME_KIND = { + index: 'index', slice: 'index', key: 'key', divide: 'divide', argument: 'value', + require: 'require', ensure: 'ensure', assert: 'assert', + 'invariant-init': 'invariant', 'invariant-keep': 'invariant', + 'decreases-bound': 'decreases', 'decreases-step': 'decreases', +}; +const PHASE = { 'invariant-init': 'init', 'invariant-keep': 'keep', 'decreases-bound': 'bound', 'decreases-step': 'step' }; + +/* ------------------------------------------------------------------ */ +/* Random numbers: one stream per program, from the seed and its number */ +/* ------------------------------------------------------------------ */ + +function mulberry(seed) { + let s = seed >>> 0; + return () => { s = (s + 0x6D2B79F5) >>> 0; let t = s; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; }; +} +const mix = (a, b) => (Math.imul(a ^ 0x9E3779B9, 0x85EBCA6B) ^ Math.imul(b + 0x632BE59B, 0xC2B2AE35)) >>> 0; + +let rnd = mulberry(1); +const ri = (lo, hi) => lo + Math.floor(rnd() * (hi - lo + 1)); +const pick = (xs) => xs[Math.floor(rnd() * xs.length)]; +const chance = (p) => rnd() < p; + +/* ------------------------------------------------------------------ */ +/* Programs */ +/* ------------------------------------------------------------------ */ + +const HELPERS = { + core: `FUNCTION Twice(N : INTEGER) RETURNS INTEGER + ENSURE RESULT == 2 * N + RETURN N + N +ENDFUNCTION + +PROCEDURE Bump(BYREF V : INTEGER, D : INTEGER) + REQUIRE D >= 0 + ENSURE V >= OLD(V) AND V == OLD(V) + D + V <- V + D +ENDPROCEDURE + +FUNCTION Last(Q : LIST OF INTEGER) RETURNS INTEGER + REQUIRE LENGTH(Q) > 0 + ENSURE RESULT == Q[LENGTH(Q) - 1] + RETURN Q[LENGTH(Q) - 1] +ENDFUNCTION +`, + impure: ` +FUNCTION Roll() RETURNS INTEGER + ENSURE RESULT >= 0 AND RESULT < 4 + RETURN RANDINT(0, 4) +ENDFUNCTION + +FUNCTION Draw() RETURNS FLOAT + RETURN RAND(1.0) +ENDFUNCTION + +FUNCTION ReadOne() RETURNS INTEGER + DECLARE V : INTEGER + INPUT V + RETURN V +ENDFUNCTION +`, +}; + +/** The state of the program being made: loop counters in scope, how deep in loops, names used. */ +let g; + +function intLeaf() { + const xs = ['A', 'B', 'X', 'Y', 'LENGTH(L)', 'LENGTH(M)', ...g.counters, String(ri(-3, 5)), String(ri(0, 3))]; + if (on('sized') && chance(0.15)) xs.push('S8'); // one sized type in INTEGER expressions: two sized types do not mix + return pick(xs); +} +function intE(d) { + if (d <= 0 || chance(0.35)) return intLeaf(); + const r = rnd(); + if (r < 0.22) return `${intE(d - 1)} + ${intE(d - 1)}`; + if (r < 0.36) return `${intE(d - 1)} - ${intE(d - 1)}`; + if (r < 0.46) return `${ri(-2, 3)} * (${intE(d - 1)})`; + if (r < 0.54) return `(${intE(d - 1)}) DIV ${ri(2, 3)}`; + if (r < 0.62) return `(${intE(d - 1)}) MOD ${ri(2, 4)}`; + if (r < 0.68) return `MIN(${intE(d - 1)}, ${intE(d - 1)})`; + if (r < 0.74) return `MAX(${intE(d - 1)}, ${intE(d - 1)})`; + if (r < 0.78) return `ABS(${intE(d - 1)})`; + if (r < 0.83) return `Twice(${intE(d - 1)})`; + if (r < 0.88) return `${pick(['L', 'M'])}[${intE(d - 1)}]`; + if (r < 0.91) return `(${intE(d - 1)}) DIV (${intE(d - 1)})`; + if (on('impure') && r < 0.95) return pick(['Roll()', 'Roll()', 'ReadOne()']); + if (on('float') && r < 0.98) return `${pick(['FLOOR', 'CEIL', 'ROUND', 'INT'])}(${floatE(1)})`; + return intLeaf(); +} +function floatE(d) { + const leaves = ['F', 'F', 'FLOAT64(G)', '0.5', '2.0', '1.5', 'A * 1.0', 'X / 2', '0.0']; + if (on('impure')) leaves.push('Draw()'); + if (d <= 0 || chance(0.3)) return pick(leaves); + const r = rnd(); + if (r < 0.2) return `${floatE(d - 1)} + ${floatE(d - 1)}`; + if (r < 0.35) return `${floatE(d - 1)} - ${floatE(d - 1)}`; + if (r < 0.5) return `${floatE(d - 1)} * ${pick(['0.5', '2.0', '-1.0', '0.1'])}`; + if (r < 0.62) return `${floatE(d - 1)} / ${pick(['2.0', '4.0', '3.0'])}`; + if (r < 0.72) return `ABS(${floatE(d - 1)})`; + if (r < 0.80) return `MIN(${floatE(d - 1)}, ${floatE(d - 1)})`; + if (r < 0.88) return `MAX(${floatE(d - 1)}, ${pick([floatE(d - 1), 'X', 'A'])})`; + return `SQRT(ABS(${floatE(d - 1)}))`; +} +function cond(d) { + const r = rnd(); + if (d <= 0 || r < 0.55) return `${intE(1)} ${pick(['==', '!=', '<', '<=', '>', '>='])} ${intE(1)}`; + if (r < 0.68) return `${cond(d - 1)} AND ${cond(d - 1)}`; + if (r < 0.80) return `${cond(d - 1)} OR ${cond(d - 1)}`; + if (r < 0.86) return `NOT (${cond(d - 1)})`; + // a call on the right of AND or OR: the run calls it only when the left side lets it + if (r < 0.93) return pick([`LENGTH(M) > 0 AND Last(M) ${pick(['>=', '<', '=='])} ${intE(1)}`, `LENGTH(M) == 0 OR Last(M) ${pick(['>=', '<'])} ${intE(1)}`, `${intE(1)} > 0 AND Twice(${intE(1)}) > ${intE(1)}`]); + return `${intE(1)} IN ${pick(['L', 'M'])}`; +} +function claim(exclude = null) { + const cs = g.counters.filter((c) => c !== exclude); + const c = cs.length ? pick(cs) : 'X'; + const forms = [ + `${c} >= 0`, `${c} < LENGTH(L)`, 'LENGTH(M) >= 0', `LENGTH(M) == ${c}`, `LENGTH(M) <= ${c}`, `X >= ${ri(-2, 2)}`, 'X <= Y', 'X == Y', 'Y >= A', + 'X >= A', `${c} <= B`, `${c} < B`, 'X MOD 2 == 0', 'X DIV 2 <= X', 'Y == 2 * X', 'LENGTH(M) > 0', 'X + Y >= 0', `${c} >= A`, `${c} <= A`, '1 == 2', + 'FORALL K IN [0, LENGTH(M)) : M[K] >= 0', 'X IN M', 'LENGTH(M) == LENGTH(L)', 'LENGTH(M) <= LENGTH(L)', `X == A + ${c}`, + ]; + if (on('float')) forms.push('F >= 0.0', 'F <= 1.5', 'G == FLOAT32(F)', 'FLOAT64(G) == F', 'FLOAT64(G) <= F', 'F >= X', 'ABS(F) >= 0.0', 'FLOOR(F) <= CEIL(F)', 'G * 2.0 == G + G', 'F == A', 'G == X', 'G <= 1.5', 'G + 1.0 > G'); + if (on('sized')) forms.push('S8 <= 127', 'S8 >= 0', 'S8 <= 100', 'U8 >= 0', 'U8 <= 250', 'X == S8', 'H16 >= INT16(S8)', 'S8 + 1 > S8', 'U8 - 1 < U8', 'H16 == X'); + if (on('impure')) forms.push('Roll() == Roll()', 'Roll() < 4', 'Roll() >= X', 'Draw() < 1.0', 'ReadOne() == ReadOne()'); + return chance(0.75) ? pick(forms) : cond(1); +} +function block(n, ind, depth) { + const out = []; + for (let i = 0; i < n; i++) out.push(...stmt(ind, depth)); + return out; +} +function featureStmt(p) { + const fs_ = FEATURES.slice(1).filter((f) => on(f) && ['float', 'sized', 'case', 'impure'].includes(f)); + if (!fs_.length) return null; + const f = pick(fs_); + if (f === 'float') { + const r = rnd(); + if (r < 0.45) return [`${p}F <- ${floatE(2)}`]; + if (r < 0.75) return [`${p}G <- ${pick([`FLOAT32(${floatE(1)})`, 'FLOAT32(X)', 'G * 2.0', 'G + 0.1', 'MAX(G, FLOAT32(X))', 'FLOAT32(F)', 'G / 3.0'])}`]; + return [`${p}X <- ${pick(['FLOOR(F)', 'CEIL(F)', 'ROUND(F)', 'INT(F)', 'INT(G)', 'ROUND(G)'])}`]; + } + if (f === 'sized') { + const r = rnd(); + if (r < 0.25) return [`${p}S8 <- S8 ${pick(['+', '-'])} ${ri(1, 60)}`]; + if (r < 0.45) return [`${p}S8 <- ${pick(['X', 'A', 'INT8(X)', 'LENGTH(L)', 'S8 * 2', '-S8', 'ABS(S8)', 'MAX(S8, X)', 'INT8(U8)'])}`]; + if (r < 0.65) return [`${p}U8 <- ${pick(['U8 + 1', 'U8 - 1', 'X', 'UINT8(ABS(S8))', 'U8 * 2', 'UINT8(A)', 'LENGTH(M)'])}`]; + if (r < 0.80) return [`${p}H16 <- ${pick(['H16 + INT16(S8)', 'INT16(S8) * INT16(S8)', 'H16 * 100', 'X * 1000', 'INT16(U8) + INT16(S8)'])}`]; + return [`${p}X <- ${pick(['S8', 'U8 + 0', 'H16 DIV 2', 'U8', 'X + S8'])}`]; + } + if (f === 'case') { + const v = pick(['X', 'Y', 'A', 'B MOD 3']); + const arms = [`${p}CASE OF ${v}`, `${p} ${ri(-1, 1)} : ${pick(['X', 'Y'])} <- ${intE(1)}`]; + if (chance(0.6)) arms.push(`${p} ${ri(2, 3)} TO ${ri(3, 6)} : ${pick(['X', 'Y'])} <- ${intE(1)}`); + if (chance(0.3)) arms.push(`${p} ${ri(7, 8)}, Twice(${pick(['A', 'B'])}) : ${pick(['X', 'Y'])} <- ${intE(1)}`); + if (chance(0.7)) arms.push(`${p} OTHERWISE : ${pick(['X', 'Y'])} <- ${intE(1)}`); + arms.push(`${p}ENDCASE`); + return arms; + } + // impure + return [`${p}${pick(['X', 'Y'])} <- ${pick(['Roll()', 'Roll() + Roll()', 'ReadOne()', 'Roll() - X', 'ReadOne() + Roll()'])}`]; +} +function stmt(ind, depth) { + const p = ' '.repeat(ind); + if (chance(0.3)) { const f = featureStmt(p); if (f) return f; } + const r = rnd(); + if (r < 0.18) return [`${p}${pick(['X', 'Y'])} <- ${intE(2)}`]; + if (r < 0.27) return [`${p}APPEND(M, ${intE(1)})`]; + if (r < 0.31) return [`${p}IF LENGTH(M) > 0 THEN`, `${p} REMOVE(M, ${pick(['0', 'LENGTH(M) - 1'])})`, `${p}ENDIF`]; + if (r < 0.34) return [`${p}CALL Bump(${pick(['X', 'Y'])}, ${chance(0.7) ? ri(0, 3) : intE(1)})`]; + if (r < 0.36) return [`${p}IF LENGTH(M) > 0 THEN`, `${p} X <- Last(M)`, `${p}ENDIF`]; + if (r < 0.52) return [`${p}ASSERT ${claim()}`]; + if (g.loopDepth > 0 && r < 0.56) return [`${p}IF ${cond(1)} THEN`, `${p} ${pick(['BREAK', 'CONTINUE'])}`, `${p}ENDIF`]; + if (depth <= 0) return [`${p}${pick(['X', 'Y'])} <- ${intE(1)}`]; + if (r < 0.67) { + const o = [`${p}IF ${cond(1)} THEN`, ...block(ri(1, 2), ind + 4, depth - 1)]; + if (chance(0.5)) o.push(`${p}ELSE`, ...block(ri(1, 2), ind + 4, depth - 1)); + o.push(`${p}ENDIF`); + return o; + } + if (r < 0.80) { + const name = ['I', 'J', 'I2', 'J2'].find((c) => !g.counters.includes(c)); + if (!name) return []; + let step = pick([null, null, null, 2, 3, -1, -2]); + if (on('step') && chance(0.35)) step = pick(['1 + 1', 'MAX(1, A)', '(B MOD 3) + 1', 'ABS(A) + 1', '-(1 + 1)', '-MAX(1, B)', '0 - 1']); + const lo = pick(['0', '0', 'A', intE(1), '1']); const hi = pick(['LENGTH(L)', 'B', 'A', intE(1), String(ri(0, 6))]); + const down = typeof step === 'number' ? step < 0 : typeof step === 'string' && /^-|^0 - /.test(step); + const head = step === null ? `FOR ${name} <- ${lo} TO ${hi}` : down ? `FOR ${name} <- ${hi} TO ${lo} STEP ${step}` : `FOR ${name} <- ${lo} TO ${hi} STEP ${step}`; + g.counters.push(name); g.loopDepth++; + const inv = chance(0.5) ? [`${p} INVARIANT ${claim()}`] : []; + const body = block(ri(1, 3), ind + 4, depth - 1); + g.counters.pop(); g.loopDepth--; + return [`${p}${head}`, ...inv, ...body, `${p}NEXT ${name}`]; + } + if (r < 0.88) { + g.loopDepth++; + const bound = pick(['B', 'LENGTH(L)', String(ri(1, 5)), 'A']); + const inv = chance(0.5) ? [`${p} INVARIANT ${claim()}`] : []; + const body = block(ri(1, 2), ind + 4, depth - 1); + g.loopDepth--; + return [`${p}WHILE X < ${bound}`, ...inv, ...body, `${p} X <- X + ${ri(1, 2)}`, `${p}ENDWHILE`]; + } + if (r < 0.94) { + const name = `E${++g.eachN}`; + // a FOR EACH INVARIANT may not name the loop variable (the checker says so), so its claim leaves it out + const inv = on('foreach') && chance(0.6) ? [`${p} INVARIANT ${claim(name)}`] : []; + g.counters.push(name); g.loopDepth++; + const body = block(ri(1, 2), ind + 4, depth - 1); + g.counters.pop(); g.loopDepth--; + return [`${p}FOR EACH ${name} IN ${pick(['L', 'M', `[${intE(0)}, ${intE(0)})`])}`, ...inv, ...body, `${p}NEXT ${name}`]; + } + g.loopDepth++; + const rinv = on('repeat') && chance(0.6) ? [`${p} INVARIANT ${claim()}`] : []; + const body = block(ri(1, 2), ind + 4, depth - 1); + g.loopDepth--; + return [`${p}REPEAT`, ...rinv, ...body, `${p} Y <- Y + 1`, `${p}UNTIL Y >= ${pick(['B', '3', 'A', 'LENGTH(L)'])}`]; +} + +/** Program number n of the seed, as text. */ +function program(n) { + rnd = mulberry(mix(SEED, n)); + g = { counters: [], eachN: 0, loopDepth: 0 }; + const req = chance(0.5) ? [` REQUIRE ${pick(['A >= 0', 'B >= A', 'LENGTH(L) > 0', 'A >= 0 AND B >= 0', 'B > 0'])}`] : []; + const decl = [' DECLARE X : INTEGER', ' DECLARE Y : INTEGER', ' DECLARE M : LIST OF INTEGER']; + const init = [` X <- ${pick(['0', 'A', '1', 'B'])}`, ` Y <- ${pick(['0', 'A', 'X', '2 * X'])}`]; + if (on('float')) { decl.push(' DECLARE F : FLOAT', ' DECLARE G : FLOAT32'); init.push(` F <- ${pick(['1.5', '0.0', 'A * 1.0', '0.5', '0.1'])}`, ` G <- ${pick(['FLOAT32(F)', '0.0', '1.0'])}`); } + if (on('sized')) { decl.push(' DECLARE S8 : INT8', ' DECLARE U8 : UINT8', ' DECLARE H16 : INT16'); init.push(' S8 <- 0', ` U8 <- ${ri(0, 3)}`, ' H16 <- 0'); } + const body = block(ri(2, 5), 4, 2); + const lines = ['PROCEDURE P(A : INTEGER, B : INTEGER, L : LIST OF INTEGER)', ...req, ...decl, ...init, ...body, ...(chance(0.5) ? [` ASSERT ${claim()}`] : []), 'ENDPROCEDURE']; + return `${HELPERS.core}${on('impure') ? HELPERS.impure : ''}\n${lines.join('\n')}\n`; +} + +/** The arguments and the INPUT lines of run t of program n, and the seed RANDINT and RAND draw from. */ +function runInputs(n, t) { + rnd = mulberry(mix(mix(SEED, n), t + 0x51ED)); + const args = [ri(-3, 6), ri(-3, 6), Array.from({ length: ri(0, 4) }, () => ri(-2, 5))]; + const input = Array.from({ length: 8 }, () => String(ri(-2, 4))); + return { call: `P(${args[0]}, ${args[1]}, [${args[2].join(', ')}])`, input, seed: t + 1 }; +} + +/* ------------------------------------------------------------------ */ +/* Proving and running */ +/* ------------------------------------------------------------------ */ + +function resolve(program, text) { + const p = parseExpression(text); + const sub = program.subs.get(p.expr.name.toUpperCase()); + const args = p.expr.args.map((a, i) => { const v = evaluateConstant(program, a); if (!valueFits(sub.params[i].type, v)) throw new Error(`${text}: argument ${i + 1} does not fit`); return v; }); + return { name: sub.name, args }; +} + +/** + * Prove a program, then run it as runs() says; return what was proved and the first finding, if any. + * @param {string} src the program + * @param {{ call: string, input: string[], seed: number }[]} runs the runs to make + */ +export function examine(src, runs, budgetMs = BUDGET_MS) { + const st = studyOf(src); + if (!st.ok) return { parsed: false }; + const program = st.program.program; + let rep; + try { rep = verify(program, { budgetMs }); } catch (e) { return { parsed: true, crashed: String(e?.message || e) }; } + const all = rep.units.filter((u) => !u.partial).flatMap((u) => u.obligations); + const proved = all.filter((o) => o.status === 'proved'); + const out = { parsed: true, proved: proved.length, partial: rep.units.filter((u) => u.partial).length, runs: 0, stopped: 0, rejected: 0, finding: null }; + if (!proved.length) return out; + const pReq = program.subs.get('P')?.requires?.map((r) => r.line) ?? []; + for (const r of runs) { + let tr; + try { tr = run(program, { call: resolve(program, r.call), input: r.input, maxSteps: 3000, contracts: true, seed: r.seed, record: false }); } catch { continue; } + out.runs++; + const e = tr.error; + if (!e) continue; + if (e.kind === 'require' && pReq.includes(e.line)) { out.rejected++; continue; } // P's own REQUIRE turned the arguments away + out.stopped++; + // the obligations this stop is about: same kind and line, the same column for an ASSERT, the same phase for an + // INVARIANT or DECREASES; a finding only when every one of them is proved, so a proved neighbor is not blamed + const same = all.filter((o) => (RUNTIME_KIND[o.kind] || o.kind) === e.kind && o.line === e.line + && (e.kind !== 'assert' || e.col === undefined || o.col === e.col) + && (!e.phase || !PHASE[o.kind] || PHASE[o.kind] === e.phase)); + if (same.length && same.every((o) => o.status === 'proved')) { + out.finding = { obligation: { kind: same[0].kind, line: same[0].line, col: same[0].col, label: same[0].label }, run: r, stop: { kind: e.kind, line: e.line, phase: e.phase, message: String(e.message) } }; + break; + } + } + return out; +} + +/** The runs of program n. */ +const runsOf = (n) => Array.from({ length: RUNS }, (_, t) => runInputs(n, t)); + +/** + * The smallest program, line by line, that still shows a finding of the same kind under the same runs: drop one + * line (or a whole block with its body) at a time, keep the drop when a finding of that kind remains. + */ +function reduce(src, runs, kind) { + let lines = src.split('\n'); + const holds = (ls) => { const r = examine(ls.join('\n'), runs); return r.finding && r.finding.obligation.kind === kind ? r : null; }; + const indent = (l) => l.match(/^ */)[0].length; + const OPEN = /^\s*(IF|FOR|WHILE|REPEAT|CASE|FUNCTION|PROCEDURE)\b/; + const CLOSE = /^\s*(ENDIF|NEXT|ENDWHILE|UNTIL|ENDCASE|ENDFUNCTION|ENDPROCEDURE)\b/; + // the last line of the block line i opens (the line itself when it opens none) + const blockEnd = (ls, i) => { + if (!OPEN.test(ls[i])) return i; + for (let j = i + 1; j < ls.length; j++) { + if (indent(ls[j]) === indent(ls[i]) && CLOSE.test(ls[j])) return j; + if (ls[j].trim() && indent(ls[j]) < indent(ls[i])) break; + } + return i; + }; + let changed = true; let tries = 0; + while (changed && tries < 400) { + changed = false; + for (let i = lines.length - 1; i >= 0 && tries < 400; i--) { + if (!lines[i].trim() || /^\s*(ENDIF|NEXT|ENDWHILE|UNTIL|ENDCASE|ENDFUNCTION|ENDPROCEDURE|ELSE|PROCEDURE P)\b/.test(lines[i])) continue; + const j = blockEnd(lines, i); + const cand = [...lines.slice(0, i), ...lines.slice(j + 1)]; + tries++; + if (holds(cand)) { lines = cand; changed = true; } + } + } + const tidy = lines.join('\n').replace(/\n{3,}/g, '\n\n').replace(/^\n+/, ''); + const last = examine(tidy, runs); + return last.finding ? { src: tidy, result: last } : { src: lines.join('\n'), result: examine(lines.join('\n'), runs) }; +} + +/* ------------------------------------------------------------------ */ +/* The run */ +/* ------------------------------------------------------------------ */ + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const started = Date.now(); + const counts = { programs: 0, accepted: 0, withProved: 0, proved: 0, partial: 0, runs: 0, stopped: 0, rejected: 0, crashed: 0 }; + const findings = []; + const numbers = ONLY !== null ? [Number(ONLY)] : Array.from({ length: PROGRAMS }, (_, i) => i); + let timedOut = false; + for (const n of numbers) { + if (TIME_S && Date.now() - started > TIME_S * 1000) { timedOut = true; break; } + const src = program(n); + const runs = runsOf(n); + counts.programs++; + if (ONLY !== null && !flag('json')) console.log(src); + const r = examine(src, runs); + if (!r.parsed) continue; + counts.accepted++; + if (r.crashed) { counts.crashed++; findings.push({ n, crash: r.crashed, src }); continue; } + counts.partial += r.partial; + if (!r.proved) continue; + counts.withProved++; counts.proved += r.proved; counts.runs += r.runs; counts.stopped += r.stopped; counts.rejected += r.rejected; + if (r.finding) { + const red = flag('no-reduce') ? { src, result: r } : reduce(src, [r.finding.run], r.finding.obligation.kind); + findings.push({ n, ...(red.result.finding ?? r.finding), src: red.src }); + } + } + const seconds = ((Date.now() - started) / 1000).toFixed(0); + const features = FEATURES.filter(on).join(', '); + if (flag('json')) { + console.log(JSON.stringify({ seed: SEED, features: FEATURES.filter(on), budgetMs: BUDGET_MS, seconds: Number(seconds), timedOut, counts, findings }, null, 2)); + } else { + console.log(`prove-fuzz seed ${SEED}: ${counts.programs} programs (${features}) in ${seconds} s${timedOut ? ` (stopped at --time ${TIME_S})` : ''}`); + console.log(` ${counts.accepted} accepted by the checker, ${counts.withProved} with something proved: ${counts.proved} proved obligations`); + console.log(` ${counts.runs} runs, ${counts.stopped} stopped by a failed check, ${counts.rejected} turned away by P's REQUIRE; ${counts.partial} routines the prover did not finish (not counted)`); + for (const f of findings) { + console.log(''); + if (f.crash) { console.log(`PROVER CRASH in program ${f.n}: ${f.crash}`); console.log(f.src); continue; } + const o = f.obligation; + console.log(`FINDING in program ${f.n} (reproduce: node scripts/prove-fuzz.mjs --seed ${SEED} --only-program ${f.n}${without.size ? ` --without ${[...without].join(',')}` : ''})`); + console.log(` ${o.kind}@${o.line} "${o.label}" is proved, and ${f.run.call} (run seed ${f.run.seed}) stops at line ${f.stop.line}: ${f.stop.message.slice(0, 200)}`); + console.log(` the program${flag('no-reduce') ? '' : ', reduced'}:`); + console.log(f.src.split('\n').map((l) => ` ${l}`).join('\n')); + } + console.log(''); + console.log(findings.length ? `${findings.length} finding(s): a proved obligation a run breaks is a soundness bug (docs/engines.md#soundness)` : 'nothing found'); + } + if (KEEP && findings.length) { + fs.mkdirSync(KEEP, { recursive: true }); + for (const f of findings) { + if (f.crash) continue; + const head = [`// Found by prove-fuzz, seed ${SEED}, program ${f.n}: ${f.obligation.kind}@${f.obligation.line} is proved and a run breaks it.`, + `// CALL: ${f.run.call}`, ...f.run.input.map((l) => `// INPUT: ${l}`), ...(f.run.seed !== 1 ? [`// A random number this run draws comes from seed ${f.run.seed}; the soundness test runs a program with seed 1.`] : []), + ]; + // the header moves every line down by its own length: name the obligation where it will be + head.push(`// MUST-NOT-PROVE: ${f.obligation.kind}@${f.obligation.line + head.length + 1}`); + const file = path.join(KEEP, `prove-fuzz-${SEED}-${f.n}.pseudo`); + fs.writeFileSync(file, `${head.join('\n')}\n${f.src.trimEnd()}\n`); + console.log(`wrote ${file}`); + } + } + process.exit(findings.length ? 1 : 0); +} diff --git a/scripts/tests/test_dafny.py b/scripts/tests/test_dafny.py new file mode 100644 index 0000000..c57500c --- /dev/null +++ b/scripts/tests/test_dafny.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +# SPDX-License-Identifier: Apache-2.0 +"""The Dafny pin, offline: every pinned platform has a release and a hash, an unpinned one is refused, and unpacking +keeps the executable bits Dafny and Z3 need.""" +import os +import stat +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import dafny # noqa: E402 + + +class Pin(unittest.TestCase): + def test_every_pinned_platform_names_this_version_and_a_sha256(self): + self.assertTrue(dafny.RELEASES) + for (system, machine), (asset, digest) in dafny.RELEASES.items(): + self.assertIn(dafny.VERSION, asset, f"{system} {machine}") + self.assertRegex(digest, r"^[0-9a-f]{64}$", f"{system} {machine}") + + def test_the_two_platforms_pixi_pins_are_covered(self): + self.assertIn(("Linux", "x86_64"), dafny.RELEASES) + self.assertIn(("Darwin", "arm64"), dafny.RELEASES) + + def test_an_unpinned_platform_is_refused_by_name(self): + with self.assertRaises(SystemExit) as e: + dafny.release("Plan9", "mips") + self.assertIn("Plan9 mips", str(e.exception)) + + +class Unpack(unittest.TestCase): + def test_unpacking_keeps_executable_bits(self): + with tempfile.TemporaryDirectory() as d: + z = Path(d) / "a.zip" + with zipfile.ZipFile(z, "w") as w: + info = zipfile.ZipInfo("dafny/dafny") + info.external_attr = (0o755 | stat.S_IFREG) << 16 + w.writestr(info, "#!/bin/sh\n") + w.writestr("dafny/readme.txt", "text") + dafny.unpack(z, Path(d) / "out") + self.assertTrue(os.access(Path(d) / "out/dafny/dafny", os.X_OK)) + self.assertTrue((Path(d) / "out/dafny/readme.txt").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/common/fuzz.ts b/src/common/fuzz.ts new file mode 100644 index 0000000..7b7b7d2 --- /dev/null +++ b/src/common/fuzz.ts @@ -0,0 +1,482 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Generated-input testing, as a fuzzer that shows its work (the Tests view). For one routine it says exactly what + * it tries and what it checks: + * + * how the inputs are made edge cases in combination; every short DNA string (when a routine reads one); + * seeded random values; and mutations of the inputs that reached lines nothing else + * had reached (coverage-guided, as a fuzzer does) + * what each input must do the checks (`checks`): finish, raise no error, keep every contract, give the same + * answer twice + * what happened every case, with its strategy, input, outcome and steps; the lines the inputs + * reached and the ones none did; the answers it gave and how often; for each way of + * failing, the input that found it and every step of shrinking it to the smallest + * + * Seeded, so a report is the same on every run until the seed changes. DOM-free: the engines run the inputs + * (pseudo: src/engines/pseudo/interp.js; Mojo: src/mojo/interp.ts), and src/ui/views/tests.ts draws the report. + */ +import { run } from '../engines/pseudo/interp.js'; +import { formatValue, cloneValue, PArray, knownDims } from '../engines/pseudo/lang.js'; +import type { Module, Stmt } from '../mojo/parse.js'; +import { probeMojo } from '../mojo/interp.js'; +import { generator, literal, rng, typeText as mojoType, type Gen } from '../mojo/properties.js'; +import { show as showMojo, type V } from '../mojo/values.js'; + +export type Strategy = 'edge' | 'exhaustive' | 'random' | 'mutate' | 'shrink' | 'repeat'; +export type Outcome = 'pass' | 'fail' | 'skip' | 'raises'; + +export const STRATEGY: Record = { + edge: { label: 'edge cases', how: 'the values most likely to break code, for each parameter, in combination: zero, one and negative numbers, the empty string and list, a single element, repeated letters' }, + exhaustive: { label: 'every short string', how: 'every DNA string up to three letters long (85 of them), with each choice of the other parameter' }, + random: { label: 'random', how: 'values drawn at random from the seed, so the same seed always makes the same inputs' }, + mutate: { label: 'mutations', how: 'small changes (a letter added, removed or replaced; a number nudged, doubled or negated; an element dropped) to inputs that reached a line nothing earlier had reached' }, + shrink: { label: 'shrinking', how: 'after a failure, smaller and smaller inputs that still fail the same way, down to one that cannot be made smaller' }, + repeat: { label: 'repeats', how: 'inputs run a second time, to check the answer does not change' }, +}; + +export interface FuzzParam { name: string; type: string; how: string } +export interface FuzzCase { n: number; strategy: Strategy; call: string; args: string[]; outcome: Outcome; result?: string; message?: string; kind?: string; line?: number; steps: number; newLines?: number[] } +export interface FuzzFailure { kind: string; message: string; line?: number; call: string; found: string; shrink: string[]; n: number } +export interface FuzzReport { + name: string; + signature: string; + lang: 'pseudo' | 'mojo'; + params: FuzzParam[]; + seed: number; + budget: number; + /** What every input is checked for. */ + checks: string[]; + strategies: Array<{ id: Strategy; label: string; how: string; count: number }>; + /** Every case run, in order (shrinking and repeats included). */ + cases: FuzzCase[]; + failures: FuzzFailure[]; + counts: Record; + /** The routine's lines: how many inputs reached each, and how often it ran. */ + coverage: Array<{ line: number; inputs: number; hits: number }>; + /** The answers given, most common first. */ + results: Array<{ value: string; count: number }>; + /** Why the routine could not be fuzzed, or why fuzzing stopped. */ + problem?: string; + /** The steps (and, for pseudo, quantifier positions) all its runs took, against the work allowed (`FuzzOptions.work`). */ + work?: number; +} +/** + * seed: where the random inputs come from. budget: how many inputs (100, 300, 1000 or 3000). work: the steps (and + * for pseudo the positions FORALL, EXISTS and SUM visit) all the runs of one routine may take together, by default + * WORK_PER_INPUT times the budget: past it no more inputs are run, and the report says so in `problem`. + */ +export interface FuzzOptions { seed?: number; budget?: number; work?: number } + +/** + * Work for each input the budget asks for. Counted rather than timed, so a report is the same on every machine; + * the corpus's routines take at most about 1,300 an input, and 400,000 in all at the default budget of 300 + * (kmers' kmer_counts; docs/limits.md). + */ +export const WORK_PER_INPUT = 3000; + +export const BUDGETS = [100, 300, 1000, 3000] as const; + +/** + * One run. `steps` are its steps, as the report shows them; `work` what it took of the work allowed (the steps, and for + * pseudo the quantifier positions; `steps` when absent); `out` marks a run stopped by the work left (`left` in + * Driver.exec), not by its own limits. + */ +interface Exec { outcome: Outcome | 'declined'; result?: string; message?: string; kind?: string; line?: number; steps: number; work?: number; hits: Map; out?: boolean } + +interface Driver { + name: string; + signature: string; + lang: 'pseudo' | 'mojo'; + params: FuzzParam[]; + checks: string[]; + lines: number[]; + edges(): T[][]; + exhaustive(): T[][]; + random(r: () => number): T[]; + mutate(args: T[], r: () => number): T[]; + smaller(args: T[]): Iterable; + call(args: T[]): string; + arg(v: T, i: number): string; + /** Run one input, with at most `left` steps (and quantifier positions) of work. */ + exec(args: T[], left: number): Exec; +} + +/** The loop every language shares. */ +function fuzz(d: Driver, opts: FuzzOptions): FuzzReport { + const seed = opts.seed ?? 1; + const budget = opts.budget ?? 300; + const r = rng(seed * 2654435761); + const cases: FuzzCase[] = []; + const failures: FuzzFailure[] = []; + const counts: Record = { pass: 0, fail: 0, skip: 0, raises: 0 }; + const cover = new Map(); + const results = new Map(); + const corpus: T[][] = []; + const seen = new Set(); + const used: Record = { edge: 0, exhaustive: 0, random: 0, mutate: 0, shrink: 0, repeat: 0 }; + const failKeys = new Set(); + /** The arguments behind each call shown, for the repeat check. */ + const byCall = new Map(); + let problem: string | undefined; + const allowed = opts.work ?? budget * WORK_PER_INPUT; + let work = 0; + const spent = () => `Stopped after ${work.toLocaleString('en-US')} steps of work, all one routine's inputs may take here (${allowed.toLocaleString('en-US')}): the inputs listed were run, and no more.`; + const report = (): FuzzReport => ({ + name: d.name, signature: d.signature, lang: d.lang, params: d.params, seed, budget, checks: d.checks, + strategies: (Object.keys(STRATEGY) as Strategy[]).filter((s) => used[s]).map((s) => ({ id: s, ...STRATEGY[s], count: used[s] })), + cases, failures, counts, + coverage: d.lines.map((line) => ({ line, inputs: cover.get(line)?.inputs ?? 0, hits: cover.get(line)?.hits ?? 0 })), + results: [...results].sort((a, b) => b[1] - a[1]).map(([value, count]) => ({ value, count })), + ...(problem ? { problem } : {}), + work, + }); + + /** Run one input and keep everything about it. Returns null when the engine cannot run it at all. */ + let declined = 0; + const tryOne = (args: T[], strategy: Strategy): { c: FuzzCase; x: Exec } | null => { + if (work >= allowed) { problem = spent(); return null; } + const raw = d.exec(args, allowed - work); + work += raw.work ?? raw.steps; + if (raw.out) { problem = spent(); return null; } + // an input the interpreter cannot follow is listed as skipped, with why; when most are, fuzzing stops and says so + if (raw.outcome === 'declined' && ++declined >= 20 && declined > cases.length / 2) { problem = raw.message; return null; } + const x = (raw.outcome === 'declined' ? { ...raw, outcome: 'skip', kind: 'declined' } : raw) as Exec & { outcome: Outcome }; + used[strategy]++; + const fresh: number[] = []; + for (const [line, n] of x.hits) { + const e = cover.get(line); + if (!e) { if (d.lines.includes(line)) fresh.push(line); cover.set(line, { inputs: 1, hits: n }); } + else { e.inputs++; e.hits += n; } + } + const c: FuzzCase = { n: cases.length + 1, strategy, call: d.call(args), args: args.map((v, i) => d.arg(v, i)), outcome: x.outcome, steps: x.steps, + ...(x.result !== undefined ? { result: x.result } : {}), ...(x.message ? { message: x.message } : {}), ...(x.kind ? { kind: x.kind } : {}), ...(x.line ? { line: x.line } : {}), ...(fresh.length ? { newLines: fresh.sort((a, b) => a - b) } : {}) }; + cases.push(c); + byCall.set(c.call, args); + counts[x.outcome]++; + if (x.outcome === 'pass' && x.result !== undefined && strategy !== 'shrink' && strategy !== 'repeat') results.set(x.result, (results.get(x.result) ?? 0) + 1); + if (fresh.length && x.outcome !== 'skip') corpus.push(args); + return { c, x }; + }; + + /** A failure: shrink it, keeping every step, while the smaller input still fails the same way. */ + const shrink = (args: T[], first: { c: FuzzCase; x: Exec }) => { + const key = `${first.x.kind}@${first.x.line ?? ''}`; + if (failKeys.has(key)) return; + failKeys.add(key); + let best = args, bestC = first.c; + const path = [first.c.call]; + // the failure is kept however shrinking ends, the work running out included + shrinking: for (let rounds = 0; rounds < 60; rounds++) { + let improved = false; + for (const cand of d.smaller(best)) { + const k = d.call(cand); + if (seen.has(k)) continue; + seen.add(k); + const t = tryOne(cand, 'shrink'); + if (!t) break shrinking; + if (t.x.outcome === 'fail' && t.x.kind === first.x.kind) { best = cand; bestC = t.c; path.push(t.c.call); improved = true; break; } + } + if (!improved) break; + } + failures.push({ kind: first.x.kind ?? 'error', message: bestC.message ?? first.c.message ?? '', line: bestC.line, call: bestC.call, found: first.c.call, shrink: path, n: first.c.n }); + }; + + const consider = (args: T[], strategy: Strategy): boolean => { + const k = d.call(args); + if (seen.has(k)) return true; + seen.add(k); + const t = tryOne(args, strategy); + if (!t) return false; + if (t.x.outcome === 'fail' && failures.length < 5) shrink(args, t); + return true; + }; + + const room = () => cases.length < budget && !problem; + // 1. edge cases, in combination (a quarter of the budget at most) + for (const a of d.edges()) { if (!room() || used.edge >= budget / 4) break; if (!consider(a, 'edge')) return report(); } + // 2. every short string, when the routine reads one + for (const a of d.exhaustive()) { if (!room() || used.exhaustive >= budget * 0.3) break; if (!consider(a, 'exhaustive')) return report(); } + // 3. random values, and mutations of the inputs that found new lines + let misses = 0; + while (room() && misses < budget * 4) { + const mutate = corpus.length > 0 && r() < 0.4; + const args = mutate ? d.mutate(corpus[Math.floor(r() * corpus.length)], r) : d.random(r); + const before = cases.length; + if (!consider(args, mutate ? 'mutate' : 'random')) return report(); + if (cases.length === before) misses++; + } + // 4. the same answer twice: a few passing inputs again + const again = cases.filter((c) => c.outcome === 'pass' && c.strategy !== 'shrink').slice(0, 8); + for (const c of again) { + const args = findArgs(c); + if (!args || problem) continue; + if (work >= allowed) { problem = spent(); break; } + const x = d.exec(args, allowed - work); + work += x.work ?? x.steps; + if (x.out) { problem = spent(); break; } + used.repeat++; + const same = x.outcome === 'pass' && x.result === c.result; + const rc: FuzzCase = { n: cases.length + 1, strategy: 'repeat', call: c.call, args: c.args, outcome: same ? 'pass' : 'fail', steps: x.steps, ...(x.result !== undefined ? { result: x.result } : {}) }; + if (!same) { rc.kind = 'nondeterministic'; rc.message = `gave ${c.result} the first time and ${x.result ?? x.message} the second`; failures.push({ kind: 'nondeterministic', message: rc.message, call: c.call, found: c.call, shrink: [c.call], n: rc.n }); } + cases.push(rc); + counts[rc.outcome]++; + } + return report(); + + function findArgs(c: FuzzCase): T[] | null { return byCall.get(c.call) ?? null; } +} + +/* ------------------------------------------------------------------ */ +/* pseudo */ +/* ------------------------------------------------------------------ */ + +const DNA = ['A', 'C', 'G', 'T']; + +interface PGen { edges: any[]; random(r: () => number): any; mutate(v: any, r: () => number): any; smaller(v: any): Iterable; how: string } + +function pseudoGen(t: any): PGen | null { + const pick = (xs: X[], r: () => number) => xs[Math.floor(r() * xs.length)]; + const int = (lo: number, hi: number, r: () => number) => lo + Math.floor(r() * (hi - lo + 1)); + switch (t?.k) { + case 'INTEGER': return { + edges: [0, 1, -1, 2, 10, -10, 100], how: 'whole numbers: 0, 1, -1, 2, 10, -10, 100 first, then mostly -3 to 12', + random: (r) => (r() < 0.3 ? pick([0, 1, 2, -1], r) : int(-3, 12, r)), + mutate: (v, r) => pick([v + 1, v - 1, v * 2, -v, 0], r), + *smaller(v) { if (v !== 0) { yield 0; yield v > 0 ? v - 1 : v + 1; if (Math.abs(v) > 2) yield Math.trunc(v / 2); } }, + }; + case 'FLOAT': return { + edges: [0, 1, -1, 0.5, -2.5, 100.25], how: 'decimals: 0, 1, -1, 0.5, -2.5, 100.25 first, then -2 to 10 in hundredths', + random: (r) => Math.round((r() * 12 - 2) * 100) / 100, mutate: (v, r) => pick([v * 2, -v, v + 0.5, 0], r), + *smaller(v) { if (v !== 0) { yield 0; yield Math.trunc(v); } }, + }; + case 'BOOL': return { edges: [false, true], how: 'TRUE and FALSE', random: (r) => r() < 0.5, mutate: (v) => !v, *smaller(v) { if (v) yield false; } }; + case 'CHAR': return { + edges: [...DNA, 'N'], how: 'a DNA letter (A, C, G, T), sometimes N', + random: (r) => (r() < 0.08 ? 'N' : pick(DNA, r)), mutate: (v, r) => pick([...DNA, 'N'].filter((c) => c !== v), r), + *smaller(v) { if (v !== 'A') yield 'A'; }, + }; + case 'STRING': return { + edges: ['', 'A', 'AA', 'AC', 'GATTACA', 'NNN', 'ACGTACGTAC'], how: 'DNA letters (A, C, G, T, sometimes N), 0 to 9 long: "", "A", "AA", "AC", "GATTACA", "NNN", "ACGTACGTAC" first', + random: (r) => { const n = r() < 0.12 ? int(0, 1, r) : int(0, 9, r); let s = ''; for (let i = 0; i < n; i++) s += r() < 0.06 ? 'N' : pick(DNA, r); return s; }, + mutate: (v, r) => { + const cs = [...v]; const i = int(0, cs.length, r); + const how = cs.length ? Math.floor(r() * 4) : 0; + if (how === 0) cs.splice(i, 0, pick(DNA, r)); else if (how === 1) cs.splice(Math.min(i, cs.length - 1), 1); else if (how === 2) cs[Math.min(i, cs.length - 1)] = pick(DNA, r); else cs.push(...cs.slice(0, 2)); + return cs.join(''); + }, + *smaller(v) { const cs = [...v]; for (let i = 0; i < cs.length; i++) yield cs.slice(0, i).concat(cs.slice(i + 1)).join(''); for (let i = 0; i < cs.length; i++) if (cs[i] !== 'A') yield cs.slice(0, i).concat(['A'], cs.slice(i + 1)).join(''); }, + }; + case 'LIST': case 'ARRAY': case 'SET': { + const el = pseudoGen(t.of); + if (!el) return null; + const make = (xs: any[]) => (t.k === 'ARRAY' ? Object.assign(new PArray(dims ?? [xs.length]), { data: xs }) : t.k === 'SET' ? new Set(xs) : xs); + const items = (v: any): any[] => (v instanceof PArray ? [...(v as any).data] : v instanceof Set ? [...v] : [...v]); + // an ARRAY declared with plain-number lengths gets exactly that many values (the run stops on any other), so + // only its values change, never how many there are + const dims: number[] | null = knownDims(t); + if (dims) { + const n = dims.reduce((a, b) => a * b, 1); + const fill = (f: (i: number) => any) => make(Array.from({ length: n }, (_, i) => f(i))); + return { + edges: el.edges.slice(0, 3).map((e: any) => fill(() => e)), + how: `arrays of ${dims.join(' by ')} values, each ${el.how.split(':')[0].split(',')[0]}`, + random: (r) => fill(() => el.random(r)), + mutate: (v, r) => { const xs = items(v); if (!xs.length) return make(xs); const i = int(0, xs.length - 1, r); xs[i] = el.mutate(xs[i], r); return make(xs); }, + *smaller(v) { const xs = items(v); for (let i = 0; i < xs.length; i++) for (const s of el.smaller(xs[i])) { const c = xs.slice(); c[i] = s; yield make(c); } }, + }; + } + return { + edges: [make([]), make([el.edges[0]]), make([el.edges[0], el.edges[0]]), make(el.edges.slice(0, 3))], + how: `${t.k === 'SET' ? 'sets' : t.k === 'ARRAY' ? 'arrays' : 'lists'} of 0 to 6 elements, each ${el.how.split(':')[0].split(',')[0]}`, + random: (r) => make(Array.from({ length: int(0, 6, r) }, () => el.random(r))), + mutate: (v, r) => { const xs = items(v); const i = int(0, Math.max(0, xs.length - 1), r); const how = xs.length ? Math.floor(r() * 3) : 0; if (how === 0) xs.splice(i, 0, el.random(r)); else if (how === 1) xs.splice(i, 1); else xs[i] = el.mutate(xs[i], r); return make(xs); }, + *smaller(v) { const xs = items(v); for (let i = 0; i < xs.length; i++) yield make(xs.slice(0, i).concat(xs.slice(i + 1))); for (let i = 0; i < xs.length; i++) for (const s of el.smaller(xs[i])) { const c = xs.slice(); c[i] = s; yield make(c); } }, + }; + } + default: return null; + } +} + +/** Every line a routine's statements are on (its coverage denominator). */ +function pseudoLines(node: any): number[] { + const out = new Set(); + const seen = new WeakSet(); + const walk = (x: any, depth: number) => { + if (!x || typeof x !== 'object' || depth > 60 || seen.has(x)) return; + seen.add(x); + if (Array.isArray(x)) { for (const y of x) walk(y, depth + 1); return; } + if (typeof x.line === 'number') out.add(x.line); + for (const [k, v] of Object.entries(x)) if (k !== 'type' && k !== 'scope' && k !== 'symbols' && v && typeof v === 'object') walk(v, depth + 1); + }; + walk(node?.body ?? [], 0); + return [...out].sort((a, b) => a - b); +} + +export function fuzzPseudo(program: any, name: string, opts: FuzzOptions = {}): FuzzReport { + const sub = program.subs.get(String(name).toUpperCase()); + const empty = (problem: string): FuzzReport => ({ name, signature: name, lang: 'pseudo', params: [], seed: opts.seed ?? 1, budget: opts.budget ?? 300, checks: [], strategies: [], cases: [], failures: [], counts: { pass: 0, fail: 0, skip: 0, raises: 0 }, coverage: [], results: [], problem }); + if (!sub) return empty(`No routine called ${name}.`); + const typeWord = (t: any): string => (!t ? '' : t.k === 'LIST' || t.k === 'SET' ? `${t.k} OF ${typeWord(t.of)}` : t.k === 'ARRAY' ? `ARRAY OF ${typeWord(t.of)}` : t.k); + const signature = `${sub.name}(${sub.params.map((p: any) => `${p.name} : ${typeWord(p.type)}`).join(', ')})${sub.returns ? ` RETURNS ${typeWord(sub.returns)}` : ''}`; + const gens = sub.params.map((p: any) => pseudoGen(p.type)); + const missing = sub.params.findIndex((_: any, i: number) => !gens[i]); + if (missing >= 0) return { ...empty(`No generator for ${sub.params[missing].name}'s type (${typeWord(sub.params[missing].type)}): whole numbers, decimals, booleans, characters, strings, and lists, arrays and sets of those are generated.`), signature }; + const g = gens as PGen[]; + const fmt = (v: any, t: any) => formatValue(v, t); + const contracts = (sub.requires?.length ? 1 : 0) + (sub.ensures?.length ? 1 : 0); + const checks = [ + 'It runs to the end without a runtime error: an index out of range, a value used before it is set, a division by zero, a value of the wrong type.', + sub.requires?.length ? `Inputs that break a REQUIRE (${sub.requires.length} written) are skipped, not failed: the caller promised never to pass them.` : 'There is no REQUIRE, so every input counts.', + sub.ensures?.length ? `Every ENSURE (${sub.ensures.length} written) holds when it returns, and every INVARIANT, DECREASES and ASSERT holds on the way.` : 'Every INVARIANT, DECREASES and ASSERT written in it holds on the way (it has no ENSURE to check the answer against).', + 'It finishes within 50,000 steps.', + 'Run again on the same input, it gives the same answer.', + ]; + void contracts; + const driver: Driver = { + name: sub.name, signature, lang: 'pseudo', checks, + params: sub.params.map((p: any, i: number) => ({ name: p.name, type: typeWord(p.type), how: g[i].how })), + lines: pseudoLines(sub.node), + edges() { + const out: any[][] = []; + const combine = (i: number, acc: any[]) => { if (out.length >= 400) return; if (i === g.length) { out.push(acc); return; } for (const v of g[i].edges) combine(i + 1, [...acc, v]); }; + combine(0, []); + return out; + }, + exhaustive() { + const strs = sub.params.map((p: any, i: number) => (p.type?.k === 'STRING' ? i : -1)).filter((i: number) => i >= 0); + if (strs.length !== 1 || sub.params.length > 2) return []; + const all: string[] = ['']; + let layer = ['']; + for (let n = 1; n <= 3; n++) { const next: string[] = []; for (const s of layer) for (const c of DNA) next.push(s + c); all.push(...next); layer = next; } + const other = sub.params.findIndex((_: any, i: number) => i !== strs[0]); + const others = other < 0 ? [null] : g[other].edges.slice(0, 5); + const out: any[][] = []; + for (const s of all) for (const o of others) { const a: any[] = []; a[strs[0]] = s; if (other >= 0) a[other] = o; out.push(a); } + return out; + }, + random: (r) => g.map((x) => x.random(r)), + mutate(args, r) { const i = Math.floor(r() * args.length); const a = args.slice(); if (a.length) a[i] = g[i].mutate(cloneValue(a[i]), r); return a; }, + *smaller(args) { for (let i = 0; i < args.length; i++) for (const s of g[i].smaller(args[i])) { const a = args.slice(); a[i] = s; yield a; } }, + call: (args) => `${sub.name}(${args.map((a, i) => fmt(a, sub.params[i].type)).join(', ')})`, + arg: (v, i) => fmt(v, sub.params[i].type), + exec(args, left) { + const maxSteps = Math.min(50000, left), maxVisits = Math.min(5_000_000, left); + const tr: any = run(program, { call: { name: sub.name, args: args.map(cloneValue) }, record: false, maxSteps, maxVisits }); + const hits: Map = tr.hits ?? new Map(); + const steps = tr.work?.steps ?? 0; + const work = steps + (tr.visits ?? 0); + // a run stopped by a limit lowered to the work left is cut short; one stopped by its own limits has failed + if (tr.error?.kind === 'limit' && ((maxSteps < 50000 && steps > maxSteps) || (maxVisits < 5_000_000 && tr.visits > maxVisits))) return { outcome: 'skip', steps, work, hits, out: true }; + if (tr.error?.kind === 'require') return { outcome: 'skip', message: tr.error.message, kind: 'require', line: tr.error.line, steps, work, hits }; + if (tr.error) return { outcome: 'fail', message: tr.error.message, kind: tr.error.kind, line: tr.error.line, steps, work, hits }; + return { outcome: 'pass', result: sub.returns ? fmt(tr.result, sub.returns) : tr.output?.length ? `prints ${tr.output.map((x: string) => JSON.stringify(x)).join(', ')}` : '(no answer)', steps, work, hits }; + }, + }; + return fuzz(driver, opts); +} + +/* ------------------------------------------------------------------ */ +/* Mojo */ +/* ------------------------------------------------------------------ */ + +type DefStmt = Extract; + +function mojoMutate(v: V, r: () => number): V { + const pick = (xs: X[]) => xs[Math.floor(r() * xs.length)]; + switch (v.k) { + case 'int': { const n = pick([v.v + 1n, v.v - 1n, v.v * 2n, -v.v, 0n]); return { ...v, v: n }; } + case 'float': return { ...v, v: pick([v.v * 2, -v.v, v.v + 0.5, 0]) }; + case 'bool': return { ...v, v: !v.v }; + case 'str': { + const cs = [...v.v]; const i = Math.floor(r() * (cs.length + 1)); const how = cs.length ? Math.floor(r() * 3) : 0; const alphabet = [...'abcAGCT 0é']; + if (how === 0) cs.splice(i, 0, pick(alphabet)); else if (how === 1) cs.splice(Math.min(i, cs.length - 1), 1); else cs[Math.min(i, cs.length - 1)] = pick(alphabet); + return { ...v, v: cs.join('') }; + } + case 'list': { + const items = v.items.slice(); const i = Math.floor(r() * Math.max(1, items.length)); const how = items.length ? Math.floor(r() * 3) : 0; + if (how === 0) items.splice(i, 0, items[0] ?? { k: 'int', v: 0n, width: 64, signed: true, type: 'Int' } as V); else if (how === 1) items.splice(i, 1); else items[i] = mojoMutate(items[i], r); + return { ...v, items, cap: items.length }; + } + default: return v; + } +} + +function* mojoSmaller(v: V): Iterable { + if (v.k === 'int' && v.v !== 0n) { yield { ...v, v: 0n }; yield { ...v, v: v.v > 0n ? v.v - 1n : v.v + 1n }; if (v.v > 2n || v.v < -2n) yield { ...v, v: v.v / 2n }; } + if (v.k === 'float' && v.v !== 0) { yield { ...v, v: 0 }; yield { ...v, v: Math.trunc(v.v) }; } + if (v.k === 'bool' && v.v) yield { ...v, v: false }; + if (v.k === 'str') { const cs = [...v.v]; for (let i = 0; i < cs.length; i++) yield { ...v, v: cs.slice(0, i).concat(cs.slice(i + 1)).join('') }; for (let i = 0; i < cs.length; i++) if (cs[i] !== 'a') yield { ...v, v: cs.slice(0, i).concat(['a'], cs.slice(i + 1)).join('') }; } + if (v.k === 'list') { for (let i = 0; i < v.items.length; i++) { const items = v.items.slice(0, i).concat(v.items.slice(i + 1)); yield { ...v, items, cap: items.length }; } } +} + +function mojoLines(d: DefStmt): number[] { + const out = new Set(); + const seen = new WeakSet(); + const walk = (x: any, depth: number) => { + if (!x || typeof x !== 'object' || depth > 80 || seen.has(x)) return; + seen.add(x); + if (Array.isArray(x)) { for (const y of x) walk(y, depth + 1); return; } + if (x.k === 'def' && depth > 0) return; // a nested function is its own routine + if (typeof x.pos?.line === 'number' && typeof x.k === 'string' && !('value' in x && x.k === 'name')) out.add(x.pos.line); + for (const [k, v] of Object.entries(x)) if (k !== 'pos' && v && typeof v === 'object') walk(v, depth + 1); + }; + walk(d.body, 1); + return [...out].filter((l) => l > d.pos.line).sort((a, b) => a - b); +} + +export function fuzzMojo(mod: Module, source: string, name: string, opts: FuzzOptions = {}): FuzzReport { + const d = mod.body.find((s): s is DefStmt => s.k === 'def' && s.name === name); + const empty = (problem: string, signature = name): FuzzReport => ({ name, signature, lang: 'mojo', params: [], seed: opts.seed ?? 1, budget: opts.budget ?? 300, checks: [], strategies: [], cases: [], failures: [], counts: { pass: 0, fail: 0, skip: 0, raises: 0 }, coverage: [], results: [], problem }); + if (!d) return empty(`No function called ${name}.`); + const params = d.params.filter((p) => p.star !== 'marker' && p.star !== 'positional-marker'); + const signature = `${d.name}(${params.map((p) => `${p.name}: ${mojoType(p.type)}`).join(', ')})${d.raises ? ' raises' : ''}${d.ret ? ` -> ${mojoType(d.ret as any)}` : ''}`; + if (d.typeParams.length) return empty('This function has compile-time parameters; generated inputs cover plain arguments only.', signature); + const gens: Gen[] = []; + for (const p of params) { + if (p.star || p.convention === 'mut' || p.convention === 'out' || p.convention === 'ref' || p.convention === 'inout') return empty(`${p.name} is passed as ${p.star ? 'a variadic' : p.convention}: generated inputs cover arguments passed by value only.`, signature); + const g = generator(mojoType(p.type)); + if (!g) return empty(`No generator for ${p.name}'s type (${mojoType(p.type) || 'none written'}): Int, the sized integers, Float64, Bool, String, and a List of those are generated.`, signature); + gens.push(g); + } + const HOW: Record = { String: 'text from the letters a b c A G C T, a space, 0 and é, 0 to 6 long: "", "a", "A", "ab", "aa", "ba", "GATTACA", " ", "é", "0" first', Bool: 'True and False', Float64: 'decimals: 0, 1, -1, 0.5, -0.5, 2.5, 1e10, -1e10 first, then -100 to 100 in hundredths', Float32: 'decimals, as Float64 but 32-bit' }; + const how = (t: string): string => HOW[t] ?? (/^List\[/.test(t) ? `lists of 0 to 5 elements of ${t.slice(5, -1)}` : `whole numbers of ${t}: 0, 1, -1, 2, 3, 10, 100 and the type's extremes first, then mostly -100 to 100, sometimes up to ±1e9`); + const driver: Driver = { + name: d.name, signature, lang: 'mojo', + params: params.map((p) => ({ name: p.name, type: mojoType(p.type), how: how(mojoType(p.type)) })), + checks: [ + 'It does not abort: an index out of bounds stops a compiled Mojo program (exit 132).', + d.raises ? 'It says raises, so what it raises is listed, not failed.' : 'It raises nothing that nobody catches (it does not say raises).', + 'It finishes within 5,000 steps of Programming+\'s interpreter.', + 'Run again on the same input, it gives the same answer.', + 'Integers that wrap and an Int division by zero (which is 0) are Mojo\'s behavior, not failures.', + ], + lines: mojoLines(d), + edges() { + const out: V[][] = []; + const combine = (i: number, acc: V[]) => { if (out.length >= 400) return; if (i === gens.length) { out.push(acc); return; } for (const v of gens[i].edges) combine(i + 1, [...acc, v]); }; + combine(0, []); + return out; + }, + exhaustive: () => [], + random: (r) => gens.map((g) => g.random(r)), + mutate(args, r) { if (!args.length) return args; const i = Math.floor(r() * args.length); const a = args.slice(); a[i] = mojoMutate(a[i], r); return a; }, + *smaller(args) { for (let i = 0; i < args.length; i++) for (const s of mojoSmaller(args[i])) { const a = args.slice(); a[i] = s; yield a; } }, + call: (args) => `${d.name}(${args.map(literal).join(', ')})`, + arg: (v) => literal(v), + exec(args) { + // a probe stops at 5,000 steps of its own; what it took counts toward the work allowed + const { outcome: o, hits, steps } = probeMojo(mod, source, d.name, args); + switch (o.kind) { + case 'declined': return { outcome: 'declined', message: `Programming+ cannot run ${d.name} yet: ${o.reason}.`, steps, hits }; + case 'returned': return { outcome: 'pass', result: o.value ? showMojo(o.value) : 'None', steps, hits }; + case 'aborted': return { outcome: 'fail', kind: 'bounds', message: `aborts: ${o.message}`, line: o.line, steps, hits }; + case 'limit': return { outcome: 'fail', kind: 'limit', message: 'ran past the 5,000-step limit: an endless loop or recursion, or a lot of work (the compiled program may finish; this says Programming+ stopped waiting)', steps, hits }; + case 'raised': return d.raises + ? { outcome: 'raises', kind: 'raises-declared', message: `raises ${o.message} (it says raises, so this may be intended)`, line: o.line ?? undefined, steps, hits } + : { outcome: 'fail', kind: 'raise', message: `raises ${o.message}, which nothing catches`, line: o.line ?? undefined, steps, hits }; + } + }, + }; + return fuzz(driver, opts); +} diff --git a/src/common/machine.ts b/src/common/machine.ts new file mode 100644 index 0000000..2fda523 --- /dev/null +++ b/src/common/machine.ts @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The machine models a run of x86-64 can use, as the state names them: a preset, with virtual memory on or off. + * DOM-free and part of core: the assembly service hands the model its machine from here, and the app's board + * (hardware.ts) reads the same names. + */ +import { PRESETS } from '../engines/asm86/machine.js'; + + +/** The machine models a run can use: `teaching` (small caches, so misses happen on small data), `realistic` (shaped like a desktop core), `ideal`. */ +export const MACHINE_PRESETS: readonly string[] = Object.keys(PRESETS); + +/** A machine as the state names it: a preset, with virtual memory (paging and a TLB) on or off. `teaching+paging` is the teaching machine with paging on. */ +export interface MachineChoice { preset: string; paging: boolean } + +const PAGING = '+paging'; + +export function parseMachine(name: string | null | undefined): MachineChoice { + const raw = String(name ?? '').trim(); + const paging = raw.endsWith(PAGING); + const preset = paging ? raw.slice(0, -PAGING.length) : raw; + return { preset: MACHINE_PRESETS.includes(preset) ? preset : 'teaching', paging }; +} + +export const machineName = (c: MachineChoice): string => `${c.preset}${c.paging ? PAGING : ''}`; + +/** What the engine's hardware model is given (`trace.spec.machine`): the preset's name, or the preset with paging switched on. */ +export function machineSpec(name: string | null | undefined): string | { preset: string; paging: { enabled: true } } { + const c = parseMachine(name); + return c.paging ? { preset: c.preset, paging: { enabled: true } } : c.preset; +} + +/** The machine in a few words, for captions. */ +export const machineLabel = (name: string | null | undefined): string => { const c = parseMachine(name); return c.paging ? `${c.preset}, with virtual memory` : c.preset; }; diff --git a/src/common/problems.ts b/src/common/problems.ts new file mode 100644 index 0000000..6eaabba --- /dev/null +++ b/src/common/problems.ts @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The problem catalog, built from the exercise registry (src/exercises/). Each exercise carries whichever layers + * the course wrote for it; its Mojo program and the compiler's listing come from the corpus (library/mojo/local/), and a + * speco spec is extracted from the pseudocode's own contracts when none is given. The app adds where each problem + * opens (its area) and the state it opens with (core/catalog.ts). + */ +import { EXERCISES, type Exercise } from '../exercises/index.js'; +import type { LangId } from './types.js'; +import type { WorkState } from './workspace.js'; +import { extractSpec } from '../services/speco.js'; +import { MOJO_PROGRAMS, type MojoProgram } from '../mojo/generated/programs.js'; +import { MOJO_LISTINGS } from '../mojo/generated/listings.js'; + +export interface Problem extends Exercise { + lang: LangId; + /** The exercise's Mojo program and godbolt's verdict on it, when it names one. */ + mojo?: MojoProgram & { id: string }; +} + +function problem(e: Exercise): Problem { + const sources = { ...e.sources }; + const mojo = e.mojoProgram ? MOJO_PROGRAMS[e.mojoProgram] : undefined; + if (e.mojoProgram && !mojo) throw new Error(`exercise ${e.id} names Mojo program ${e.mojoProgram}, which is not in library/mojo/local/`); + if (mojo && !sources.mojo) sources.mojo = mojo.source; + // the compiler's listing for the program, when `pixi run play` has stored one, fills an x86-64 pane the exercise left empty + const listing = e.mojoProgram ? MOJO_LISTINGS[e.mojoProgram] : undefined; + const fromListing = !!listing && !sources.asm; + if (listing && fromListing) sources.asm = listing.asm; + if (sources.pseudo && !sources.speco) { + const spec = extractSpec(sources.pseudo); + if (spec) sources.speco = spec; + } + return { ...e, sources, setup: e.setup ?? (fromListing ? listing!.setup : undefined), mojo: mojo && { ...mojo, id: e.mojoProgram! }, lang: e.lang ?? (sources.mojo ? 'mojo' : sources.pseudo ? 'pseudo' : sources.lingo ? 'lingo' : 'asm') }; +} + +export const PROBLEMS: Problem[] = EXERCISES.map(problem); + +export function problemById(id: string): Problem | undefined { + return PROBLEMS.find((p) => p.id === id); +} + +/** What a workspace opens a problem with: its sources, the pane it starts in, its call, setup and machine. */ +export function workForProblem(p: Problem): WorkState { + return { sources: { ...p.sources }, lang: p.lang, call: p.call ?? null, setup: p.setup, machine: p.machine }; +} diff --git a/src/common/reference.ts b/src/common/reference.ts new file mode 100644 index 0000000..d0351da --- /dev/null +++ b/src/common/reference.ts @@ -0,0 +1,478 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The language reference, as data. One module feeds three things: the + * reference section on the home page, the hover explanations in every + * editor, and the tests that check every example here still parses. + * + * Wherever an engine already holds the fact (keywords, built-ins, + * precedence, sentence forms, instruction glosses, condition codes), this + * module reads it from the engine rather than restating it. The tables + * written here by hand (statement forms, what stops a run, speco syntax) + * carry examples, and `tests/reference.test.mjs` parses every one. + */ +import { KEYWORDS, BUILTINS, PRECEDENCE, CONTEXTUAL, CONVERSIONS, SOFT_BUILTINS } from '../engines/pseudo/lang.js'; +import { SENTENCE_FORMS, COMPARISONS, IN_PHRASES } from '../engines/pseudo/english.js'; +import { OPS, CONDITION_TEXT, REG64, SYSCALLS } from '../engines/asm86/interp.js'; +import { PRESETS } from '../engines/asm86/machine.js'; +import type { LangId } from './types.js'; + +export interface RefEntry { term: string; meaning: string; example?: string; note?: string } +export interface RefSection { id: string; title: string; intro?: string; entries: RefEntry[]; columns?: [string, string, string?] } +export interface RefLanguage { id: LangId; title: string; blurb: string; sections: RefSection[] } + +/* ------------------------------------------------------------------ */ +/* Pseudocode */ +/* ------------------------------------------------------------------ */ + +const KEYWORD_GROUPS: Array<[string, string[]]> = [ + ['Declarations', ['DECLARE', 'CONSTANT', 'DEFINE', 'TYPE', 'ENDTYPE', 'OF']], + ['Libraries', ['USE']], + ['Types', ['INTEGER', 'FLOAT', 'CHAR', 'STRING', 'BOOL', 'BOOLEAN', 'ARRAY', 'LIST', 'HASHMAP', 'SET']], + ['Sized numbers', ['INT8', 'INT16', 'INT32', 'INT64', 'UINT8', 'UINT16', 'UINT32', 'UINT64', 'FLOAT32', 'FLOAT64']], + ['Selection', ['IF', 'THEN', 'ELSEIF', 'ELSE', 'ENDIF', 'CASE', 'OTHERWISE', 'ENDCASE']], + ['Iteration', ['FOR', 'TO', 'STEP', 'NEXT', 'PARALLEL', 'WHILE', 'ENDWHILE', 'REPEAT', 'UNTIL', 'BREAK', 'CONTINUE']], + ['Subroutines', ['FUNCTION', 'ENDFUNCTION', 'PROCEDURE', 'ENDPROCEDURE', 'RETURNS', 'RETURN', 'CALL', 'BYREF', 'BYVAL']], + ['Input and output', ['INPUT', 'OUTPUT']], + ['Operators', ['AND', 'OR', 'NOT', 'DIV', 'MOD', 'IN']], + ['Values', ['TRUE', 'FALSE']], + ['Reasoning', ['REQUIRE', 'ENSURE', 'INVARIANT', 'DECREASES', 'ASSERT', 'FORALL', 'EXISTS']], +]; + +/** What each keyword means, for hover. */ +export const KEYWORD_MEANING: Record = { + DECLARE: 'Creates a name and fixes its type, with or without a starting value: DECLARE Counter : INTEGER, or DECLARE Counter : INTEGER <- 0', CONSTANT: 'Names a value that never changes: CONSTANT BasesPerCodon = 3', DEFINE: 'Makes one fixed set of a named set type: DEFINE Purines (\'A\', \'G\') : BaseSet', TYPE: 'Defines a record (TYPE … ENDTYPE with one DECLARE per field) or names a set type (TYPE BaseSet = SET OF CHAR)', ENDTYPE: 'Closes a TYPE block', OF: 'Joins a collection to its element type: LIST OF INTEGER', + USE: 'Brings routines from the library into the program: USE stats (the whole module) or USE stats.Mean (one routine). USE IEEE lets NaN and infinities flow as IEEE 754 says instead of stopping the run', + INTEGER: 'A whole number; a run holds it exactly from -9,007,199,254,740,991 to 9,007,199,254,740,991 and stops past either (the prover treats it as a whole number of any size)', FLOAT: 'A number with a fractional part, in IEEE 754 double precision; / always produces one', + INT8: 'A whole number from -128 to 127; a result outside that stops the run', INT16: 'A whole number from -32,768 to 32,767', INT32: 'A whole number from -2,147,483,648 to 2,147,483,647', INT64: 'A 64-bit whole number (held exactly in this lab to ±9,007,199,254,740,991)', + UINT8: 'A whole number from 0 to 255: a byte', UINT16: 'A whole number from 0 to 65,535', UINT32: 'A whole number from 0 to 4,294,967,295', UINT64: 'A 64-bit whole number of 0 or more (held exactly in this lab to 9,007,199,254,740,991)', + FLOAT32: 'A FLOAT in single precision (about 7 significant digits): every +, -, *, / and ^ of FLOAT32 values, and every value stored in one, is rounded to the nearest single; SQRT, EXP and the other functions give a FLOAT, converted with FLOAT32(...)', FLOAT64: 'Another name for FLOAT: double precision, about 16 significant digits', CHAR: 'Exactly one character, in single quotes', STRING: 'Any number of characters, in double quotes; positions count from 0', BOOL: 'TRUE or FALSE', ARRAY: 'A fixed number of values of one type, declared by length: ARRAY[4] OF INTEGER, or ARRAY[R, C] for two dimensions', LIST: 'Any number of values of one type, in order; starts empty and grows with APPEND', HASHMAP: 'Values reached by key: HASHMAP OF STRING : CHAR', SET: 'Distinct values in no order; ask IN', + IF: 'Runs its body when the condition is TRUE', THEN: 'Ends the condition of an IF', ELSEIF: 'One word: another condition under the same IF, tested only if the ones above failed', ELSE: 'The body that runs when no condition above was TRUE', ENDIF: 'Closes an IF', CASE: 'CASE OF X: picks the first arm whose value, list of values, or half-open range matches X', OTHERWISE: 'The arm of a CASE that runs when nothing else matched; must come last', ENDCASE: 'Closes a CASE', + FOR: 'A counted loop: FOR I <- 0 TO N runs with I = 0, 1, …, N-1. The bound is not reached. FOR EACH X IN C visits every value', PARALLEL: 'PARALLEL FOR I <- 0 TO N: a counted loop whose passes are independent, as on a GPU where each runs on its own thread. The lab runs them and stops with a data race if one pass writes a location another pass touches', TO: 'The stop value of a FOR, which the counter never reaches (half-open), or the upper end of a CASE range', STEP: 'The increment of a FOR counter; negative counts down', NEXT: 'Closes a FOR, naming its counter: NEXT I', WHILE: 'Tests before each pass; the body may run zero times', ENDWHILE: 'Closes a WHILE', REPEAT: 'Runs the body, then tests; the body runs at least once', UNTIL: 'Closes a REPEAT with the condition that stops it', BREAK: 'Leaves the innermost loop at once', CONTINUE: 'Skips the rest of this pass and starts the next', + FUNCTION: 'A subroutine that hands back one value: FUNCTION F(X : INTEGER) RETURNS INTEGER', ENDFUNCTION: 'Closes a FUNCTION', PROCEDURE: 'A subroutine that does something and returns nothing; called with CALL', ENDPROCEDURE: 'Closes a PROCEDURE', RETURNS: 'States the type of a FUNCTION\'s answer', RETURN: 'Hands back the answer and ends the subroutine at once', CALL: 'Runs a PROCEDURE on a line of its own: CALL REPORT("ATT", Found)', BYREF: 'Marks a parameter the procedure may change in the caller; everything else is a copy', BYVAL: 'States the default explicitly: the parameter is a copy', + INPUT: 'Fills one variable from outside the program', OUTPUT: 'Shows any number of values, listed with commas and printed one after another with nothing between; text prints without its quotes', + AND: 'TRUE when both sides are; the right side is skipped once the left is FALSE, so put the guard first', OR: 'TRUE when either side is (or both)', NOT: 'TRUE when its operand is FALSE; binds looser than a comparison, so NOT A == B means NOT (A == B)', DIV: 'Integer division, rounded down (toward minus infinity): 7 DIV 2 is 3 and -7 DIV 2 is -4', MOD: 'The remainder after DIV, with the sign of the divisor: 7 MOD 2 is 1, -7 MOD 2 is 1 and 7 MOD -2 is -1', IN: 'Membership: X IN S for a SET, a LIST, a STRING, or the keys of a HASHMAP; also joins a FOR EACH variable to its collection', + TRUE: 'The BOOL value for yes', FALSE: 'The BOOL value for no', + REQUIRE: 'What every caller promises about the arguments; sits directly under the FUNCTION or PROCEDURE line', ENSURE: 'What the subroutine promises about its answer, RESULT; sits under the FUNCTION line after any REQUIRE', INVARIANT: 'What is true at the top of every pass of a loop; the first line of the loop body', DECREASES: 'A whole number that shrinks every pass (or every recursive call) and never goes below 0: why the loop ends', ASSERT: 'What is true at this line; checked on every run', FORALL: 'FORALL K IN [0, N) : P(K) is TRUE when P holds at every position', EXISTS: 'EXISTS K IN [0, N) : P(K) is TRUE when P holds at some position', + SUM: 'SUM K IN [0, N) : Q[K] adds the values over the range', EACH: 'FOR EACH X IN C visits every value of a collection', RESULT: 'Inside an ENSURE, the value the function returned', OLD: 'Inside an ENSURE, OLD(X) is a BYREF parameter\'s value when the call began', BOOLEAN: 'Another spelling of BOOL', +}; + +const OPERATOR_MEANING: Record = { + '<-': 'Assignment: the value on the right travels into the name on the left', '==': 'Equal to; produces a BOOL', '!=': 'Not equal to', '<': 'Less than', '<=': 'Less than or equal', '>': 'Greater than', '>=': 'Greater than or equal', + '+': 'Addition', '-': 'Subtraction, or negation', '*': 'Multiplication', '/': 'Division; always produces a FLOAT, even 4 / 2', '^': 'Power; binds right: 2 ^ 3 ^ 2 is 2 ^ 9', '&': 'Joins two pieces of text into a STRING', + '=': 'Used only in CONSTANT X = … and TYPE X = SET OF …; everywhere else it is a mistake for <- or ==', ':': 'Separates a name from its type, a key type from a value type, or a CASE label from its body', + '[': 'Indexing: T[I] is the character at position I, counting from 0; A[R, C] for two dimensions; [0, N) is a half-open range of positions; [1, 2] is a LIST literal', '{': 'A SET literal {\'A\', \'G\'}, a HASHMAP literal {"ATG": \'M\'}, or a set written by rule { K IN [0, N) : T[K] == B }', '.': 'Reaches a field of a record: R.Name', '//': 'A comment to the end of the line', +}; + +const PSEUDO_STATEMENTS: RefEntry[] = [ + { term: 'DECLARE : ', meaning: 'Creates a name of that type with no value yet. Several names may share a line.', example: 'DECLARE Counter : INTEGER\nDECLARE N, M : INTEGER' }, + { term: 'DECLARE : <- ', meaning: 'Creates a name and gives it its starting value on the same line, as Mojo\'s var total: Int = 0 does. One name per line in this form; the two-line form (DECLARE, then an assignment) means the same.', example: 'DECLARE T : STRING <- "GATTACA"\nDECLARE N : INTEGER <- LENGTH(T)\nDECLARE Total : INTEGER <- 0\nOUTPUT N, " ", Total' }, + { term: 'USE / USE .', meaning: 'Brings verified routines from the library into the program, to be called like your own. A run does not step into them (open the routine in the library to do that); the prover uses their REQUIRE and ENSURE lines at each call. Lines go at the top of the program.', example: 'USE IEEE\nDECLARE X : FLOAT\nX <- 1.0 / 0.0\nOUTPUT IS_INF(X)' }, + { term: 'DECLARE : INT32 (or another sized type)', meaning: 'A number of a fixed width. A whole-number result that does not fit stops the run rather than wrapping around; a FLOAT32 result is rounded to single precision. Two different sized types meet only through a conversion such as INT32(X) or FLOAT32(X), and a literal takes the type beside it.', example: 'DECLARE Count : UINT8 <- 250\nCount <- Count + 5\nOUTPUT Count, " ", WRAP(Count + 1, UINT8)\nDECLARE X : FLOAT32 <- 0.1\nOUTPUT X, " holds ", EXACT(X)' }, + { term: 'CONSTANT = ', meaning: 'A value fixed for the whole program, worked out once before it starts from literals, other constants and built-ins (2 + 3, LENGTH("ACGT")); a variable is refused. A random draw is drawn once and keeps its value.', example: 'CONSTANT BasesPerCodon = 3' }, + { term: ' <- ', meaning: 'Assignment. The right side is worked out first, then stored.', example: 'DECLARE Counter : INTEGER\nCounter <- 0\nCounter <- Counter + 1' }, + { term: 'OUTPUT , , ...', meaning: 'Shows values in order. A collection prints the way you would type it.', example: 'OUTPUT "Found ", 3, " matches"' }, + { term: 'INPUT ', meaning: 'Fills a declared variable from outside the program.', example: 'DECLARE Threshold : INTEGER\nINPUT Threshold' }, + { term: 'IF THEN ... ELSEIF THEN ... ELSE ... ENDIF', meaning: 'The first TRUE condition wins; ELSEIF and ELSE are optional; one ENDIF closes the lot.', example: 'DECLARE Q : FLOAT\nQ <- 25.0\nIF Q < 20 THEN\n OUTPUT "discard"\nELSEIF Q < 30 THEN\n OUTPUT "marginal"\nELSE\n OUTPUT "keep"\nENDIF' }, + { term: 'CASE OF ... : ... TO : ... OTHERWISE : ... ENDCASE', meaning: 'Picks the first arm whose value, comma-separated values, or half-open range matches. Ranges are [lo, hi).', example: "DECLARE Base : CHAR\nDECLARE Partner : CHAR\nBase <- 'A'\nCASE OF Base\n 'A' : Partner <- 'T'\n 'T' : Partner <- 'A'\n 'G', 'C' : OUTPUT \"strong\"\n OTHERWISE : Partner <- 'N'\nENDCASE" }, + { term: 'FOR <- TO [STEP ] ... NEXT ', meaning: 'Counts from start while the counter is strictly less than stop (strictly greater, with a negative STEP). The counter needs no DECLARE and has no value after the loop.', example: 'DECLARE T : STRING\nT <- "GATTACA"\nFOR I <- 0 TO LENGTH(T)\n OUTPUT T[I]\nNEXT I\nFOR J <- 10 TO 0 STEP -2\n OUTPUT J\nNEXT J' }, + { term: 'PARALLEL FOR <- TO ... NEXT ', meaning: 'A counted loop whose passes may run at the same time. Each pass may write only its own slots (indexed by the counter) and names it declares; writing a shared name, or reading a slot another pass writes, is a data race and stops the run naming both passes. This is the map step of GPU programming.', example: 'DECLARE Xs, Squares : LIST OF FLOAT\nXs <- [1.0, 2.0, 3.0]\nSquares <- FILL(LENGTH(Xs), 0.0)\nPARALLEL FOR I <- 0 TO LENGTH(Xs)\n Squares[I] <- Xs[I] * Xs[I]\nNEXT I\nOUTPUT SUM_OF(Squares)' }, + { term: 'FOR EACH IN ... NEXT ', meaning: 'Visits every character of a STRING, value of a LIST or ARRAY, member of a SET, or key of a HASHMAP.', example: 'DECLARE GC : INTEGER\nGC <- 0\nFOR EACH B IN "GATTACA"\n IF B == \'G\' OR B == \'C\' THEN\n GC <- GC + 1\n ENDIF\nNEXT B' }, + { term: 'WHILE ... ENDWHILE', meaning: 'Tests before each pass; may run zero times.', example: 'DECLARE I : INTEGER\nDECLARE T : STRING\nT <- "GATNACA"\nI <- 0\nWHILE I < LENGTH(T) AND T[I] != \'N\'\n I <- I + 1\nENDWHILE' }, + { term: 'REPEAT ... UNTIL ', meaning: 'Runs the body, then tests; runs at least once; stops when the condition becomes TRUE.', example: 'DECLARE N : INTEGER\nN <- 0\nREPEAT\n N <- N + 1\nUNTIL N >= 3' }, + { term: 'BREAK / CONTINUE', meaning: 'Leave the innermost loop at once / skip to the next pass.', example: 'FOR I <- 0 TO 10\n IF I == 2 THEN\n CONTINUE\n ENDIF\n IF I == 5 THEN\n BREAK\n ENDIF\n OUTPUT I\nNEXT I' }, + { term: 'FUNCTION (

: , ...) RETURNS ... RETURN ... ENDFUNCTION', meaning: 'A subroutine with one answer. Parameters are copies. A call stands where a value stands.', example: 'FUNCTION MAX_OF(A : INTEGER, B : INTEGER) RETURNS INTEGER\n IF A > B THEN\n RETURN A\n ENDIF\n RETURN B\nENDFUNCTION\nOUTPUT MAX_OF(3, 7)' }, + { term: 'PROCEDURE (BYREF

: , ...) ... ENDPROCEDURE, and CALL (...)', meaning: 'A subroutine with no answer, run for its effect. BYREF lets it change the caller\'s variable.', example: 'PROCEDURE SWAP(BYREF X : INTEGER, BYREF Y : INTEGER)\n DECLARE Temp : INTEGER\n Temp <- X\n X <- Y\n Y <- Temp\nENDPROCEDURE\nDECLARE A, B : INTEGER\nA <- 1\nB <- 2\nCALL SWAP(A, B)' }, + { term: 'TYPE ... DECLARE : ... ENDTYPE', meaning: 'A record: several named fields that travel together. Reach a field with a dot.', example: 'TYPE Read\n DECLARE Name : STRING\n DECLARE Bases : STRING\nENDTYPE\nDECLARE R : Read\nR.Name <- "read_1"\nR.Bases <- "GATTACA"\nOUTPUT R.Name, LENGTH(R.Bases)' }, + { term: 'TYPE = SET OF , and DEFINE (, ...) : ', meaning: 'Names a kind of set, then makes one fixed set of that kind. A growing set is DECLARE S : SET OF plus ADD.', example: "TYPE BaseSet = SET OF CHAR\nDEFINE Purines ('A', 'G') : BaseSet\nIF 'A' IN Purines THEN\n OUTPUT \"purine\"\nENDIF" }, + { term: 'REQUIRE / ENSURE / INVARIANT / DECREASES / ASSERT ', meaning: 'Claims the lab checks on every run and the prover tries to prove for every input. See the speco reference.', example: 'FUNCTION COUNT_BASE(B : CHAR, T : STRING) RETURNS INTEGER\n ENSURE RESULT == LENGTH({ K IN [0, LENGTH(T)) : T[K] == B })\n DECLARE Total : INTEGER\n Total <- 0\n FOR I <- 0 TO LENGTH(T)\n INVARIANT Total == LENGTH({ K IN [0, I) : T[K] == B })\n IF T[I] == B THEN\n Total <- Total + 1\n ENDIF\n NEXT I\n RETURN Total\nENDFUNCTION' }, +]; + +const PSEUDO_LITERALS: RefEntry[] = [ + { term: '42, -3', meaning: 'an INTEGER' }, + { term: '0.5, 2.0, 1.5e-3', meaning: 'a FLOAT; a digit on each side of the point is required' }, + { term: '1_000_000, 0xFF, 0b1010', meaning: 'an INTEGER with underscores between digits, in hexadecimal, or in binary' }, + { term: '0.1 next to a FLOAT32', meaning: 'a literal takes the type of the sized value beside it: 0.1 becomes the nearest FLOAT32, and 1 next to an INT8 is an INT8' }, + { term: "'A'", meaning: 'a CHAR: exactly one character in single quotes' }, + { term: '"GATTACA", ""', meaning: 'a STRING in double quotes; \\n, \\t, \\" and \\\\ are escapes' }, + { term: 'TRUE, FALSE', meaning: 'a BOOL' }, + { term: '[4, 17]', meaning: 'a LIST' }, + { term: "{'A', 'G'}", meaning: 'a SET' }, + { term: '{"ATG": \'M\', "TAA": \'*\'}', meaning: 'a HASHMAP' }, + { term: '[0, N)', meaning: 'a half-open range of positions, inside FORALL, EXISTS, SUM, and sets written by rule' }, + { term: '{ K IN [0, N) : T[K] == B }', meaning: 'a set written by rule: every K in the range for which the condition holds' }, + { term: '← ≤ ≥ ≠', meaning: 'accepted spellings of <-, <=, >=, !=' }, +]; + +const PSEUDO_STOPS: RefEntry[] = [ + { term: 'Position does not exist', meaning: 'an index outside [0, LENGTH), a SLICE outside [0, LENGTH], or a row or column outside an ARRAY' }, + { term: 'No value yet', meaning: 'reading a name, a position, or a field before anything is stored in it, or a FOR counter after its loop' }, + { term: 'Missing key', meaning: 'reading H[K] when K is not a key of H; test with IN first' }, + { term: 'Division by zero', meaning: 'dividing by 0 with /, DIV, or MOD (with USE IEEE, a FLOAT / gives an infinity or NaN instead; DIV and MOD still stop), or 0.0 raised to a negative power' }, + { term: 'No answer', meaning: 'a whole number to a negative power, a negative FLOAT to a power that is not whole, SQRT of a negative, LOG of a non-positive, LOG1P at or below -1, ASIN or ACOS outside [-1, 1], CLAMP with its low end above its high end, a shift by a negative count, FILL with a negative count, RANDINT from an empty range, MIN or MAX or MEAN_OF of an empty collection, text TO_INTEGER or TO_FLOAT cannot read (the FLOAT cases give NaN or an infinity instead when the program says USE IEEE)' }, + { term: 'Wrong length', meaning: 'a store, an argument, or a RETURN that gives an ARRAY a LIST or ARRAY of another length than it was declared with (an error before the run when both lengths are written as numbers), or DOT of two collections of different lengths' }, + { term: 'STEP 0', meaning: 'a FOR whose STEP is 0 when the loop starts (STEP 0 written as a number is an error before the run)' }, + { term: 'Too large', meaning: 'a whole number beyond ±9,007,199,254,740,991 (a product or partial sum inside SUM_OF or DOT too), a result outside its sized type (an INT8 past 127, a FOR counter declared INT8 past 127), a conversion such as INT32(X) that does not fit, a FLOAT past about 1.8 × 10^308 or a FLOAT32 past about 3.4 × 10^38 (unless the program says USE IEEE)' }, + { term: 'Too much to hold', meaning: 'a collection of more than 2,000,000 values (an ARRAY, FILL, or a range FOR EACH or a quantifier walks), or a text of more than 10,000,000 characters' }, + { term: 'Input', meaning: 'INPUT with nothing left, or input that does not convert (a FLOAT is read only from decimal text, such as 2.5 or -1e3)' }, + { term: 'A value from outside', meaning: 'a call from outside the program (the Run tab\'s call) whose argument does not fit its parameter: the wrong type, an ARRAY of another length, or NaN or an infinity for a FLOAT without USE IEEE' }, + { term: 'A broken promise', meaning: 'a REQUIRE, ENSURE, INVARIANT, DECREASES, or ASSERT that is FALSE' }, + { term: 'No end in sight', meaning: 'more steps than the lab allows (20,000 here); the lab names the loop and says whether its condition has stopped changing. The positions FORALL, EXISTS, SUM and sets written by rule visit count too: 100 for every step allowed (2,000,000 here)' }, + { term: 'Too deep', meaning: 'recursion more than 400 calls deep, or fewer when the call sits inside many nested blocks and the lab cannot follow it that far' }, + { term: 'A data race', meaning: 'in a PARALLEL FOR, one pass writes a location (a shared name, or a slot of a list) that another pass reads or writes; the message names the location and both passes' }, +]; + +const PSEUDO_LIMITS: RefEntry[] = [ + { term: '200 levels of nesting', meaning: 'a line inside more than 200 blocks, or brackets, calls or types nested more than 200 deep, is the error too-deep before the run: give the inner part a FUNCTION of its own' }, + { term: '400 operands in a chain', meaning: 'a chain of more than 400 operands joined by +, AND, & or the like (1 + 1 + ... + 1) is the error too-deep; so are brackets that each hold a long chain, once they build a value more than 400 levels deep' }, + { term: '2,000,000 values', meaning: 'the most one collection may hold; past it the run stops (Too much to hold)' }, + { term: '10,000,000 characters', meaning: 'the longest text a run holds; past it the run stops (Too much to hold)' }, + { term: '20,000 steps', meaning: 'the steps a run may take here, and 100 positions visited by quantifiers for each (No end in sight)' }, + { term: '400 calls deep', meaning: 'the deepest recursion a run follows (Too deep)' }, + { term: '10,000,000 recorded values', meaning: 'what the pictures of a recorded run may copy; past it the trace ends with a step that says so and the run goes on without recording, its output and how it ends shown at the end' }, + { term: '3 seconds to prove, 3 per routine to try', meaning: 'the Prove tab gives the prover 3 seconds for each routine, then tries generated inputs on each routine for up to 3 seconds; what each found in that time stands, and an obligation the prover could not finish stays open' }, +]; + +/** + * Idioms: the handful of shapes the stress test (docs/pseudo-stress-test.md) + * found worth writing down once, so nobody has to rediscover them. Every + * example runs, and `tests/reference.test.mjs` checks the OUTPUT in `note`. + */ +export const PSEUDO_IDIOMS: RefEntry[] = [ + { + term: 'Find the first, three ways', + meaning: 'A search can stop with RETURN inside a FUNCTION, with BREAK and a flag in a loop, or with a WHILE whose condition names what it is waiting for. All three agree on the answer: the position, or LENGTH when there is none. For a plain character or piece of text, INDEX_OF does the same search and answers -1 when absent; the three shapes are for searches with a condition of your own.', + example: 'FUNCTION FIRST_N(T : STRING) RETURNS INTEGER\n FOR I <- 0 TO LENGTH(T)\n IF T[I] == \'N\' THEN\n RETURN I\n ENDIF\n NEXT I\n RETURN LENGTH(T)\nENDFUNCTION\nDECLARE T : STRING\nT <- "GATNACA"\nDECLARE Found, J : INTEGER\nFound <- LENGTH(T)\nFOR I <- 0 TO LENGTH(T)\n IF T[I] == \'N\' THEN\n Found <- I\n BREAK\n ENDIF\nNEXT I\nJ <- 0\nWHILE J < LENGTH(T) AND T[J] != \'N\'\n J <- J + 1\nENDWHILE\nOUTPUT FIRST_N(T), " ", Found, " ", J, " ", INDEX_OF(T, \'N\')', + note: '3 3 3 3', + }, + { + term: 'Return two values: use a record', + meaning: 'A FUNCTION hands back one value. When the answer has two parts, name a TYPE for the pair and return that; the reader sees what each part means instead of guessing which is which.', + example: 'TYPE Extremes\n DECLARE Low : INTEGER\n DECLARE High : INTEGER\nENDTYPE\nFUNCTION RANGE_OF(Xs : LIST OF INTEGER) RETURNS Extremes\n REQUIRE LENGTH(Xs) > 0\n DECLARE E : Extremes\n E.Low <- MIN(Xs)\n E.High <- MAX(Xs)\n RETURN E\nENDFUNCTION\nDECLARE R : Extremes\nR <- RANGE_OF([7, 2, 9, 4])\nOUTPUT R.Low, " to ", R.High', + note: '2 to 9', + }, + { + term: 'Memoize with a PROCEDURE and two BYREF parameters', + meaning: 'A FUNCTION is pure, so it cannot take a table BYREF. Memoized recursion is therefore a PROCEDURE: the table travels BYREF, and the answer comes back BYREF too. It is the heaviest idiom in the notation; it is also the one that makes "the table is shared" visible.', + example: 'PROCEDURE FIB(N : INTEGER, BYREF Memo : HASHMAP OF INTEGER : INTEGER, BYREF Answer : INTEGER)\n IF N < 2 THEN\n Answer <- N\n RETURN\n ENDIF\n IF N IN Memo THEN\n Answer <- Memo[N]\n RETURN\n ENDIF\n DECLARE A, B : INTEGER\n CALL FIB(N - 1, Memo, A)\n CALL FIB(N - 2, Memo, B)\n Memo[N] <- A + B\n Answer <- A + B\nENDPROCEDURE\nDECLARE Table : HASHMAP OF INTEGER : INTEGER\nDECLARE F : INTEGER\nCALL FIB(40, Table, F)\nOUTPUT F', + note: '102334155', + }, + { + term: 'Split text: SPLIT, and what it does by hand', + meaning: 'SPLIT(S, " ") cuts a text at every space and keeps empty pieces, so two spaces give an empty word; drop those yourself when you want words. Writing the walk by hand once is still worth it: grow a word until a separator, APPEND it, start a new one, and append the last word after the loop; forgetting that last step is the classic mistake, and the trace shows it.', + example: 'FUNCTION WORDS_OF(S : STRING) RETURNS LIST OF STRING\n DECLARE Out : LIST OF STRING\n DECLARE W : STRING\n W <- ""\n FOR EACH C IN S\n IF C == \' \' THEN\n IF LENGTH(W) > 0 THEN\n APPEND(Out, W)\n ENDIF\n W <- ""\n ELSE\n W <- W & C\n ENDIF\n NEXT C\n IF LENGTH(W) > 0 THEN\n APPEND(Out, W)\n ENDIF\n RETURN Out\nENDFUNCTION\nOUTPUT WORDS_OF("the cat sat"), " ", SPLIT("the cat sat", " "), " ", JOIN(WORDS_OF("the cat sat"), "_")', + note: '["the", "cat", "sat"] ["the", "", "cat", "sat"] the_cat_sat', + }, + { + term: 'A stack from a LIST', + meaning: 'Push with APPEND; the top is the last position; pop with REMOVE of that position. A LIST already is a stack, so there is no stack type.', + example: 'DECLARE Stack : LIST OF CHAR\nAPPEND(Stack, \'(\')\nAPPEND(Stack, \'[\')\nDECLARE Top : CHAR\nTop <- Stack[LENGTH(Stack) - 1]\nREMOVE(Stack, LENGTH(Stack) - 1)\nOUTPUT Top, " ", LENGTH(Stack)', + note: '[ 1', + }, + { + term: 'A queue from a LIST', + meaning: 'Join at the end with APPEND; leave from the front with REMOVE(Q, 0). REMOVE at the front shifts every later value, so this costs LENGTH steps per leave; a lesson on cost measures exactly that.', + example: 'DECLARE Q : LIST OF STRING\nAPPEND(Q, "read_1")\nAPPEND(Q, "read_2")\nAPPEND(Q, "read_3")\nDECLARE Next_Up : STRING\nNext_Up <- Q[0]\nREMOVE(Q, 0)\nOUTPUT Next_Up, " ", Q', + note: 'read_1 ["read_2", "read_3"]', + }, + { + term: 'Count with a HASHMAP', + meaning: 'Test IN before reading a key: a missing key stops the run. "If seen, add one; else start at one" is the shape every tally takes, and the checker recognizes it.', + example: 'DECLARE Tally : HASHMAP OF CHAR : INTEGER\nFOR EACH B IN "GATTACA"\n IF B IN Tally THEN\n Tally[B] <- Tally[B] + 1\n ELSE\n Tally[B] <- 1\n ENDIF\nNEXT B\nOUTPUT Tally', + note: '{\'A\': 3, \'C\': 1, \'G\': 1, \'T\': 2}', + }, + { + term: 'Swap through BYREF', + meaning: 'The one place BYREF is the natural tool: a procedure that changes two of the caller\'s variables and has no answer of its own.', + example: 'PROCEDURE SWAP(BYREF X : INTEGER, BYREF Y : INTEGER)\n DECLARE Temp : INTEGER\n Temp <- X\n X <- Y\n Y <- Temp\nENDPROCEDURE\nDECLARE A, B : INTEGER\nA <- 1\nB <- 2\nCALL SWAP(A, B)\nOUTPUT A, " ", B', + note: '2 1', + }, + { + term: 'ARRAY for fixed shapes, LIST for everything a subroutine takes', + meaning: 'An ARRAY type always states its lengths, so a parameter of ARRAY type works for one size only. Anything a FUNCTION or PROCEDURE takes is a LIST; an ARRAY is for a table whose shape is part of the problem, such as a 3×3 grid.', + example: 'FUNCTION TOTAL(Xs : LIST OF INTEGER) RETURNS INTEGER\n DECLARE S : INTEGER\n S <- 0\n FOR EACH X IN Xs\n S <- S + X\n NEXT X\n RETURN S\nENDFUNCTION\nDECLARE Grid : ARRAY[2, 2] OF INTEGER\nGrid[0, 0] <- 1\nGrid[0, 1] <- 2\nGrid[1, 0] <- 3\nGrid[1, 1] <- 4\nOUTPUT TOTAL([1, 2, 3, 4]), " ", ROWS(Grid) * COLUMNS(Grid)', + note: '10 4', + }, +]; + +/** Names a program cannot use: keywords, contextual keywords, and built-ins, in any capitalization. */ +export function reservedNames(): RefEntry[] { + const kw = [...(KEYWORDS as Set)].sort().map((w) => ({ term: w, meaning: 'keyword', note: KEYWORD_MEANING[w] ? KEYWORD_MEANING[w].split(/[:;.]/)[0] : '' })); + const ctx = Object.keys(CONTEXTUAL as Record).sort().map((w) => ({ term: w, meaning: 'contextual keyword', note: KEYWORD_MEANING[w] ? KEYWORD_MEANING[w].split(/[:;.]/)[0] : '' })); + const bi = Object.keys(BUILTINS as Record).filter((w) => !(SOFT_BUILTINS as Set).has(w)).sort().map((w) => ({ term: w, meaning: 'built-in function', note: ((BUILTINS as any)[w].doc || '').split(/[:;.]/)[0] })); + return [...kw, ...ctx, ...bi]; +} + +function builtinEntries(): RefEntry[] { + return Object.entries(BUILTINS as Record).map(([name, b]) => ({ term: b.sig || name, meaning: b.doc || '', note: b.group || '' })) + .concat(Object.values(CONVERSIONS as Record).map((c) => ({ term: c.sig, meaning: c.doc, note: c.group }))); +} + +export const PSEUDO_REF: RefLanguage = { + id: 'pseudo', title: 'pseudo', blurb: 'The course notation: uppercase keywords, <- for assignment, half-open bounds everywhere, typed names, and claims the prover can check.', + sections: [ + { id: 'pseudo-keywords', title: 'Keywords', intro: 'Uppercase, always. Names are case-insensitive and may not reuse a keyword or a built-in.', entries: KEYWORD_GROUPS.flatMap(([group, words]) => words.filter((w) => (KEYWORDS as Set).has(w)).map((w) => ({ term: w, meaning: KEYWORD_MEANING[w] || '', note: group }))).concat(Object.entries(CONTEXTUAL as Record).map(([w, where]) => ({ term: w, meaning: `${KEYWORD_MEANING[w] || ''} (a keyword only ${where})`, note: 'Contextual' }))), columns: ['Keyword', 'Meaning', 'Group'] }, + { id: 'pseudo-literals', title: 'Literals', entries: PSEUDO_LITERALS, columns: ['Written', 'Is'] }, + { id: 'pseudo-operators', title: 'Operators, loosest first', intro: 'Parenthesize wherever a reader might have to look this table up.', entries: (PRECEDENCE as any[]).map((p) => ({ term: p.ops.join(' '), meaning: `${p.assoc === 'chain' ? 'chains' : p.assoc}${p.note ? ': ' + p.note : ''}`, note: `level ${p.level}` })), columns: ['Operators', 'Associativity and notes', 'Level'] }, + { id: 'pseudo-statements', title: 'Statements', intro: 'Every form, with a runnable example.', entries: PSEUDO_STATEMENTS, columns: ['Form', 'Meaning', 'Example'] }, + { id: 'pseudo-builtins', title: 'Built-in functions', entries: builtinEntries(), columns: ['Signature', 'Returns', 'Group'] }, + { id: 'pseudo-idioms', title: 'Idioms', intro: 'Shapes worth learning once. Each example runs; the last column is what it prints.', entries: PSEUDO_IDIOMS, columns: ['Idiom', 'Why it is written this way', 'Example and output'] }, + { id: 'pseudo-stops', title: 'What stops a run', entries: PSEUDO_STOPS, columns: ['Stop', 'When'] }, + { id: 'pseudo-limits', title: 'Limits', intro: 'What the lab will read, run and record. Each limit is far above what the course\'s programs need.', entries: PSEUDO_LIMITS, columns: ['Limit', 'What happens'] }, + { id: 'pseudo-reserved', title: 'Names you cannot use', intro: 'Names ignore case, so Next, step, and Keys are refused just as NEXT, STEP, and KEYS would be. Pick a longer word: Neighbor, Generation, KeyList.', entries: reservedNames(), columns: ['Word', 'Reserved as', 'Means'] }, + ], +}; + +/* ------------------------------------------------------------------ */ +/* lingo */ +/* ------------------------------------------------------------------ */ + +const LINGO_QUESTIONS: RefEntry[] = [ + { term: 'range-ends', meaning: '"from 0 to the length of T" does not say whether the end is included. Offers "up to but not including" and "up to and including".', example: 'To count in T:\n Start with Count at 0.\n For each i from 0 to the length of T:\n Add 1 to Count.\n The answer is Count.' }, + { term: 'between', meaning: '"between 20 and 30" leaves both ends undecided. Offers the three readings.', example: 'To test Q:\n If Q is between 20 and 30, the answer is 1.\n The answer is 0.' }, + { term: 'answer-in-loop', meaning: '"the answer is i" inside a loop could stop now or keep looking. Offers "Stop and answer i" and "Set Best to i".', example: 'To find it in T:\n For each position i of T:\n If the letter of T at i is \'N\', the answer is i.\n The answer is the length of T.' }, + { term: 'is-step', meaning: '"Count is 0" could set Count or claim it. As a step, say "Set Count to 0" or "Check that Count is 0".', example: 'To do it with T:\n Count is 0.\n The answer is Count.' }, + { term: 'and-or', meaning: '"A or B and C" does not say which binds. Parentheses do.', example: "To test b and i:\n If b is 'A' or b is 'T' and i is 0, the answer is 1.\n The answer is 0." }, + { term: 'not-or', meaning: '"is not \'G\' or \'C\'" has two readings; say "is neither \'G\' nor \'C\'" or "is not one of".', example: "To test b:\n If b is not 'G' or 'C', the answer is 1.\n The answer is 0." }, + { term: 'stop', meaning: '"Stop" inside a loop could end the loop or the routine.', example: 'To scan T:\n Start with Count at 0.\n For each letter b of T:\n If b is \'N\', stop.\n Add 1 to Count.\n The answer is Count.' }, + { term: 'pronoun', meaning: '"it", "they", "them" could stand for several names. Name the thing you mean.', example: "To scan T:\n Start with Count at 0.\n For each letter b of T:\n If it is 'G', add 1 to Count.\n The answer is Count." }, + { term: 'used-before-set', meaning: 'A name used before any step gives it a value: the set-up step is missing.', example: 'To scan T:\n For each letter b of T:\n Add 1 to Count.\n The answer is Count.' }, + { term: 'text-place', meaning: '"Add b to R" with R a text does not say where b goes; say "to the end of" or "to the front of".', example: 'To reverse T:\n Start with an empty text called R.\n For each letter b of T:\n Add b to R.\n The answer is R.' }, +]; + +export const LINGO_REF: RefLanguage = { + id: 'lingo', title: 'lingo', blurb: 'Plain-English steps with exactly one meaning each. The reader knows a fixed set of sentence shapes; anything else is a phrase you define, or a question it asks you.', + sections: [ + ...(SENTENCE_FORMS as any[]).map((g, i) => ({ id: `lingo-forms-${i}`, title: g.group as string, entries: (g.forms as string[][]).map(([form, example]) => ({ term: form, meaning: '', example: example || undefined })), columns: ['Sentence', '', 'Example'] as [string, string, string] })), + { id: 'lingo-comparisons', title: 'Comparison phrasings', intro: 'Every phrase in a column means the same operator.', entries: Object.entries((COMPARISONS as Array<[string, string]>).reduce((acc, [phrase, op]) => { (acc[op] ||= []).push(phrase); return acc; }, {} as Record)).map(([op, phrases]) => ({ term: op, meaning: phrases.join(' · ') })).concat((IN_PHRASES as Array<[string, string]>).map(([p, k]) => ({ term: k === 'in' ? 'IN' : 'NOT IN', meaning: p.replace(/\|/g, ' / ') }))), columns: ['Means', 'Phrasings'] }, + { id: 'lingo-questions', title: 'Questions the reader asks', intro: 'Each is deterministic: the same sentence always raises the same question, with the readings as one-click rewrites.', entries: LINGO_QUESTIONS, columns: ['Id', 'When, and what it offers', 'A sentence that raises it'] }, + ], +}; + +/* ------------------------------------------------------------------ */ +/* speco */ +/* ------------------------------------------------------------------ */ + +/** The symbol table that lowers math to the notation and renders it back. speco.ts imports this. */ +export const SPECO_SYMBOLS: Array<{ math: string; ascii: string; meaning: string }> = [ + { math: '∀', ascii: 'FORALL', meaning: 'for every position in a range: ∀ k ∈ [0, n) : P(k)' }, + { math: '∃', ascii: 'EXISTS', meaning: 'for some position in a range' }, + { math: 'Σ', ascii: 'SUM', meaning: 'the sum over a range: Σ k ∈ [0, n) : Q[k]' }, + { math: '∈', ascii: 'IN', meaning: 'membership, or the range a bound variable runs over' }, + { math: '∧', ascii: 'AND', meaning: 'both' }, + { math: '∨', ascii: 'OR', meaning: 'either' }, + { math: '¬', ascii: 'NOT', meaning: 'not' }, + { math: '≤', ascii: '<=', meaning: 'at most' }, + { math: '≥', ascii: '>=', meaning: 'at least' }, + { math: '≠', ascii: '!=', meaning: 'different' }, + { math: '=', ascii: '==', meaning: 'equal (a claim, never an assignment)' }, + { math: '|X|', ascii: 'LENGTH(X)', meaning: 'the size of a text, list, set, or set written by rule' }, + { math: 'result', ascii: 'RESULT', meaning: 'the value the function returns' }, + { math: 'X₀', ascii: 'OLD(X)', meaning: 'a BYREF parameter\'s value when the call began' }, +]; + +const SPECO_FORMS: RefEntry[] = [ + { term: 'SPEC ()', meaning: 'Opens the claims about one FUNCTION or PROCEDURE in the pseudocode pane. The parameter names are documentation; the pseudocode header is authoritative.', example: 'SPEC COUNT_BASE(B, T)\n ENSURE result = |{ K ∈ [0, |T|) : T[K] = B }|' }, + { term: 'REQUIRE ', meaning: 'What every caller promises about the arguments.', example: 'SPEC GC_FRACTION(T)\n REQUIRE |T| ≥ 1' }, + { term: 'ENSURE ', meaning: 'What the routine promises about result (and about BYREF parameters, with X₀ for the starting value).', example: 'SPEC MAX_OF(A, B)\n ENSURE result ≥ A ∧ result ≥ B' }, + { term: 'LOOP FOR / LOOP FOR EACH / LOOP WHILE / LOOP REPEAT / LOOP ', meaning: 'Names the loop the lines under it describe: by its counter, by its kind, or by its position among the routine\'s loops.', example: 'SPEC COUNT_BASE(B, T)\n LOOP FOR I\n INVARIANT Total = |{ K ∈ [0, I) : T[K] = B }|' }, + { term: 'INVARIANT ', meaning: 'True at the top of every pass. Checked before the first pass and after every pass. When the loop ends normally it is checked once more, except in FOR EACH for a claim that names the loop\'s own variable, which is about the value a pass is about to visit; in a FOR, that last check sees the counter at the value that ended the loop.', example: 'SPEC CONTAINS_SORTED(L, X)\n LOOP WHILE\n INVARIANT 0 ≤ Lo ∧ Lo ≤ Hi ∧ Hi ≤ |L|' }, + { term: 'DECREASES ', meaning: 'A measure that is at least 0 whenever the loop continues and shrinks every pass: why it ends. Under SPEC directly, it measures a recursive routine.', example: 'SPEC CONTAINS_SORTED(L, X)\n LOOP WHILE\n DECREASES Hi - Lo' }, + { term: '{ k ∈ [lo, hi) : }', meaning: 'A set written by rule; |…| around it counts its members.', example: 'SPEC COUNT_GC(T)\n ENSURE result = |{ k ∈ [0, |T|) : T[k] = \'G\' ∨ T[k] = \'C\' }|' }, +]; + +export const SPECO_REF: RefLanguage = { + id: 'speco', title: 'speco', blurb: 'Specifications as a document of their own, in mathematical notation. Programming+ weaves them into the pseudocode and the prover checks every claim for every input it can reason about.', + sections: [ + { id: 'speco-symbols', title: 'Symbols', intro: 'Type either column; the pane accepts the math and the notation\'s ASCII spelling alike.', entries: SPECO_SYMBOLS.map((s) => ({ term: s.math, meaning: s.meaning, note: s.ascii })), columns: ['Math', 'Meaning', 'Notation'] }, + { id: 'speco-forms', title: 'Lines', entries: SPECO_FORMS, columns: ['Form', 'Meaning', 'Example'] }, + { + id: 'speco-verdicts', title: 'What the prover reports', entries: [ + { term: 'proved', meaning: 'on every path the checker examined, the claim follows from what is known; no input can break it' }, + { term: 'broken by a run', meaning: 'the checker could not prove it, and a generated input violates it; Programming+ shows the smallest it found' }, + { term: 'open', meaning: 'not proved and not broken by any input tried: usually a loop needs an invariant, or the claim needs reasoning the checker lacks (products of unknowns, facts about whole sets, exact FLOAT values)' }, + ], columns: ['Verdict', 'Meaning'] + }, + ], +}; + +/* ------------------------------------------------------------------ */ +/* x86-64 */ +/* ------------------------------------------------------------------ */ + +const REG_MEANING: Record = { + rax: 'accumulator; the answer of a procedure and the system-call number', rbx: 'callee-saved general register', rcx: '4th argument; loop counter by convention', rdx: '3rd argument; high half of a 128-bit product or dividend', rsi: '2nd argument; source pointer by convention', rdi: '1st argument; destination pointer by convention', rbp: 'frame pointer (callee-saved)', rsp: 'stack pointer: the address of the top of the stack', + r8: '5th argument', r9: '6th argument', r10: 'scratch', r11: 'scratch', r12: 'callee-saved', r13: 'callee-saved', r14: 'callee-saved', r15: 'callee-saved', +}; + +const SETUP_FORMS: RefEntry[] = [ + { term: 'rdi = 0x1000', meaning: 'any register, at any width: rdi, edi, di, dil' }, + { term: "dl = 'A'", meaning: 'a character is its ASCII code' }, + { term: 'memory 0x1000 "GATTACA"', meaning: 'text, one byte per letter; \\n and \\t work' }, + { term: 'memory 0x2000 bytes 1 2 3 255', meaning: 'individual bytes' }, + { term: 'memory 0x3000 qwords 3 11 14', meaning: '8-byte values, little-endian' }, + { term: 'memory 0x4000 zero 64', meaning: 'a run of zeros, for output' }, + { term: 'stdin "what read returns"', meaning: 'input for the read system call' }, + { term: 'flags CF ZF', meaning: 'flags that start set' }, +]; + +function regEntries(): RefEntry[] { + const widths = (r: string) => { + if (/^r\d+$/.test(r)) return `${r}, ${r}d, ${r}w, ${r}b`; + const m: Record = { rax: 'rax, eax, ax, al / ah', rbx: 'rbx, ebx, bx, bl / bh', rcx: 'rcx, ecx, cx, cl / ch', rdx: 'rdx, edx, dx, dl / dh', rsi: 'rsi, esi, si, sil', rdi: 'rdi, edi, di, dil', rbp: 'rbp, ebp, bp, bpl', rsp: 'rsp, esp, sp, spl' }; + return m[r] || r; + }; + return (REG64 as string[]).map((r) => ({ term: r, meaning: REG_MEANING[r] || '', note: widths(r) })); +} + +export const ASM_REF: RefLanguage = { + id: 'asm', title: 'x86-64', blurb: 'Intel syntax, destination first, the course\'s instruction subset. Every program also gets a 512-byte stack; an address outside every region is a page fault.', + sections: [ + { id: 'asm-registers', title: 'Registers', intro: 'A 32-bit write zeroes the upper half; 8- and 16-bit writes leave the rest alone.', entries: regEntries(), columns: ['64-bit name', 'Role (System V convention)', 'Narrower names'] }, + { id: 'asm-instructions', title: 'Instructions', entries: Object.entries(OPS as Record).map(([op, d]) => ({ term: op, meaning: d.gloss || '', note: d.arity != null ? `${d.arity} operand${d.arity === 1 ? '' : 's'}` : '' })), columns: ['Mnemonic', 'What it does', 'Operands'] }, + { id: 'asm-conditions', title: 'Condition codes', intro: 'Append to j, set, or cmov: je, setg, cmovae.', entries: Object.entries(CONDITION_TEXT as Record).map(([cc, text]) => ({ term: cc, meaning: text })), columns: ['Code', 'Meaning'] }, + { + id: 'asm-flags', title: 'Flags', entries: [ + { term: 'ZF', meaning: 'the result was zero' }, { term: 'SF', meaning: 'the result\'s top bit is set (negative when read as signed)' }, { term: 'CF', meaning: 'an unsigned carry out, or borrow in' }, { term: 'OF', meaning: 'signed overflow: the sign of the result is wrong' }, { term: 'PF', meaning: 'the low byte of the result has an even number of one bits; after ucomisd, set when either side was NaN' }, + ], columns: ['Flag', 'Set when'] + }, + { id: 'asm-syscalls', title: 'System calls', intro: 'Number in rax, arguments in rdi, rsi, rdx; the answer returns in rax; rcx and r11 are clobbered.', entries: Object.entries(SYSCALLS as Record).map(([n, s]) => ({ term: `${n}`, meaning: s.name ? `${s.name}${s.gloss ? ': ' + s.gloss : ''}` : String(s) })), columns: ['rax', 'Call'] }, + { id: 'asm-setup', title: 'The setup box', entries: SETUP_FORMS, columns: ['Line', 'Meaning'] }, + { id: 'asm-machines', title: 'Machine presets', entries: Object.values(PRESETS as Record).map((m) => ({ term: m.name, meaning: m.description || '', note: m.caches?.l1d ? `L1d ${m.caches.l1d.size} B, ${m.caches.l1d.line}-byte lines, ${m.caches.l1d.ways}-way, ${m.caches.l1d.latency} cycles` : '' })), columns: ['Preset', 'Description', 'L1d'] }, + ], +}; + +/* ------------------------------------------------------------------ */ +/* matho (equations, written in LaTeX) */ +/* ------------------------------------------------------------------ */ + +/** What the matho pane reads, with an example of each; every example is run through the translator by the tests. */ +export const MATHO_FORMS: RefEntry[] = [ + { term: 'f(x) = …', meaning: 'An equation with a function on the left defines FUNCTION F(X : FLOAT) RETURNS FLOAT. Names used on the right but not named on the left become extra parameters, with a note.', example: 'f(x) = a x^2 + b' }, + { term: 'name = …', meaning: 'A plain name on the left names the function; its parameters are the free names on the right, in order of appearance. \\bar{x} → X_BAR, E_k → E_K, \\sigma^2 → SIGMA_SQUARED.', example: '\\bar{x} = \\frac{1}{n} \\sum_{i=0}^{n-1} x_i' }, + { term: 'x_i', meaning: 'A subscript that is an index (bound by a sum, or free on the left side) makes the name a LIST OF FLOAT and the subscript a position. Any other subscript is part of the name: v_0 is V0, x_{max} is XMax.', example: 'h(t) = v_0 t - \\frac{1}{2} g t^2' }, + { term: '\\sum_{i=a}^{b} …', meaning: 'A FOR loop with an accumulator, Total. The upper bound is inclusive in mathematics and exclusive in pseudo, so n-1 becomes TO N. From i = 1, a subscript i becomes the position I - 1, with a note.', example: 's = \\sum_{i=1}^{n} x_i^2' }, + { term: '\\sum_i … \\sum_{x \\in S} …', meaning: 'With no bounds, the loop runs over every position of the list the index walks. Over a set, it is FOR EACH.', example: 'm = \\sum_{x \\in S} x' }, + { term: '\\prod, n!', meaning: 'A product loop with an accumulator starting at 1.0; n! is the same loop from 2 to n.', example: 'c = \\frac{n!}{k! (n-k)!}' }, + { term: '\\frac{a}{b}, \\sqrt{x}, x^k, e^x', meaning: 'Division, SQRT, ^ (x^{1/2} is SQRT too), and EXP. A plain e is a name; e^{…} is the exponential.', example: 'f(x) = \\frac{1}{\\sqrt{2\\pi\\sigma^2}} e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}' }, + { term: '|x|, \\lfloor x \\rfloor, \\lceil x \\rceil, \\bmod', meaning: 'ABS, FLOOR, CEIL and MOD. MOD makes its operands INTEGER; FLOOR and CEIL give a whole number from any number.', example: 'r = \\lfloor n / 2 \\rfloor + a \\bmod b' }, + { term: '\\sin \\cos \\tan \\exp \\ln \\log \\max \\min', meaning: 'The built-ins of the same name. \\log is LOG10; \\log_2 is LOG2; \\log_b x is LOG(x) / LOG(b). \\sin^2 x is SIN(X) ^ 2.', example: 'y = \\sin^2 \\theta + \\log_{10} n' }, + { term: '\\binom{n}{k}', meaning: 'CHOOSE(N, K); both become INTEGER.', example: 'P(k) = \\binom{n}{k} p^k (1-p)^{n-k}' }, + { term: '\\vec{a} \\cdot \\vec{b}, \\|\\vec{a}\\|', meaning: 'A \\vec or \\mathbf name is a LIST OF FLOAT; \\cdot between two of them is DOT and the double bar is NORM_OF.', example: '\\cos\\theta = \\frac{\\vec{a} \\cdot \\vec{b}}{\\|\\vec{a}\\| \\|\\vec{b}\\|}' }, + { term: 'A_{ij}', meaning: 'Two index letters in one subscript: a LIST OF LIST OF FLOAT, indexed A[I][J].', example: 'y_i = \\sum_{j=0}^{n-1} A_{ij} x_j' }, + { term: '\\begin{cases} … \\end{cases}', meaning: 'IF / ELSEIF / ELSE. Conditions use < \\le > \\ge = \\ne, chains such as 0 \\le x < 1, and \\text{and} / \\text{or}. The last case must be the one with no condition.', example: 'f(x) = \\begin{cases} 0 & \\text{if } x < 0 \\\\ x & \\text{otherwise} \\end{cases}' }, + { term: 'an earlier equation\'s name', meaning: 'A name defined by an equation higher up is a call to that function, with its parameters joining this one\'s. Inside a loop, a call whose arguments do not change is computed once, before the loop, as a local.', example: 'g(x) = x^2 + 1\nh(x) = 2 g(x) + c' }, + { term: 'Greek letters, \\text{…}, \\bar \\hat \\tilde', meaning: 'Names: \\sigma is Sigma, \\text{pH} is PH, \\bar{x} is XBar. Two symbols that would be one pseudo name (p and P) are told apart with a suffix, and a note says so.', example: '\\text{pH} = -\\log_{10} [H]' }, +]; + +export const MATHO_REFUSALS: RefEntry[] = [ + { term: '\\int', meaning: 'An integral has no finite procedure behind it. Write the sum it approximates: \\sum_{i=0}^{n-1} f(a + i h) h with a step h, which becomes a loop.' }, + { term: '\\infty', meaning: 'A program cannot run to infinity. Give the sum or product an upper bound, such as n, and make n a parameter.' }, + { term: '\\lim, \\partial, \\nabla, f\'', meaning: 'Limits and derivatives are not procedures. Write the difference quotient (f(x + h) - f(x)) / h for a step h, or pick the value of n you mean.' }, + { term: '\\forall, \\exists', meaning: 'A statement about all or some values is a specification, not a procedure: it belongs in the speco pane as an ENSURE, or becomes the loop that searches.' }, + { term: 'a = b = c, x < 0 on its own, \\pm, \\approx, \\propto, \\sim', meaning: 'Statements and relations rather than definitions: keep the two sides you want computed, split ± into two equations, name the constant a proportionality leaves out, write RANDN() or RAND(x) for a draw.' }, +]; + +export const MATHO_REF: RefLanguage = { + id: 'matho', title: 'matho', blurb: 'Equations as a source language, written in LaTeX. Each equation becomes a FUNCTION in pseudo; sums become loops, subscripts become positions, cases become IF. Every decision the translation makes (which letters are whole numbers, which are lists, where a 1-based sum meets a 0-based list) is a note on the equation\'s line.', + sections: [ + { id: 'matho-forms', title: 'What it reads', entries: MATHO_FORMS, columns: ['Written', 'Becomes', 'Example'] }, + { id: 'matho-what-a-program-can-be', title: 'What it refuses, and what to write instead', intro: 'Some mathematics has no finite procedure behind it. The reader says so, and says what the procedure would be.', entries: MATHO_REFUSALS, columns: ['Written', 'Why, and what to write'] }, + ], +}; + +export const REFERENCE: RefLanguage[] = [LINGO_REF, MATHO_REF, PSEUDO_REF, SPECO_REF, ASM_REF]; + +/* ------------------------------------------------------------------ */ +/* Hover explanations */ +/* ------------------------------------------------------------------ */ + +export interface Explanation { title: string; body: string; where?: string } + +const LINGO_WORDS: Record = { + 'up to but not including': 'The range stops just before this value: the last position of T is one less than its length.', 'up to and including': 'The range reaches this value; the end is the last one visited.', 'down to but not including': 'Counts down and stops just before this value.', 'down to and including': 'Counts down through this value.', 'for each position': 'Visits every position 0, 1, …, length-1 of a text or list.', 'for each letter': 'Visits every letter of a text, without its position.', 'the letter of': '"the letter of T at i": the character at position i, counting from 0.', 'the length of': 'How many letters or items.', 'start with': 'Gives a name its first value.', 'the answer is': 'Ends the routine with that value.', 'stop and answer': 'Ends the routine at once with that value, even inside a loop.', 'stop the loop': 'Leaves the loop (BREAK).', 'skip to the next': 'Starts the next pass (CONTINUE).', 'an empty list called': 'Makes a new empty list with that name.', 'an empty tally called': 'Makes a new empty tally (a HASHMAP from keys to counts).', 'the number of positions': 'Counts the positions where a condition holds: a set written by rule, measured.', 'is one of': 'Membership in a short list of values.', 'is neither': 'Not either of two values.', 'for every': 'A FORALL over a range.', 'to the end of': 'Appends to a text or list.', 'to the front of': 'Prepends to a text.', 'repeat while': 'Tests before each pass (WHILE).', + to: 'Opens a routine: "To with :". The names used in the steps are its inputs.', start: '"Start with Count at 0." gives a name its first value; "Start with an empty list called Starts."', set: '"Set Best to i." changes a value.', add: '"Add 1 to Count." increases a number; "Add i to Starts." appends to a list; "Add b to the end of R." extends a text.', subtract: '"Subtract 1 from Left."', count: '"Count the word in Counts." tallies a key.', for: '"For each position i of T:", "For each letter b of T:", "For each i from A up to but not including B:"', each: 'Part of "For each …".', repeat: '"Repeat while :" tests before each pass.', while: 'Part of "Repeat while …".', if: '"If , ." or "If :" with steps under it.', otherwise: 'The steps that run when the If above did not.', stop: '"Stop the loop." leaves the loop; "Stop and answer i." ends the routine.', skip: '"Skip to the next position."', the: '"The answer is ." reports; "The is ." defines a phrase.', answer: '"The answer is ." ends the routine with that value.', when: '" when ." defines a yes-or-no phrase.', letter: '"the letter of T at i": the character at a position, counting from 0.', length: '"the length of T": how many letters.', position: '"For each position i of T" visits 0, 1, …, length-1.', up: '"up to but not including B" stops just before B; "up to and including B" reaches it.', including: 'Decides whether the end of a range counts.', including_: '', empty: '"T is empty" tests for length 0.', neither: '"is neither \'G\' nor \'C\'": not one of them.', one: '"is one of \'G\', \'C\'" is membership.', every: '"for every j from A up to but not including B, " is a FORALL.', larger: '"the larger of A and B".', smaller: '"the smaller of A and B".', number: '"the number of positions k of T where " counts.', show: '"Show ." prints it.', check: '"Check that ." is an ASSERT.', +}; + +export function explain(lang: LangId, token: string, lineText = ''): Explanation | null { + const t = token.trim(); + if (!t) return null; + switch (lang) { + case 'pseudo': { + const up = t.toUpperCase(); + if (KEYWORD_MEANING[up] && ((KEYWORDS as Set).has(up) || up in CONTEXTUAL || up === 'SUM' || up === 'RESULT' || up === 'OLD' || up === 'BOOLEAN')) return { title: up, body: KEYWORD_MEANING[up], where: 'keyword' }; + const b = (BUILTINS as Record)[up]; + if (b) return { title: b.sig || up, body: b.doc || '', where: `built-in · ${b.group || ''}` }; + if (OPERATOR_MEANING[t]) return { title: t, body: OPERATOR_MEANING[t], where: 'operator' }; + if (/^'.'$/.test(t)) return { title: t, body: 'A CHAR: one character. Compare it against T[I], never against a "string".', where: 'literal' }; + if (/^".*"$/.test(t)) return { title: 'STRING', body: 'Any number of characters. LENGTH counts characters; positions start at 0.', where: 'literal' }; + if (/^\d+\.\d+/.test(t)) return { title: t, body: 'A FLOAT literal.', where: 'literal' }; + if (/^\d+$/.test(t)) return { title: t, body: 'An INTEGER literal.', where: 'literal' }; + if (/^\/\//.test(t)) return { title: 'comment', body: 'Runs to the end of the line. Say why, not what.' }; + return null; + } + case 'lingo': { + const lc = t.toLowerCase(); + const cmp = (COMPARISONS as Array<[string, string]>).find(([p]) => p === lc); + if (cmp) return { title: `"${lc}"`, body: `Means ${cmp[1]}.`, where: 'comparison' }; + if (LINGO_WORDS[lc]) return { title: lc, body: LINGO_WORDS[lc] }; + if (/^'.'$/.test(t)) return { title: t, body: 'A letter.' }; + if (/^".*"$/.test(t)) return { title: 'a text', body: 'A text in double quotes.' }; + if (/^\d+$/.test(t)) return { title: t, body: 'A whole number.' }; + if (/^[A-Z][A-Za-z0-9]*$/.test(t) || /^[a-z]$/.test(t)) return { title: t, body: 'A name: an input of the routine, or something a step gave a value to.', where: 'name' }; + return null; + } + case 'matho': { + if (!t.startsWith('\\')) { + if (/^\d/.test(t)) return { title: t, body: 'A number. Whole numbers stay whole; a decimal point makes a FLOAT.', where: 'literal' }; + if (/^[A-Za-z]$/.test(t)) return { title: t, body: `A name. On its own it becomes the parameter ${t.toUpperCase()}; with an index subscript (${t}_i) it becomes a LIST OF FLOAT; bound by a sum it becomes the loop counter.`, where: 'name' }; + return null; + } + const cmd = t.slice(1); + const refused = MATHO_REFUSALS.find((e) => e.term.split(/,\s*/).some((x) => x.replace(/^\\/, '') === cmd)); + if (refused) return { title: t, body: refused.meaning, where: 'refused' }; + const form = MATHO_FORMS.find((e) => new RegExp(`\\\\${cmd}(?![A-Za-z])`).test(e.term)); + if (form) return { title: form.term, body: form.meaning, where: 'LaTeX' }; + const greek: Record = { alpha: 'Alpha', beta: 'Beta', gamma: 'Gamma', delta: 'Delta', theta: 'Theta', lambda: 'Lambda', mu: 'Mu', sigma: 'Sigma', tau: 'Tau', phi: 'Phi', omega: 'Omega', pi: 'PI()' }; + if (greek[cmd]) return { title: t, body: cmd === 'pi' ? 'The constant: PI().' : `A Greek letter is a name: ${greek[cmd]}.`, where: 'name' }; + return null; + } + case 'speco': { + const sym = SPECO_SYMBOLS.find((s) => s.math === t || s.ascii === t.toUpperCase() || (s.math === '|X|' && t === '|') || (s.math === 'result' && t.toLowerCase() === 'result') || (s.ascii === 'OLD(X)' && t.toUpperCase() === 'OLD')); + if (sym) return { title: `${sym.math} ${sym.ascii}`, body: sym.meaning, where: 'speco' }; + const kw: Record = { SPEC: SPECO_FORMS[0].meaning, REQUIRE: SPECO_FORMS[1].meaning, ENSURE: SPECO_FORMS[2].meaning, LOOP: SPECO_FORMS[3].meaning, INVARIANT: SPECO_FORMS[4].meaning, DECREASES: SPECO_FORMS[5].meaning }; + if (kw[t.toUpperCase()]) return { title: t.toUpperCase(), body: kw[t.toUpperCase()], where: 'speco line' }; + return explain('pseudo', t, lineText); + } + case 'asm': { + const lc = t.toLowerCase(); + const op = (OPS as Record)[lc]; + if (op) return { title: lc, body: op.gloss || '', where: 'instruction' }; + const ccm = /^(j|set|cmov)([a-z]{1,3})$/.exec(lc); + if (ccm && (CONDITION_TEXT as Record)[ccm[2]]) return { title: lc, body: `${ccm[1] === 'j' ? 'jump' : ccm[1] === 'set' ? 'set the byte to 1' : 'copy'} if ${(CONDITION_TEXT as Record)[ccm[2]]}`, where: 'conditional' }; + const base = regBase(lc); + if (base) return { title: lc, body: `${REG_MEANING[base] || 'general register'}${base !== lc ? ` (the ${widthOf(lc)} part of ${base})` : ''}`, where: 'register' }; + if (/^(byte|word|dword|qword|xmmword)$/.test(lc)) return { title: `${lc} ptr`, body: `Says how many bytes to fetch: ${({ byte: 1, word: 2, dword: 4, qword: 8, xmmword: 16 } as any)[lc]}.` }; + if (/^(ZF|SF|CF|OF)$/i.test(t)) return { title: t.toUpperCase(), body: (ASM_REF.sections.find((s) => s.id === 'asm-flags')!.entries.find((e) => e.term === t.toUpperCase())?.meaning) || '' }; + if (/^\.?[A-Za-z_$][\w$.]*:?$/.test(t) && /:$/.test(t)) return { title: t.replace(/:$/, ''), body: 'A label: a name for the address of the next instruction.' }; + if (/^0x[0-9a-f]+$/i.test(t)) return { title: t, body: `${parseInt(t, 16)} in decimal.` }; + return null; + } + default: + return null; + } +} + +function regBase(name: string): string | null { + const m: Record = { eax: 'rax', ax: 'rax', al: 'rax', ah: 'rax', ebx: 'rbx', bx: 'rbx', bl: 'rbx', bh: 'rbx', ecx: 'rcx', cx: 'rcx', cl: 'rcx', ch: 'rcx', edx: 'rdx', dx: 'rdx', dl: 'rdx', dh: 'rdx', esi: 'rsi', si: 'rsi', sil: 'rsi', edi: 'rdi', di: 'rdi', dil: 'rdi', ebp: 'rbp', bp: 'rbp', bpl: 'rbp', esp: 'rsp', sp: 'rsp', spl: 'rsp' }; + if ((REG64 as string[]).includes(name)) return name; + if (m[name]) return m[name]; + const r = /^(r(?:8|9|1[0-5]))[dwb]$/.exec(name); + return r ? r[1] : null; +} +function widthOf(name: string): string { + if (/^e|d$/.test(name) && !/^(d[lhix]|dil)$/.test(name)) return 'low 32-bit'; + if (/^[abcd]h$/.test(name)) return 'second byte'; + if (/[lb]$/.test(name)) return 'low byte'; + return 'low 16-bit'; +} diff --git a/src/common/runview.ts b/src/common/runview.ts new file mode 100644 index 0000000..a4626af --- /dev/null +++ b/src/common/runview.ts @@ -0,0 +1,244 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The run, read top to bottom (the Developer and Bridge areas' Run view): one row per step, and a column for each + * variable the student chose to follow, holding its value after that step, with the lifecycle of the value it + * names (made, copied, moved in, moved out, destroyed) marked in the same cell. It replaces four views that each + * showed part of this (the variables now, the trace table, the table of values, the lifetimes). + * + * DOM-free: src/ui/views/run.ts draws it; tests/runview.test.mjs checks it against the interpreters' traces. + */ +import type { Trace } from './types.js'; +import { formatValue, UNSET } from '../engines/pseudo/lang.js'; + +/** A variable as the run view knows it: its name in a routine. Recursion's frames of one routine share a column. */ +export interface RunColumn { + /** `routine.name`, the column's identity. */ + key: string; + fn: string; + name: string; + /** The type as the interpreter recorded it, in words. */ + type: string; + /** How many steps write it, and the first. */ + writes: number; + firstStep: number; +} + +export type LifeKind = 'made' | 'copied' | 'moved-in' | 'moved-out' | 'destroyed'; +export interface LifeMark { kind: LifeKind; text: string } + +export interface RunCell { + /** The value after the step, as the language prints it; null when the variable is not in scope (or has no value yet). */ + text: string | null; + /** This step wrote it, or reads it. */ + wrote: boolean; + read: boolean; + /** What happened to the value it names during this step (Mojo values whose lifecycle can be seen). */ + life: LifeMark[]; + /** The positions this step wrote inside it (`2` for L[2], `5, 6` for D[5, 6]), when it wrote an element. */ + at: string[]; +} + +export interface RunRow { + /** 0-based step index; the stepper's position after it is `step + 1`. */ + step: number; + line: number | null; + text: string; + kind: string; + depth: number; + /** Names written (with a position, `L[2]`, when one element was). */ + writes: string[]; + cells: Record; + /** Lifecycle events of values no column names: temporaries, fields. */ + otherLife: string[]; + /** Lines printed by this step. */ + printed: string[]; +} + +const frameName = (f: any): string => (f?.name ? String(f.name) : 'main'); + +export function typeText(t: any): string { + if (!t) return ''; + if (typeof t === 'string') return t; + if (t.k === 'LIST') return `LIST OF ${typeText(t.of)}`; + if (t.k === 'SET') return `SET OF ${typeText(t.of)}`; + return t.k || t.name || ''; +} + +/** A value as its language prints it, or null for "no value yet". Mojo values carry their own display. */ +export function valueText(v: any): string | null { + if (!v) return null; + if (typeof v.text === 'string') return v.text; + if (v.value === undefined || v.value === UNSET || typeof v.value === 'symbol') return null; + return formatValue(v.value, v.type); +} + +/** The newest frame of routine `fn` in a state, the one a column reads (recursion shows its innermost call). */ +function frameOf(state: any, fn: string): any { + const frames = (state?.frames ?? []).filter((f: any) => frameName(f) === fn); + return frames[frames.length - 1] ?? null; +} + +/** Every variable that appears anywhere in the run, in order of first appearance. */ +export function runColumns(trace: Trace): RunColumn[] { + if (trace.lang === 'asm') return []; + const cols = new Map(); + const states = trace.states as any[]; + for (let i = 0; i < states.length; i++) { + for (const f of states[i]?.frames ?? []) { + for (const v of Object.values(f.vars ?? {}) as any[]) { + const key = `${frameName(f)}.${v.name}`; + if (!cols.has(key)) cols.set(key, { key, fn: frameName(f), name: v.name, type: typeText(v.type), writes: 0, firstStep: Math.max(0, i - 1) }); + } + } + } + (trace.steps as any[]).forEach((s, i) => { + const st = trace.states[i + 1] as any; + for (const w of s.writes ?? []) { + const f = (st?.frames ?? []).find((x: any) => w.frame == null || x.id === w.frame); + const c = cols.get(`${frameName(f)}.${w.name}`) ?? [...cols.values()].find((x) => x.name === w.name); + if (c) c.writes++; + } + }); + return [...cols.values()]; +} + +/** + * The columns to follow when the student has not chosen: the variables of the routine that does the most work + * (the most writes to its variables; `self` and `_` do not count), most changed first, up to `max`, shown in the + * order they appear. A routine with nothing to change (a `main` that only calls) gives way to one that does. + */ +export function defaultFollow(cols: RunColumn[], max = 5): string[] { + const plain = cols.filter((c) => c.name !== 'self' && c.name !== '_'); + const work = new Map(); + for (const c of plain) work.set(c.fn, (work.get(c.fn) ?? 0) + c.writes); + const fn = [...work].sort((a, b) => b[1] - a[1])[0]?.[0]; + if (fn == null) return []; + const mine = plain.filter((c) => c.fn === fn); + const pick = [...mine.filter((c) => c.writes > 0).sort((a, b) => b.writes - a.writes || a.firstStep - b.firstStep), ...mine.filter((c) => c.writes === 0)].slice(0, max); + return pick.sort((a, b) => a.firstStep - b.firstStep).map((c) => c.key); +} + +/** Which value (by its id in the run) each variable names, and when: from the interpreter's `named` events. */ +function namesOfValues(trace: Trace): Map { + const out = new Map(); + (trace.steps as any[]).forEach((s) => { + for (const e of s.life ?? []) { + if (e.kind !== 'named') continue; + const key = `${s.sub ?? 'main'}.${e.name}`; + const list = out.get(e.id) ?? []; + if (!list.includes(key)) list.push(key); + out.set(e.id, list); + } + }); + return out; +} + +/** + * The rows for steps [from, to) with cells for the followed columns. Values are those after each step; a + * variable not in scope after the step is blank. `onlyWrites` keeps the steps that change something, print, or + * end a routine, as the old trace table's filter did. + */ +export function runRows(trace: Trace, follow: string[], from = 0, to = trace.count, opts: { onlyWrites?: boolean } = {}): RunRow[] { + if (trace.lang === 'asm') return []; + const steps = trace.steps as any[]; + const states = trace.states as any[]; + const named = namesOfValues(trace); + const cols = follow.map((key) => { const dot = key.indexOf('.'); return { key, fn: key.slice(0, dot), name: key.slice(dot + 1) }; }); + const rows: RunRow[] = []; + for (let i = Math.max(0, from); i < Math.min(to, trace.count); i++) { + const s = steps[i]; + const before = states[i]; + // a step that stops the run is recorded after the frames unwound: its values are the ones it stopped on + const after = s.kind === 'error' && !(states[i + 1]?.frames ?? []).length ? before : states[i + 1]; + const printed = (trace.output ?? []).slice(before?.outputLength ?? 0, after?.outputLength ?? 0); + if (opts.onlyWrites && !(s.writes?.length || printed.length || s.kind === 'return' || s.kind === 'error' || s.life?.length)) continue; + const cells: Record = {}; + for (const c of cols) { + const f = frameOf(after, c.fn); + const v = varIn(f?.vars, c.name); + const own = (a: any) => a.name === c.name && (a.frame == null || a.frame === f?.id); + cells[c.key] = { text: f && v ? valueText(v) : null, wrote: !!f && (s.writes ?? []).some(own), read: !!f && (s.reads ?? []).some(own), life: [], at: f ? [...new Set((s.writes ?? []).filter((w: any) => own(w) && w.path).map((w: any) => w.path.join(', ')))] : [] }; + } + const otherLife: string[] = []; + for (const e of s.life ?? []) { + if (e.kind === 'named') continue; + const mark = (id: number, kind: LifeKind, text: string) => { + const keys = (named.get(id) ?? []).filter((k) => cells[k]); + if (keys.length) for (const k of keys) cells[k].life.push({ kind, text }); + else otherLife.push(`#${id} ${text}`); + }; + if (e.kind === 'made') mark(e.id, 'made', `made: ${e.label}`); + else if (e.kind === 'copied') mark(e.id, 'copied', `copied from #${e.from}`); + else if (e.kind === 'moved') { mark(e.id, 'moved-in', `moved in from #${e.from}`); if (e.from != null) mark(e.from, 'moved-out', `moved out into #${e.id}`); } + else if (e.kind === 'destroyed') mark(e.id, 'destroyed', 'destroyed'); + } + rows.push({ + step: i, line: s.line ?? null, text: s.text || s.kind, kind: s.kind, depth: s.depth ?? (after?.frames?.length ?? 1) - 1, + writes: [...new Set((s.writes ?? []).map((w: any) => (w.path ? `${w.name}[${w.path.join(', ')}]` : w.name)))] as string[], + cells, otherLife, printed, + }); + } + return rows; +} + +/** + * A string or a list of plain values as a row of cells with their positions (the ruler a tutor draws), or null + * for anything else, a single character, or more than 80 cells. + */ +export function rulerCells(v: any): string[] | null { + if (!v) return null; + let cells: string[] | null = null; + const val = v.value; + if (typeof v.text === 'string') { + // Mojo: the interpreter's own values + if (val?.k === 'str') cells = [...val.v]; + else if (val?.k === 'list' && val.items.every((x: any) => ['int', 'bool', 'str', 'float'].includes(x?.k))) + cells = val.items.map((x: any) => (x.k === 'bool' ? (x.v ? 'True' : 'False') : String(x.v))); + } else if (typeof val === 'string') cells = [...val]; + else if (val && Array.isArray(val.dims) && Array.isArray(val.data) && val.dims.length === 1 && val.data.every((x: any) => typeof x !== 'object')) + cells = val.data.map((x: any) => (x === undefined || x === UNSET || typeof x === 'symbol' ? '' : formatValue(x, v.type?.of, { bareChar: true }))); + else if (Array.isArray(val) && val.every((x) => typeof x !== 'object')) cells = val.map((x) => formatValue(x, v.type?.of, { bareChar: true })); + return cells && cells.length > 1 && cells.length <= 80 ? cells : null; +} + +/** A list of lists of plain values (a table, a grid) as rows of cell texts, at most 16 by 16; null otherwise. */ +export function gridCells(v: any): string[][] | null { + if (!v || typeof v.text === 'string') { + const val = v?.value; + if (val?.k !== 'list' || !val.items.length || val.items.length > 16) return null; + if (!val.items.every((r: any) => r?.k === 'list' && r.items.length <= 16 && r.items.every((x: any) => ['int', 'bool', 'str', 'float'].includes(x?.k)))) return null; + return val.items.map((r: any) => r.items.map((x: any) => (x.k === 'bool' ? (x.v ? 'True' : 'False') : String(x.v)))); + } + const val = v.value; + // a pseudo ARRAY: its dimensions and its cells in row order + if (val && Array.isArray(val.dims) && Array.isArray(val.data)) { + if (val.dims.length !== 2 || val.dims[0] > 16 || val.dims[1] > 16) return null; + const [r, c] = val.dims; + return Array.from({ length: r }, (_, i) => val.data.slice(i * c, i * c + c).map((x: any) => (x === undefined || x === UNSET || typeof x === 'symbol' ? '' : formatValue(x, v.type?.of)))); + } + if (!Array.isArray(val) || !val.length || val.length > 16 || !val.every((r) => Array.isArray(r) && r.length <= 16)) return null; + return val.map((r: any[]) => r.map((x) => (x === undefined || x === UNSET || typeof x === 'symbol' ? '' : typeof x === 'object' ? '…' : formatValue(x, v.type?.of?.of)))); +} + +/** Each followed variable at stepper position `t` (after `t` steps): its text, and its ruler or grid when it has one. */ +export function valuesAt(trace: Trace, follow: string[], t: number): Array<{ key: string; text: string | null; ruler: string[] | null; grid: string[][] | null; inScope: boolean; type: string }> { + const at = Math.max(0, Math.min(t, trace.count)); + let st = (trace.states as any[])[at]; + if (at > 0 && (trace.steps as any[])[at - 1]?.kind === 'error' && !(st?.frames ?? []).length) st = (trace.states as any[])[at - 1]; + return follow.map((key) => { + const dot = key.indexOf('.'); + const f = frameOf(st, key.slice(0, dot)); + const v = varIn(f?.vars, key.slice(dot + 1)); + return { key, text: v ? valueText(v) : null, ruler: v ? rulerCells(v) : null, grid: v ? gridCells(v) : null, inScope: !!f && !!v, type: typeText(v?.type) }; + }); +} + +/** + * A frame's variable by the name the program writes. Pseudo keys a frame's variables by that name, and lingo (whose + * names are not case-sensitive) by the name in upper case, with the written name in `.name`: `Count` is under COUNT. + */ +export function varIn(vars: Record | undefined, name: string): any { + if (!vars) return undefined; + return vars[name] ?? vars[name.toUpperCase()] ?? Object.values(vars).find((v: any) => v?.name === name); +} diff --git a/src/common/types.ts b/src/common/types.ts new file mode 100644 index 0000000..fa19106 --- /dev/null +++ b/src/common/types.ts @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The two shared contracts every language service emits, and the service + * interface the shell programs against. Nothing in `ui/` imports an engine + * directly; it goes through these. + */ + +export type LangId = 'lingo' | 'matho' | 'pseudo' | 'speco' | 'mojo' | 'asm'; + +export type Severity = 'error' | 'warning' | 'note'; + +/** One finding, in the student's own line numbers for that language. */ +export interface Diagnostic { + lang: LangId; + severity: Severity; + /** A stable id such as `range-ends`, `unset-read`, `invariant-keep`. */ + id: string; + line: number; + col?: number; + message: string; + /** Two or more readings of an ambiguous sentence; each is a one-click rewrite. */ + question?: { text: string; options: QuickFix[] }; + /** A single fix with no question attached (a casing fix, a missing closer). */ + fix?: QuickFix; + /** Anchor into the pseudocode guide, e.g. `#for-stops-one-short`. */ + guide?: string; + /** Who says so, when it is not Programming+'s own checker: "the Mojo 1.2.0 compiler". */ + origin?: string; + /** A page that explains it (the Mojo manual, say), when `guide` does not apply. */ + docUrl?: string; +} + +export interface QuickFix { + label: string; + /** Replace this range of the source (1-based line, 1-based inclusive cols) with `text`. */ + line: number; + /** A fix that spans lines replaces from `line`/`col` to `endLine`/`endCol`; absent, it stays on `line`. */ + endLine?: number; + col: number; + endCol: number | null; + text: string; +} + +/** A recorded run: `states[t]` is the state before `steps[t]` runs. */ +export interface Trace { + lang: LangId; + steps: Step[]; + states: State[]; + count: number; + output: string[]; + error: { kind: string; message: string; line?: number | null } | null; + truncated: boolean; + /** How often each source line ran. */ + hits: Map; + /** The language's own trace object, for views that know the language. */ + raw: unknown; +} + +export interface Frame { + t: number; + state: State; + last: Step | null; + next: Step | null; +} + +export function frameAt(trace: Trace, t: number): Frame { + const c = Math.max(0, Math.min(t, trace.count)); + return { t: c, state: trace.states[c], last: c > 0 ? trace.steps[c - 1] : null, next: c < trace.count ? trace.steps[c] : null }; +} + +/** A line map from one language's source to another's. */ +export type LineMap = Array<{ from: number; to: number }>; + +export interface Lowered { + target: LangId; + source: string; + map: LineMap; + notes: string[]; +} + +export interface Obligation { + kind: string; + label: string; + line: number; + col?: number; + status: 'proved' | 'broken' | 'open'; + paths: number; + facts: string[]; + goal: string; + reason?: string; + counterexample?: string; +} + +export interface ProofReport { + units: Array<{ name: string; kind: string; line: number; obligations: Obligation[]; inferred: string[]; notes: string[]; partial: boolean }>; + counts: { proved: number; broken: number; open: number; total: number }; +} + +export interface PropertyReport { + name: string; + checked: number; + failures: Array<{ call: string; message: string; line?: number; kind: string }>; + problem?: string; +} + +/** What a parse hands back: an opaque program plus diagnostics. */ +export interface Parsed { + program: Program | null; + diagnostics: Diagnostic[]; + ok: boolean; +} + +export interface RunOptions { + /** A call such as `COUNT_BASE('A', "GATTACA")`, for programs with no main body. */ + call?: string | null; + input?: string[]; + maxSteps?: number; + /** Pseudo: the positions FORALL, EXISTS, SUM and sets written by rule may visit (default: 100 per step allowed). */ + maxVisits?: number; + /** Assembly: register and memory setup lines, in the sandbox's syntax. */ + setup?: string; + machine?: string; +} + +/** + * The language-service interface. Methods a language does not support are + * simply absent, and the shell hides the matching view. + */ +export interface LanguageService { + id: LangId; + name: string; + parse(source: string): Parsed; + run?(parsed: Parsed, options: RunOptions): Trace; + lower?(parsed: Parsed, target: LangId): Lowered | null; + prove?(parsed: Parsed): ProofReport; + checkProperties?(parsed: Parsed, name?: string): PropertyReport[]; + /** A default call for `run`, when the program has subroutines and no main body. */ + defaultCall?(parsed: Parsed): string | null; + /** Highlight classes per token, for the editor. */ + tokenize?(line: string): Array<{ text: string; cls: string }>; + /** The whole document at once, one token list per line, for languages whose strings span lines. */ + tokenizeLines?(source: string): Array>; +} + +export type ViewId = 'problems' | 'trace' | 'proof' | 'tests' | 'asm' | 'flow' | 'data' | 'table' | 'compiler' | 'life'; + +export const LANG_NAMES: Record = { + lingo: 'lingo', matho: 'matho', pseudo: 'pseudo', speco: 'speco', mojo: 'Mojo', asm: 'x86-64', +}; + +/** The app's three areas. Core names them only so an exercise can say where it opens (exercises/types.ts). */ +export type AreaId = 'bridge' | 'dev' | 'hw'; diff --git a/src/common/workspace.ts b/src/common/workspace.ts new file mode 100644 index 0000000..4c6ed1c --- /dev/null +++ b/src/common/workspace.ts @@ -0,0 +1,316 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * A workspace: the sources of one piece of work and everything they mean, memoized by the inputs it depends on. + * The linked panes kept in step by the hub (sync/hub.ts), each language parsed, the diagnostics, the run, the proof + * and the fuzzer's report. No DOM, no storage, no areas: what a program means is decided here, and a front end (the + * app's Session) adds where it is kept and how it is shown. + */ +import type { Diagnostic, LangId, Parsed, ProofReport, PropertyReport, Trace } from './types.js'; +import { SERVICES } from '../services/index.js'; +import { proveAgainst, type SpecoProgram } from '../services/speco.js'; +import type { MathoProgram } from '../services/matho.js'; +import { provePseudo, studyOf, defaultCall } from '../services/pseudo.js'; +import { mojoReadsInput, type MojoProgram } from '../services/mojo.js'; +import { runMojo } from '../mojo/interp.js'; +import { fuzzPseudo, fuzzMojo, WORK_PER_INPUT, type FuzzReport, type FuzzOptions } from './fuzz.js'; +import { SyncHub, LINKED, isLinked, mojoFromPseudo, FAULT_ID, type HubMaps, type Linked, type MojoDerived, type Sources, type SyncInfo } from '../sync/hub.js'; + +/** + * A pane whose reader throws (a fault in Programming+, past every reader's own limits) gets one diagnostic saying + * so, and nothing else in the workspace throws because of it: the last line of defense for text from a share link + * (tests/reader-limits.test.mjs). + */ +function faulted(lang: LangId, read: () => Parsed): Parsed { + try { return read(); } catch (e) { + return { program: null, ok: false, diagnostics: [faultDiagnostic(lang, e)] }; + } +} +function faultDiagnostic(lang: LangId, e: unknown): Diagnostic { + const what = String((e as Error)?.message ?? e).slice(0, 200); + return { lang, severity: 'error', id: FAULT_ID, line: 1, col: 1, message: `Programming+ could not read this pane (${what}). That is a fault in Programming+, not in the program; the other panes still work.` }; +} + +/** What a workspace reads: the sources and how to run them. The app's saved state is a superset of this. */ +export interface WorkState { + sources: Sources; + /** The hub's bookkeeping: which linked pane is the authority, which are stale, the student's spelling of names. */ + sync?: SyncInfo; + /** The pane that is active, which decides what runs. */ + lang: LangId; + /** The call to run, such as COUNT_BASE('A', "GATTACA"); absent: the program's own default. */ + call?: string | null; + /** The run's input, one value per line. */ + input?: string; + /** For x86-64: the setup block, and the machine model. */ + setup?: string; + machine?: string; +} + +/** The routines of one program the fuzzer tests (Workspace.fuzz), as for Mojo's generated inputs (docs/limits.md). */ +export const FUZZ_ROUTINES = 100; +/** The work all the routines of one program may take together, in inputs at the default budget of 300 an input. */ +export const FUZZ_PROGRAM_INPUTS = 600; + +export interface ProofResult { report: ProofReport; wovenSource: string | null; wovenDiagnostics: Diagnostic[]; study: Parsed | null } + +export class Workspace { + state: WorkState; + protected memo = new Map(); + /** The last run error (a call that could not be resolved, a listing that did not assemble). */ + runError: string | null = null; + /** Line maps between the linked panes and the pseudo, from the last cascade (or the fill that opened the problem). */ + protected maps: HubMaps = { up: {}, down: {} }; + + constructor(state: WorkState) { this.state = state; } + + protected cached(name: string, key: string, make: () => T): T { + const hit = this.memo.get(name); + if (hit && hit.key === key) return hit.value as T; + const value = make(); + this.memo.set(name, { key, value }); + return value; + } + + /** Whether the Mojo pane shows the pseudo's literal translation instead of its own text (the app's Bridge). */ + protected derivesMojo(): boolean { return false; } + + /* ---------- editing ---------- */ + + /** + * Pane `lang` now holds `text`. For the four linked languages the hub makes the other panes follow in the same + * change; Mojo and x86-64 are stored as they are. `force` runs the cascade even when the text is what the pane + * already shows (a stale pane chosen to edit becomes the authority). Returns whether anything changed. + */ + edit(lang: LangId, text: string, force = false): boolean { + if (isLinked(lang)) { + if (!force && this.source(lang) === text) return false; + const r = new SyncHub(this.linkedSources(), this.state.sync, { call: this.state.call ?? null }).edit(lang, text); + // a pane that was absent and is still empty stays absent, so editing a program back to the problem's own text is no edit + for (const l of LINKED) if (r.sources[l] !== undefined && (r.sources[l] !== '' || this.state.sources[l] !== undefined)) this.state.sources[l] = r.sources[l]; + this.state.sync = { authority: r.authority, stale: r.stale, names: r.names }; + this.maps = r.maps; + return true; + } + if (this.state.sources[lang] === text) return false; + this.state.sources[lang] = text; + return true; + } + + /** Make a stale pane the authority without changing its text: the others are regenerated from it. */ + takeAuthority(lang: Linked): boolean { return this.edit(lang, this.source(lang), true); } + + /** The linked panes as shown: what is written, with an empty one generated from the pseudo until the first edit. */ + protected fill(): { sources: Sources; stale: Partial>; maps: HubMaps } { + if (this.state.sync) return { sources: this.state.sources, stale: {}, maps: this.maps }; + const key = JSON.stringify(LINKED.map((l) => this.state.sources[l] ?? '')); + return this.cached('fill', key, () => new SyncHub(this.state.sources).fill()); + } + protected linkedSources(): Sources { return { ...this.state.sources, ...this.fill().sources }; } + + lineMaps(): HubMaps { + const base: HubMaps = this.state.sync ? this.maps : (() => { const f = this.fill().maps; return { up: f.up, down: { ...f.down, ...this.maps.down } }; })(); + return { up: base.up, down: { ...base.down, mojo: this.derivedMojo().map } }; + } + /** Where a pane stands against the others: as authored (never synced), in step, or stale with the reason. */ + linkState(lang: Linked): { kind: 'authored' | 'sync' | 'stale' | 'authority'; reason?: string } { + const sync = this.state.sync; + const reason = sync ? sync.stale?.[lang] : this.fill().stale[lang]; + if (reason) return { kind: 'stale', reason }; + if (!sync) return { kind: 'authored' }; + return { kind: sync.authority === lang ? 'authority' : 'sync' }; + } + + /** The literal Mojo for the pseudo now in play, labeled as generated by every view that shows it. Memoized by the pseudo and the call. */ + derivedMojo(): MojoDerived { + const pseudo = this.source('pseudo'); + const call = this.state.call ?? null; + return this.cached('mojo.derived', JSON.stringify([pseudo, call]), () => mojoFromPseudo(pseudo, call)); + } + + /** The INPUT lines of a run: one per line of the input box, trailing blank lines dropped. */ + inputLines(): string[] { const lines = (this.state.input || '').split('\n'); while (lines.length && lines[lines.length - 1].trim() === '') lines.pop(); return lines; } + /** Whether the program that will run reads INPUT. */ + readsInput(): boolean { if (this.state.lang === 'mojo') return mojoReadsInput(this.source('mojo')); const l = this.traceLang(); const src = l === 'lingo' ? this.source('lingo') : l === 'matho' ? '' : this.source('pseudo'); return l !== 'asm' && /^\s*INPUT\b/mi.test(src.replace(/\/\/.*$/gm, '')); } + + /* ---------- derived ---------- */ + + /** The text of a pane as shown: a linked pane may be generated from the pseudo until the first edit (fill). */ + source(lang: LangId): string { + if (lang === 'mojo' && this.derivesMojo()) return this.derivedMojo().source; + return isLinked(lang) ? (this.fill().sources[lang] ?? '') : (this.state.sources[lang] ?? ''); + } + + parsedLingo(): Parsed | null { + const src = this.source('lingo'); + if (!src.trim()) return null; + return this.cached('lingo', src, () => faulted('lingo', () => SERVICES.lingo.parse(src))); + } + parsedMatho(): Parsed | null { + const src = this.source('matho'); + if (!src.trim()) return null; + return this.cached('matho', src, () => faulted('matho', () => SERVICES.matho.parse(src) as Parsed)); + } + parsedPseudo(): Parsed | null { + const source = this.source('pseudo'); + if (!source.trim()) return null; + return this.cached('pseudo', source, () => faulted('pseudo', () => studyOf(source))); + } + parsedSpeco(): Parsed | null { + const src = this.source('speco'); + if (!src.trim()) return null; + return this.cached('speco', src, () => faulted('speco', () => SERVICES.speco.parse(src) as Parsed)); + } + parsedMojo(): Parsed | null { + const src = this.source('mojo'); + if (!src.trim()) return null; + return this.cached('mojo', src, () => faulted('mojo', () => SERVICES.mojo.parse(src) as Parsed)); + } + parsedAsm(): Parsed | null { + const src = this.source('asm'); + if (!src.trim()) return null; + return this.cached('asm', src, () => faulted('asm', () => SERVICES.asm.parse(src))); + } + + /** What each language's own reader says about its pane. */ + diagnostics(lang: LangId): Diagnostic[] { + switch (lang) { + case 'lingo': return this.parsedLingo()?.diagnostics ?? []; + case 'matho': return this.parsedMatho()?.diagnostics ?? []; + case 'pseudo': return this.parsedPseudo()?.diagnostics ?? []; + case 'speco': { const m = this.parsedSpeco(); const w = this.proof(); return [...(m?.diagnostics ?? []), ...(w?.wovenDiagnostics ?? []), ...(this.proofFault ? [this.proofFault] : [])]; } + case 'mojo': return this.parsedMojo()?.diagnostics ?? []; + case 'asm': return this.parsedAsm()?.diagnostics ?? []; + } + } + + allDiagnostics(): Diagnostic[] { + return (['lingo', 'matho', 'pseudo', 'speco', 'mojo', 'asm'] as LangId[]).flatMap((l) => this.diagnostics(l)); + } + + /** Which language a run steps: the active pane's, or the pseudocode behind it. */ + traceLang(): LangId { + const l = this.state.lang; + if (l === 'asm') return 'asm'; + if (l === 'mojo') return 'mojo'; + // a pane that is not in step holds text the other panes have moved on from: step the pseudo instead + if (l === 'lingo' && this.linkState('lingo').kind !== 'stale' && this.parsedLingo()?.ok) return 'lingo'; + if (l === 'matho' && this.linkState('matho').kind !== 'stale' && this.parsedMatho()?.ok) return 'matho'; + return 'pseudo'; + } + + callText(): string | null { + if (this.state.call !== undefined && this.state.call !== null && this.state.call !== '') return this.state.call; + if (this.traceLang() === 'matho') { const r = this.parsedMatho(); return r && SERVICES.matho.defaultCall ? SERVICES.matho.defaultCall(r) : null; } + const p = this.parsedPseudo(); + return p?.program ? defaultCall(p.program) : null; + } + + trace(): Trace | null { + this.runError = null; + try { + const lang = this.traceLang(); + if (lang === 'mojo') { + const p = this.parsedMojo(); + if (!p) return null; + if (!p.ok) { this.runError = 'Fix the problems in the Mojo pane first; the Problems view lists them.'; return null; } + const key = JSON.stringify([this.source('mojo'), this.state.input]); + return this.cached('trace.mojo', key, () => runMojo(p.program!.module, this.source('mojo'), { input: this.inputLines(), maxSteps: 20000 }).trace); + } + if (lang === 'asm') { + const p = this.parsedAsm(); + if (!p || !p.ok) return null; + const key = JSON.stringify([this.source('asm'), this.state.setup, this.state.machine]); + return this.cached('trace.asm', key, () => SERVICES.asm.run!(p, { setup: this.state.setup, machine: this.state.machine, maxSteps: 20000 })); + } + if (lang === 'matho') { + const r = this.parsedMatho(); + if (!r || !r.ok) return null; + const key = JSON.stringify([this.source('matho'), this.state.call, this.state.input]); + return this.cached('trace.matho', key, () => SERVICES.matho.run!(r, { call: this.state.call ?? null, input: this.inputLines(), maxSteps: 20000 })); + } + if (lang === 'lingo') { + const r = this.parsedLingo(); + if (!r || !r.ok) return null; + const key = JSON.stringify([this.source('lingo'), this.state.call, this.state.input]); + return this.cached('trace.lingo', key, () => SERVICES.lingo.run!(r, { call: this.state.call ?? null, input: this.inputLines(), maxSteps: 20000 })); + } + const p = this.parsedPseudo(); + if (!p || !p.ok) return null; + const key = JSON.stringify([this.source('pseudo'), this.state.call, this.state.input]); + return this.cached('trace.pseudo', key, () => SERVICES.pseudo.run!(p, { call: this.state.call ?? null, input: this.inputLines(), maxSteps: 20000 })); + } catch (e: any) { + this.runError = e?.message || String(e); + return null; + } + } + + /** The prover's own failure, said on the speco pane (proof() is then null, as for a program that does not check). */ + proofFault: Diagnostic | null = null; + + proof(): ProofResult | null { + const p = this.parsedPseudo(); + if (!p || !p.ok) return null; + const m = this.parsedSpeco(); + const key = JSON.stringify([this.source('pseudo'), this.source('speco')]); + const r = this.cached('proof', key, () => { + try { return { result: this.proofOf(p, m), fault: null }; } catch (e) { return { result: null, fault: faultDiagnostic('speco', e) }; } + }); + this.proofFault = r?.fault ?? null; + return r?.result ?? null; + } + + private proofOf(p: Parsed, m: Parsed | null): ProofResult { + if (m && m.ok && m.program!.spec.units.length) { + const r = proveAgainst(m, this.source('pseudo')); + return { report: r.report, wovenSource: r.woven.source, wovenDiagnostics: [...r.woven.diagnostics, ...r.study.diagnostics.filter((d) => d.severity === 'error').map((d) => ({ ...d, lang: 'speco' as LangId, message: `In the woven pseudocode, line ${d.line}: ${d.message}` }))], study: r.study }; + } + return { report: provePseudo(p), wovenSource: null, wovenDiagnostics: [], study: p }; + } + + /** + * The fuzzer's report for each routine of the program that runs (fuzz.ts): the Mojo pane's functions other than + * main, or the pseudo's routines with the speco woven in. Memoized by the source, the seed and the budget. + * + * The whole program shares one budget of work, FUZZ_PROGRAM_INPUTS inputs' worth (fuzz.ts WORK_PER_INPUT), and + * the first FUZZ_ROUTINES routines are fuzzed: a routine past either gets a report with no cases that says why in + * `problem`, so a program of thousands of routines is answered as quickly as one of a few. + */ + fuzz(opts: FuzzOptions = {}): FuzzReport[] { + const o = { seed: opts.seed ?? 1, budget: opts.budget ?? 300 }; + const share = (names: string[], one: (name: string, o: FuzzOptions) => FuzzReport, lang: 'pseudo' | 'mojo'): FuzzReport[] => { + const total = o.budget * WORK_PER_INPUT * FUZZ_PROGRAM_INPUTS / 300; + let left = total; + return names.map((name, i) => { + const skip = (problem: string): FuzzReport => ({ name, signature: name, lang, params: [], seed: o.seed, budget: o.budget, checks: [], strategies: [], cases: [], failures: [], counts: { pass: 0, fail: 0, skip: 0, raises: 0 }, coverage: [], results: [], problem, work: 0 }); + if (i >= FUZZ_ROUTINES) return skip(`Not tested: generated inputs cover the first ${FUZZ_ROUTINES} routines of a program, and this is routine ${i + 1}.`); + if (left <= 0) return skip(`Not tested: the routines before this one took all the work the tests of one program may take (${total.toLocaleString('en-US')} steps).`); + const r = one(name, { ...o, work: Math.min(left, o.budget * WORK_PER_INPUT) }); + left -= r.work ?? 0; + return r; + }); + }; + if (this.state.lang === 'mojo') { + const m = this.parsedMojo(); + if (!m || !m.ok || !m.program || m.program.module.unread) return []; + const src = this.source('mojo'); + const mod = m.program.module; + return this.cached('fuzz.mojo', JSON.stringify([src, o]), () => share(mod.body.filter((x: any) => x.k === 'def' && x.name !== 'main').map((d: any) => d.name), (name, oo) => fuzzMojo(mod, src, name, oo), 'mojo')); + } + const study = this.proof()?.study ?? this.parsedPseudo(); + if (!study || !study.ok || !study.program) return []; + const program = (study.program as any).program; + return this.cached('fuzz.pseudo', JSON.stringify([this.source('pseudo'), this.source('speco'), o]), () => share([...program.subs.values()].map((sub: any) => sub.name), (name, oo) => fuzzPseudo(program, name, oo), 'pseudo')); + } + + /** The fuzzer's reports in brief: per routine, the inputs checked and each way of failing. */ + tests(): PropertyReport[] { + return this.fuzz().map((r) => ({ + name: r.name, + checked: r.cases.filter((c) => c.strategy !== 'shrink' && c.strategy !== 'repeat' && c.outcome !== 'skip').length, + failures: [...r.failures.map((f) => ({ call: f.call, message: f.message, line: f.line, kind: f.kind })), + ...r.cases.filter((c) => c.outcome === 'raises').slice(0, 3).map((c) => ({ call: c.call, message: c.message ?? '', line: c.line, kind: 'raises-declared' }))], + ...(r.problem ? { problem: r.problem } : {}), + })); + } +} diff --git a/src/curriculum/advanced.ts b/src/curriculum/advanced.ts new file mode 100644 index 0000000..5c67c10 --- /dev/null +++ b/src/curriculum/advanced.ts @@ -0,0 +1,307 @@ +import type { Lesson } from './types.js'; + +/** Advanced: recursion, specification and proof, cost, the classic algorithms, the machine, and reading code you did not write. */ +export const ADVANCED: Lesson[] = [ + { + id: 'recursion', title: 'A function that calls itself', stage: 'advanced', minutes: 25, + purpose: 'You will be able to write a recursive function with a base case, trace its frames, and say why it ends.', + teaches: ['recursion'], requires: ['function', 'selection'], + explanation: [ + 'A function may call itself. `FACTORIAL(N)` is `N * FACTORIAL(N - 1)`, and `FACTORIAL(0)` is 1. The second rule is the base case: the input for which the answer is known without another call. Without it the calls never end; the lab stops them at 400 deep and says so. With it, every call is on a smaller input, so the chain reaches the base case and unwinds.', + 'Watch the frames in the trace. Each call adds a frame with its own `N`; the frames stack up to the base case, then each returns into the one below, which finishes its multiplication. Nothing magical happens: it is the same parameter-is-a-copy rule from the functions lesson, applied to a function that happens to call itself.', + '`DECREASES N` is how you write down why it ends: a whole number that is smaller in every recursive call and never below 0. The lab checks it on every run and the prover checks it for every input. The same line will appear in loops, where it plays the same role.', + 'Recursion is natural when the problem is defined in terms of a smaller copy of itself: a factorial, a sum of a list (first element plus sum of the rest), a search in a sorted range (look in one half). It is not faster than a loop, and a loop is often clearer; use recursion when the problem is recursive.', + '- A recursive function needs a base case and a recursive case on a smaller input.', + '- Each call has its own frame and its own copies of the parameters.', + '- DECREASES names the whole number that shrinks every call. Depth is limited to 400.', + ], + example: { + pseudo: 'FUNCTION FACTORIAL(N : INTEGER) RETURNS INTEGER\n REQUIRE N >= 0\n DECREASES N\n IF N == 0 THEN\n RETURN 1\n ENDIF\n RETURN N * FACTORIAL(N - 1)\nENDFUNCTION\nFUNCTION SUM_DIGITS(N : INTEGER) RETURNS INTEGER\n DECREASES N\n IF N < 10 THEN\n RETURN N\n ENDIF\n RETURN N MOD 10 + SUM_DIGITS(N DIV 10)\nENDFUNCTION\nOUTPUT FACTORIAL(5)\nOUTPUT FACTORIAL(15)\nOUTPUT SUM_DIGITS(9075)', + expect: ['120', '1307674368000', '21'], + watch: 'The call stack grows to six frames for FACTORIAL(5), N going 5, 4, 3, 2, 1, 0, then shrinks as each RETURN multiplies. SUM_DIGITS peels one digit per call.', + }, + checks: [ + { question: 'What happens to a recursive function with no base case?', options: ['It returns 0', 'The run stops at 400 calls deep', 'It runs once'], answer: 1, why: 'Every call makes another. The lab stops the run and names the function.' }, + { question: 'In `FACTORIAL(3)`, how many frames are on the stack at the deepest point?', options: ['1', '3', '4'], answer: 2, why: 'N = 3, 2, 1, and 0: four calls before the first RETURN.' }, + ], + flavors: [ + { id: 'cells', hook: 'The number of ways to fold, pair, or tile is often defined recursively. The simplest: how many sequences of length N over four bases.', task: 'Write SEQUENCES(N) recursively: 1 for N = 0, otherwise 4 times SEQUENCES(N - 1). Print SEQUENCES(3) and SEQUENCES(10).', solution: { pseudo: 'FUNCTION SEQUENCES(N : INTEGER) RETURNS INTEGER\n REQUIRE N >= 0\n DECREASES N\n IF N == 0 THEN\n RETURN 1 // ~ the base case: one empty sequence\n ENDIF\n RETURN 4 * SEQUENCES(N - 1) // ~ four choices for the last base\nENDFUNCTION\nOUTPUT SEQUENCES(3), " ", SEQUENCES(10)', expect: ['64 1048576'] } }, + { id: 'money', hook: 'Compound interest is recursive by definition: this year\'s balance is last year\'s balance plus interest on it.', task: 'Write BALANCE(Years) returning 10000 cents after Years years at 5% (whole cents, rounded down each year), recursively. Print BALANCE(0), BALANCE(1), BALANCE(10).', solution: { pseudo: 'FUNCTION BALANCE(Years : INTEGER) RETURNS INTEGER\n REQUIRE Years >= 0\n DECREASES Years\n IF Years == 0 THEN\n RETURN 10000\n ENDIF\n DECLARE Last : INTEGER\n Last <- BALANCE(Years - 1) // ~ last year\'s balance\n RETURN Last + Last * 5 DIV 100\nENDFUNCTION\nOUTPUT BALANCE(0), " ", BALANCE(1), " ", BALANCE(10)', expect: ['10000 10500 16284'] } }, + { id: 'words', hook: 'A palindrome reads the same backwards. The recursive definition: the first and last letters match, and the middle is a palindrome.', task: 'Write IS_PALINDROME(S) recursively, using SLICE to take the middle. Print it for "racecar", "noon", and "note".', solution: { pseudo: 'FUNCTION IS_PALINDROME(S : STRING) RETURNS BOOL\n DECREASES LENGTH(S)\n IF LENGTH(S) < 2 THEN\n RETURN TRUE // ~ nothing left to compare\n ENDIF\n IF S[0] != S[LENGTH(S) - 1] THEN\n RETURN FALSE\n ENDIF\n RETURN IS_PALINDROME(SLICE(S, 1, LENGTH(S) - 1)) // ~ the middle\nENDFUNCTION\nOUTPUT IS_PALINDROME("racecar"), " ", IS_PALINDROME("noon"), " ", IS_PALINDROME("note")', expect: ['TRUE TRUE FALSE'] } }, + ], + }, + { + id: 'specifications', title: 'Saying what, not how', stage: 'advanced', minutes: 25, + purpose: 'You will be able to write REQUIRE and ENSURE lines for a function, read them as a contract, and use the tests and prover to check them.', + teaches: ['specification'], requires: ['function', 'logic'], + explanation: [ + 'A specification says what a subroutine does without saying how. `REQUIRE Lo <= Hi` is what the caller promises; if a call breaks it, the call is wrong, not the function. `ENSURE Lo <= RESULT AND RESULT <= Hi` is what the function promises back; `RESULT` names the value it returned. Together they are a contract, and the body is one way of keeping it.', + 'Why write it down? Because "what it does" is otherwise in your head, and the code is the only record. A reader, a tester, a prover, and an agent asked to rewrite the function in another language all need the contract, not your memory of it. The platform uses it three ways: the lab checks every `REQUIRE` and `ENSURE` on every run; the Tests view generates inputs and reports any that break a promise; the Proof view tries to prove the `ENSURE` for every possible input from the body.', + '`ASSERT` is the same idea mid-body: "this is true here". It is a specification of one point in the procedure, and the fastest way to turn a hunch into a checked fact. The speco pane shows the same contracts in mathematical notation and weaves them into the pseudo; that pane is the specification as a document of its own.', + '- REQUIRE: what the caller promises. ENSURE: what the subroutine promises, with RESULT for the answer.', + '- The lab checks them on every run; the tests search for inputs that break them; the prover tries to prove them.', + '- A function with a contract can be rewritten, in any language, and checked against the same contract.', + ], + example: { + pseudo: 'FUNCTION CLAMP(X : INTEGER, Lo : INTEGER, Hi : INTEGER) RETURNS INTEGER\n REQUIRE Lo <= Hi\n ENSURE Lo <= RESULT AND RESULT <= Hi\n IF X < Lo THEN\n RETURN Lo\n ENDIF\n IF X > Hi THEN\n RETURN Hi\n ENDIF\n RETURN X\nENDFUNCTION\nOUTPUT CLAMP(15, 0, 10), " ", CLAMP(-3, 0, 10), " ", CLAMP(4, 0, 10)', + expect: ['10 0 4'], + watch: 'Open the Proof view: the ENSURE is proved for every X, Lo, Hi with Lo <= Hi. Then change RETURN Hi to RETURN Hi + 1 and watch the Tests view find an input that breaks the promise.', + }, + checks: [ + { question: 'A call breaks a REQUIRE. Whose mistake is it?', options: ['The function\'s', 'The caller\'s', 'Nobody\'s'], answer: 1, why: 'REQUIRE is the caller\'s promise. The function may assume it.' }, + { question: 'What does RESULT name?', options: ['The first parameter', 'The value the function returned', 'The last variable assigned'], answer: 1, why: 'RESULT, inside an ENSURE, is the returned value.' }, + ], + flavors: [ + { id: 'cells', hook: 'A function that returns a position must promise the position is valid. Without the ENSURE, "returns -1 when absent" lives in a comment nobody reads.', task: 'Write FIRST_OF(B, T) returning the first position of B in T or LENGTH(T) when absent, with an ENSURE that RESULT is within [0, LENGTH(T)] and, when it is a real position, T[RESULT] == B.', solution: { pseudo: 'FUNCTION FIRST_OF(B : CHAR, T : STRING) RETURNS INTEGER\n ENSURE 0 <= RESULT AND RESULT <= LENGTH(T) // ~ a position or the not-found value\n ENSURE RESULT == LENGTH(T) OR T[RESULT] == B // ~ when found, it really is B\n FOR I <- 0 TO LENGTH(T)\n IF T[I] == B THEN\n RETURN I\n ENDIF\n NEXT I\n RETURN LENGTH(T)\nENDFUNCTION\nOUTPUT FIRST_OF(\'T\', "GATTACA"), " ", FIRST_OF(\'N\', "GATTACA")', expect: ['2 7'] } }, + { id: 'money', hook: 'A tax function must never return a negative amount or more than the income. Writing that down catches the rounding mistake before the auditor does.', task: 'Write TAX(Income) at 20% in whole cents with REQUIRE Income >= 0 and ENSURE 0 <= RESULT <= Income. Print TAX(0), TAX(99), TAX(10000).', solution: { pseudo: 'FUNCTION TAX(Income : INTEGER) RETURNS INTEGER\n REQUIRE Income >= 0 // ~ the caller\'s promise\n ENSURE 0 <= RESULT AND RESULT <= Income // ~ the function\'s promise\n RETURN Income * 20 DIV 100\nENDFUNCTION\nOUTPUT TAX(0), " ", TAX(99), " ", TAX(10000)', expect: ['0 19 2000'] } }, + { id: 'machines', hook: 'When you ask an agent for a function, hand it the REQUIRE and ENSURE. Then whatever it writes can be checked against the contract rather than read on trust.', task: 'Here is a contract: ABS_DIFF(A, B) ENSUREs RESULT >= 0 and RESULT == A - B OR RESULT == B - A. Write a body that keeps it, and add an ASSERT in the body stating which branch you are in.', solution: { pseudo: 'FUNCTION ABS_DIFF(A : INTEGER, B : INTEGER) RETURNS INTEGER\n ENSURE RESULT >= 0\n ENSURE RESULT == A - B OR RESULT == B - A\n IF A >= B THEN\n ASSERT A - B >= 0 // ~ what is true on this branch\n RETURN A - B\n ENDIF\n RETURN B - A\nENDFUNCTION\nOUTPUT ABS_DIFF(3, 10), " ", ABS_DIFF(10, 3)', expect: ['7 7'] } }, + ], + }, + { + id: 'loop-invariants', title: 'What is true at the top of every pass', stage: 'advanced', minutes: 30, + purpose: 'You will be able to state the invariant of a loop with an accumulator, show that it gives the result at exit, and use DECREASES to show the loop ends.', + teaches: ['invariant'], requires: ['specification', 'accumulator', 'half-open', 'while'], + explanation: [ + 'The accumulators lesson said that `Total` has a one-sentence meaning at the top of every pass: "Total is the number of `B` among positions before `I`". That sentence is the loop invariant, and pseudo lets you write it as the first line of the loop body: `INVARIANT Total == LENGTH({ K IN [0, I) : T[K] == B })`. The set written by rule, `{ K IN [0, I) : … }`, is the positions before `I` that hold `B`; its `LENGTH` is their count.', + 'Three facts make an invariant useful. It holds before the first pass (`I` is 0, the range is empty, `Total` is 0). Each pass keeps it true (if `T[I]` is `B`, the count and `Total` both grow by one). And at exit, `I` is `LENGTH(T)`, so the invariant says `Total` counts all of `T`, which is the `ENSURE`. The Proof view does exactly this reasoning, for every input, and reports which of the three steps fails when one does.', + 'Why the half-open bound again: the invariant talks about positions `[0, I)`, the ones already handled. After the pass, that range grows to `[0, I + 1)`, which is `[0, I)` plus `I`. The arithmetic of the proof is the arithmetic of half-open ranges, and that is why the notation uses them everywhere.', + '`DECREASES` does for ending what `INVARIANT` does for correctness: a whole number that shrinks every pass and stays at or above 0. For a `FOR` it is implied; for a `WHILE` you write it, and the prover checks that the body really does shrink it.', + '- INVARIANT: true at the top of every pass, in terms of the positions handled so far, [0, I).', + '- At exit, the invariant plus the stopping condition give the ENSURE.', + '- DECREASES: a whole number that shrinks each pass; it is why the loop ends.', + ], + example: { + pseudo: 'FUNCTION COUNT_BASE(B : CHAR, T : STRING) RETURNS INTEGER\n ENSURE RESULT == LENGTH({ K IN [0, LENGTH(T)) : T[K] == B })\n DECLARE Total : INTEGER\n Total <- 0\n FOR I <- 0 TO LENGTH(T)\n INVARIANT Total == LENGTH({ K IN [0, I) : T[K] == B })\n IF T[I] == B THEN\n Total <- Total + 1\n ENDIF\n NEXT I\n RETURN Total\nENDFUNCTION\nOUTPUT COUNT_BASE(\'A\', "GATTACA")', + expect: ['3'], + watch: 'The Proof view: three obligations, all proved. Then break the loop on purpose: start I at 1. The lab reports the invariant fails on the first run that has B at position 0, and the prover says which step no longer holds.', + }, + problems: ['count-base'], + checks: [ + { question: 'Where does an INVARIANT line go?', options: ['Before the loop', 'As the first line of the loop body', 'After NEXT'], answer: 1, why: 'It states what is true at the top of every pass.' }, + { question: 'The invariant talks about [0, I). After the pass, which positions are handled?', options: ['[0, I)', '[0, I + 1)', '[1, I)'], answer: 1, why: 'Position I was just handled, so the range grows by one at the top end.' }, + { question: 'What does DECREASES prove?', options: ['That the result is right', 'That the loop ends', 'That the loop runs at least once'], answer: 1, why: 'A whole number that shrinks every pass and cannot go below 0 cannot shrink forever.' }, + ], + flavors: [ + { id: 'cells', hook: 'The GC count\'s invariant is the same sentence with an OR in it. Proving it once means every reimplementation, in any language, has something to be checked against.', task: 'Write GC_COUNT(T) with an ENSURE and an INVARIANT, using a set written by rule with an OR condition. Prove it.', solution: { pseudo: 'FUNCTION GC_COUNT(T : STRING) RETURNS INTEGER\n ENSURE RESULT == LENGTH({ K IN [0, LENGTH(T)) : T[K] == \'G\' OR T[K] == \'C\' })\n DECLARE N : INTEGER\n N <- 0\n FOR I <- 0 TO LENGTH(T)\n INVARIANT N == LENGTH({ K IN [0, I) : T[K] == \'G\' OR T[K] == \'C\' }) // ~ the positions handled so far\n IF T[I] == \'G\' OR T[I] == \'C\' THEN\n N <- N + 1\n ENDIF\n NEXT I\n RETURN N\nENDFUNCTION\nOUTPUT GC_COUNT("GGCATC")', expect: ['4'] } }, + { id: 'money', hook: 'A running balance has an invariant: the balance equals the opening amount plus the sum of transactions so far. SUM over a half-open range writes it down.', task: 'Write FINAL_BALANCE(Opening, Txns) with an INVARIANT using SUM K IN [0, I) : Txns[K], and an ENSURE for the whole list.', solution: { pseudo: 'FUNCTION FINAL_BALANCE(Opening : INTEGER, Txns : LIST OF INTEGER) RETURNS INTEGER\n ENSURE RESULT == Opening + (SUM K IN [0, LENGTH(Txns)) : Txns[K])\n DECLARE Balance : INTEGER\n Balance <- Opening\n FOR I <- 0 TO LENGTH(Txns)\n INVARIANT Balance == Opening + (SUM K IN [0, I) : Txns[K]) // ~ opening plus everything applied so far\n Balance <- Balance + Txns[I]\n NEXT I\n RETURN Balance\nENDFUNCTION\nOUTPUT FINAL_BALANCE(1000, [250, -400, 75])', expect: ['925'] } }, + { id: 'machines', hook: 'A WHILE that halves needs a DECREASES to be trusted. The prover checks that the body really shrinks it; an agent\'s loop that forgets to is caught before it runs forever.', task: 'Write HALVINGS(N) counting how many times N can be halved (DIV 2) before reaching 1, with an INVARIANT that N >= 1 and a DECREASES N. Print it for 1, 8, 100.', solution: { pseudo: 'FUNCTION HALVINGS(N : INTEGER) RETURNS INTEGER\n REQUIRE N >= 1\n DECLARE Count : INTEGER\n Count <- 0\n WHILE N > 1\n INVARIANT N >= 1\n DECREASES N // ~ the body halves N, so this shrinks every pass\n N <- N DIV 2\n Count <- Count + 1\n ENDWHILE\n RETURN Count\nENDFUNCTION\nOUTPUT HALVINGS(1), " ", HALVINGS(8), " ", HALVINGS(100)', expect: ['0 3 6'] } }, + ], + }, + { + id: 'memoization', title: 'Remembering answers', stage: 'advanced', minutes: 25, + purpose: 'You will be able to turn an exponential recursion into a linear one with a table, and write that in pseudo with a PROCEDURE and BYREF.', + teaches: ['memoization'], requires: ['recursion', 'hashmap', 'byref'], + explanation: [ + 'The naive Fibonacci, `FIB(N) = FIB(N - 1) + FIB(N - 2)`, computes `FIB(30)` by computing `FIB(28)` twice, `FIB(27)` three times, and `FIB(1)` over a million times. The work doubles with every step of `N`. The fix is to remember: the first time `FIB(K)` is worked out, store it in a table under key `K`; every later call looks it up instead. Each value is computed once, so the work grows with `N` rather than doubling.', + 'In pseudo the table has to be shared between calls, and a `FUNCTION` cannot take a `BYREF` parameter (it is pure by design). So memoized recursion is a `PROCEDURE`: the table travels `BYREF`, and the answer comes back `BYREF` too. It reads heavier than in most languages, and that heaviness is honest: it makes the shared table visible at every call site, where other languages hide it in a global or a closure.', + 'Count the calls to see the difference. The example prints how many times the body ran: 40 for `FIB(20)` with the table, against 21,891 without. The trace shows the table filling once and the later calls returning at the `IN Memo` test.', + '- Memoization: store each answer under its input the first time; look it up after that.', + '- Exponential becomes linear because every input is computed at most once.', + '- In pseudo: a PROCEDURE with the table and the answer both BYREF.', + ], + example: { + pseudo: 'PROCEDURE FIB(N : INTEGER, BYREF Memo : HASHMAP OF INTEGER : INTEGER, BYREF Answer : INTEGER, BYREF Calls : INTEGER)\n Calls <- Calls + 1\n IF N < 2 THEN\n Answer <- N\n RETURN\n ENDIF\n IF N IN Memo THEN\n Answer <- Memo[N]\n RETURN\n ENDIF\n DECLARE A, B : INTEGER\n CALL FIB(N - 1, Memo, A, Calls)\n CALL FIB(N - 2, Memo, B, Calls)\n Memo[N] <- A + B\n Answer <- A + B\nENDPROCEDURE\nDECLARE Table : HASHMAP OF INTEGER : INTEGER\nDECLARE F, Calls : INTEGER\nCalls <- 0\nCALL FIB(20, Table, F, Calls)\nOUTPUT F, " in ", Calls, " calls"\nCALL FIB(50, Table, F, Calls)\nOUTPUT F', + expect: ['6765 in 39 calls', '12586269025'], + watch: 'Table in the variables panel fills from 2 upward, each key once. The second CALL reuses everything up to 20 and adds only the keys 21 to 50.', + }, + checks: [ + { question: 'Why is memoized FIB(40) fast?', options: ['Addition is fast', 'Each FIB(K) is computed once and looked up afterwards', 'The table is sorted'], answer: 1, why: 'The table turns repeated subproblems into lookups.' }, + { question: 'Why is the pseudo version a PROCEDURE?', options: ['Procedures are faster', 'A FUNCTION cannot take the table BYREF', 'Recursion needs CALL'], answer: 1, why: 'Functions are pure by design. Sharing a table between calls needs BYREF, which only a PROCEDURE may ask for.' }, + ], + flavors: [ + { id: 'cells', hook: 'How many ways can a sequence of length N be read as a string of codons and single bases? Counting it naively is exponential; with a table it is instant. This is the shape of every dynamic-programming alignment.', task: 'WAYS(N): 1 for N = 0, and WAYS(N - 1) + WAYS(N - 3) otherwise (a single base, or a codon, at the end), with WAYS of a negative N being 0. Memoize it and print WAYS(10) and WAYS(40).', solution: { pseudo: 'PROCEDURE WAYS(N : INTEGER, BYREF Memo : HASHMAP OF INTEGER : INTEGER, BYREF Answer : INTEGER)\n IF N < 0 THEN\n Answer <- 0\n RETURN\n ENDIF\n IF N == 0 THEN\n Answer <- 1\n RETURN\n ENDIF\n IF N IN Memo THEN\n Answer <- Memo[N] // ~ already known\n RETURN\n ENDIF\n DECLARE A, B : INTEGER\n CALL WAYS(N - 1, Memo, A)\n CALL WAYS(N - 3, Memo, B)\n Memo[N] <- A + B // ~ remember it\n Answer <- A + B\nENDPROCEDURE\nDECLARE Table : HASHMAP OF INTEGER : INTEGER\nDECLARE W : INTEGER\nCALL WAYS(10, Table, W)\nOUTPUT W\nCALL WAYS(40, Table, W)\nOUTPUT W', expect: ['28', '2670964'] } }, + { id: 'money', hook: 'How many ways to make change for an amount with given coins is the textbook memoization problem, and the reason a vending machine can answer instantly.', task: 'CHANGE(Amount, Coins as a list, I): the number of ways to make Amount using coins from position I onward. Memoize on a key of Amount * 100 + I. Print the ways to make 100 cents from [50, 25, 10, 5, 1].', solution: { pseudo: 'PROCEDURE CHANGE(Amount : INTEGER, Coins : LIST OF INTEGER, I : INTEGER, BYREF Memo : HASHMAP OF INTEGER : INTEGER, BYREF Answer : INTEGER)\n IF Amount == 0 THEN\n Answer <- 1\n RETURN\n ENDIF\n IF Amount < 0 OR I >= LENGTH(Coins) THEN\n Answer <- 0\n RETURN\n ENDIF\n DECLARE Key : INTEGER\n Key <- Amount * 100 + I // ~ one key for the pair (Amount, I)\n IF Key IN Memo THEN\n Answer <- Memo[Key]\n RETURN\n ENDIF\n DECLARE With, Without : INTEGER\n CALL CHANGE(Amount - Coins[I], Coins, I, Memo, With) // ~ use coin I again\n CALL CHANGE(Amount, Coins, I + 1, Memo, Without) // ~ or move past it\n Memo[Key] <- With + Without\n Answer <- With + Without\nENDPROCEDURE\nDECLARE Table : HASHMAP OF INTEGER : INTEGER\nDECLARE Ways : INTEGER\nCALL CHANGE(100, [50, 25, 10, 5, 1], 0, Table, Ways)\nOUTPUT Ways', expect: ['292'] } }, + ], + }, + { + id: 'counting-cost', title: 'How much work is that?', stage: 'advanced', minutes: 20, + purpose: 'You will be able to count the steps of a loop, a nested loop, and a halving loop as a function of N, and recognize N, N², and log N when you see them.', + teaches: ['cost'], requires: ['nesting', 'while', 'repeat', 'function', 'procedure-sub', 'byref', 'set'], + explanation: [ + 'Two programs that print the same answer can differ by a factor of a million in work. The way to see it is to count passes as a function of the input size `N`. A single loop over `N` items runs its body `N` times. A loop over `N` inside a loop over `N` runs `N²` times. A loop that halves `N` each pass runs about `log₂ N` times: 7 passes for 100, 10 for 1,000, 20 for a million.', + 'These are the three shapes you will meet most, and the gaps between them are enormous. For a genome of three billion bases, `N` is three billion steps (minutes), `N²` is nine billion billion (never), and `log N` is 32 (instantly). Knowing which shape a procedure has is more important than knowing which language it is in; the language changes the constant, the shape changes the exponent.', + 'The example measures the three shapes directly by counting passes. Try `N` at 10, 30, and 60 and watch the three counts: one grows like `N`, one like `N²`, one barely moves. The stacks-and-queues lesson will show a hidden cost of this kind: `REMOVE(Q, 0)` shifts every later value, so a loop that dequeues `N` items does `N²` work without a nested loop in sight.', + '- Count passes as a function of N. One loop: N. Nested: N². Halving: log N.', + '- The shape matters more than the language. Exponents beat constants.', + '- Hidden loops (INSERT, REMOVE at the front, IN on a LIST) add their own factor of N.', + ], + example: { + pseudo: 'FUNCTION LINEAR(N : INTEGER) RETURNS INTEGER\n DECLARE Passes : INTEGER\n Passes <- 0\n FOR I <- 0 TO N\n Passes <- Passes + 1\n NEXT I\n RETURN Passes\nENDFUNCTION\nFUNCTION SQUARE(N : INTEGER) RETURNS INTEGER\n DECLARE Passes : INTEGER\n Passes <- 0\n FOR I <- 0 TO N\n FOR J <- 0 TO N\n Passes <- Passes + 1\n NEXT J\n NEXT I\n RETURN Passes\nENDFUNCTION\nFUNCTION HALVING(N : INTEGER) RETURNS INTEGER\n DECLARE Passes : INTEGER\n Passes <- 0\n WHILE N > 1\n N <- N DIV 2\n Passes <- Passes + 1\n ENDWHILE\n RETURN Passes\nENDFUNCTION\nOUTPUT LINEAR(60), " ", SQUARE(60), " ", HALVING(60)', + expect: ['60 3600 5'], + watch: 'The step counter at the end of the run. Nearly all of it was SQUARE. Change 60 to 30: LINEAR halves, SQUARE quarters, HALVING loses one pass.', + }, + checks: [ + { question: 'A loop over N inside a loop over N inside a loop over N. How many passes?', options: ['3N', 'N²', 'N³'], answer: 2, why: 'Each level multiplies by N.' }, + { question: 'Halving a million down to 1 takes about how many passes?', options: ['20', '1000', '500,000'], answer: 0, why: '2 to the 20 is about a million, so twenty halvings.' }, + { question: 'Which matters more for a genome-sized input?', options: ['Using a fast language', 'Using an N log N procedure instead of N²', 'Using fewer variables'], answer: 1, why: 'The shape of the cost decides whether it finishes at all; the language decides how fast a finishing program runs.' }, + ], + flavors: [ + { id: 'cells', hook: 'Comparing every read with every other is N². Counting k-mers into a hashmap is N. For a million reads that is the difference between a coffee and a career.', task: 'Count the passes of the pairwise comparison (each pair once) over N reads and of a single pass, for N = 20 and N = 40, and print the four numbers.', solution: { pseudo: 'FUNCTION PAIRS(N : INTEGER) RETURNS INTEGER\n DECLARE Passes : INTEGER\n Passes <- 0\n FOR I <- 0 TO N\n FOR J <- I + 1 TO N // ~ each pair once\n Passes <- Passes + 1\n NEXT J\n NEXT I\n RETURN Passes\nENDFUNCTION\nOUTPUT PAIRS(20), " ", 20, " ", PAIRS(40), " ", 40', expect: ['190 20 780 40'] } }, + { id: 'money', hook: 'Doubling your money takes log₂ steps, not linear ones. The same mathematics says why a halving search is fast.', task: 'Count how many doublings take 1 cent past 1,000,000 cents, and how many single-cent steps the linear way would take (do not loop that one; print the number).', solution: { pseudo: 'DECLARE Cents, Doublings : INTEGER\nCents <- 1\nDoublings <- 0\nREPEAT\n Cents <- Cents * 2 // ~ one doubling\n Doublings <- Doublings + 1\nUNTIL Cents > 1000000 // ~ past a million\nOUTPUT Doublings, " doublings, versus ", 1000000, " steps"', expect: ['20 doublings, versus 1000000 steps'] } }, + { id: 'machines', hook: 'An agent will happily hand you an N² loop where N log N was available, and both print the right answer on the test data. Counting passes is how you catch it.', task: 'This program finds whether a list has a duplicate by comparing every pair. Count its passes for a 30-item list, then rewrite it with a SET and count again.', solution: { pseudo: 'PROCEDURE HAS_DUP_PAIRS(Xs : LIST OF INTEGER, BYREF Passes : INTEGER, BYREF Found : BOOL)\n Found <- FALSE\n FOR I <- 0 TO LENGTH(Xs)\n FOR J <- I + 1 TO LENGTH(Xs)\n Passes <- Passes + 1\n IF Xs[I] == Xs[J] THEN\n Found <- TRUE\n RETURN\n ENDIF\n NEXT J\n NEXT I\nENDPROCEDURE\nPROCEDURE HAS_DUP_SET(Xs : LIST OF INTEGER, BYREF Passes : INTEGER, BYREF Found : BOOL)\n DECLARE Seen : SET OF INTEGER\n Found <- FALSE\n FOR EACH X IN Xs\n Passes <- Passes + 1\n IF X IN Seen THEN // ~ one step, however many are in the set\n Found <- TRUE\n RETURN\n ENDIF\n ADD(Seen, X)\n NEXT X\nENDPROCEDURE\nDECLARE L : LIST OF INTEGER\nFOR K <- 0 TO 30\n APPEND(L, K * 7)\nNEXT K\nDECLARE P1, P2 : INTEGER\nDECLARE Dup : BOOL\nP1 <- 0\nP2 <- 0\nCALL HAS_DUP_PAIRS(L, P1, Dup)\nOUTPUT Dup, " after ", P1, " passes"\nCALL HAS_DUP_SET(L, P2, Dup)\nOUTPUT Dup, " after ", P2, " passes"', expect: ['FALSE after 435 passes', 'FALSE after 30 passes'] } }, + ], + }, + { + id: 'stacks-and-queues', title: 'Last in first out, first in first out', stage: 'advanced', minutes: 20, + purpose: 'You will be able to use a LIST as a stack and as a queue, choose which a problem needs, and name the cost of each operation.', + teaches: ['stack-queue'], requires: ['list', 'early-exit', 'cost'], + explanation: [ + 'A stack gives back the most recently added value: `APPEND` to push, read `L[LENGTH(L) - 1]` for the top, `REMOVE(L, LENGTH(L) - 1)` to pop. A queue gives back the oldest: `APPEND` to join, `L[0]` for the front, `REMOVE(L, 0)` to leave. Neither is a new type. Both are a `LIST` used with discipline: you only touch one end (stack) or opposite ends (queue).', + 'Which one a problem needs is usually obvious once you ask "which should come out first, the newest or the oldest?". Matching brackets wants the newest open bracket, so a stack. Serving requests in the order they arrived wants the oldest, so a queue. Undo is a stack; a printer is a queue; exploring a maze by nearest first (next lesson) is a queue.', + 'The cost lesson predicted a hidden factor: `REMOVE(L, 0)` shifts every later value left, so leaving a queue costs `LENGTH` steps, and a loop that drains a queue of `N` items does `N²` work. Popping a stack costs one step. For the small queues in these lessons it does not matter; for a million-item queue it does, and languages give you a deque or a ring buffer for exactly this reason.', + '- Stack: APPEND, read and REMOVE at LENGTH - 1. Newest first.', + '- Queue: APPEND, read and REMOVE at 0. Oldest first.', + '- REMOVE at 0 shifts everything: a queue drained with it costs N² in total.', + ], + example: { + pseudo: 'FUNCTION BALANCED(S : STRING) RETURNS BOOL\n DECLARE Stack : LIST OF CHAR\n FOR EACH C IN S\n IF C == \'(\' OR C == \'[\' THEN\n APPEND(Stack, C)\n ELSEIF C == \')\' OR C == \']\' THEN\n IF LENGTH(Stack) == 0 THEN\n RETURN FALSE\n ENDIF\n DECLARE Top : CHAR\n Top <- Stack[LENGTH(Stack) - 1]\n REMOVE(Stack, LENGTH(Stack) - 1)\n IF (C == \')\' AND Top != \'(\') OR (C == \']\' AND Top != \'[\') THEN\n RETURN FALSE\n ENDIF\n ENDIF\n NEXT C\n RETURN LENGTH(Stack) == 0\nENDFUNCTION\nOUTPUT BALANCED("([()])"), " ", BALANCED("([)]"), " ", BALANCED("(("), " ", BALANCED(")")', + expect: ['TRUE FALSE FALSE FALSE'], + watch: 'Stack in the variables panel grows on every opener and shrinks on every closer. The second input fails when ) meets a [ on top: the newest opener is the one that must match.', + }, + checks: [ + { question: 'Which comes off a stack first?', options: ['The oldest value', 'The newest value', 'The largest value'], answer: 1, why: 'Last in, first out.' }, + { question: 'Why does draining a queue with REMOVE(Q, 0) cost N² overall?', options: ['REMOVE is slow in general', 'Each REMOVE at 0 shifts every remaining value, and there are N of them', 'Queues are always slower than stacks'], answer: 1, why: 'N removals, each shifting up to N values.' }, + ], + flavors: [ + { id: 'cells', hook: 'RNA folds back on itself, and whether a dot-bracket structure like "((..))" is well formed is exactly the bracket-matching stack.', task: 'Write WELL_FORMED(S) for dot-bracket strings of \'(\', \')\', and \'.\': TRUE when every \')\' closes an earlier \'(\' and none are left open. Print it for "((..))", "(.))", and "(()".', solution: { pseudo: 'FUNCTION WELL_FORMED(S : STRING) RETURNS BOOL\n DECLARE Open : INTEGER\n Open <- 0 // ~ with one bracket kind, the stack is just a count\n FOR EACH C IN S\n IF C == \'(\' THEN\n Open <- Open + 1\n ELSEIF C == \')\' THEN\n IF Open == 0 THEN\n RETURN FALSE // ~ a closer with nothing to close\n ENDIF\n Open <- Open - 1\n ENDIF\n NEXT C\n RETURN Open == 0\nENDFUNCTION\nOUTPUT WELL_FORMED("((..))"), " ", WELL_FORMED("(.))"), " ", WELL_FORMED("(()")', expect: ['TRUE FALSE FALSE'] } }, + { id: 'everyday', hook: 'A ticket queue at a counter is first come, first served. The program that calls the next number is a queue with a counter.', task: 'People join in the order ["Ada", "Lin", "Sam"]. Serve two with REMOVE at 0, printing "Now serving X" each time, then print who is still waiting.', solution: { pseudo: 'DECLARE Queue : LIST OF STRING\nQueue <- ["Ada", "Lin", "Sam"]\nFOR N <- 0 TO 2\n OUTPUT "Now serving ", Queue[0] // ~ the front of the queue\n REMOVE(Queue, 0) // ~ they leave\nNEXT N\nOUTPUT Queue', expect: ['Now serving Ada', 'Now serving Lin', '["Sam"]'] } }, + { id: 'games', hook: 'Undo is a stack of moves. Each undo pops the most recent; redo would be a second stack.', task: 'Moves "e4", "e5", "Nf3" are played. Undo one, then play "Nc3", and print the move list.', solution: { pseudo: 'DECLARE Moves : LIST OF STRING\nAPPEND(Moves, "e4")\nAPPEND(Moves, "e5")\nAPPEND(Moves, "Nf3")\nREMOVE(Moves, LENGTH(Moves) - 1) // ~ undo: pop the newest\nAPPEND(Moves, "Nc3")\nOUTPUT Moves', expect: ['["e4", "e5", "Nc3"]'] } }, + ], + }, + { + id: 'binary-search', title: 'Halving the haystack', stage: 'advanced', minutes: 30, + purpose: 'You will be able to write binary search over a sorted list with its invariant, say why it is log N, and name the off-by-one traps it sets.', + teaches: ['binary-search'], requires: ['invariant', 'while', 'list', 'cost'], + explanation: [ + 'In a sorted list you can throw away half the candidates with one comparison. Keep a half-open range `[Lo, Hi)` that must contain the target if it is anywhere; look at the middle; if it is too small, the target is in `[Mid + 1, Hi)`; if too large, in `[Lo, Mid)`; if equal, found. Each pass halves the range, so a million items take twenty passes.', + 'The invariant is the whole algorithm: "if the target is in the list, it is in `[Lo, Hi)`". It holds at the start with `[0, LENGTH)`, each branch keeps it, and when `Lo == Hi` the range is empty, so the target is absent. `DECREASES Hi - Lo` is why it ends: every branch shrinks the range by at least one. Write both down and the prover will hold you to them.', + 'Binary search is famous for being short and famous for being written wrong. The traps: using a closed range and then looping forever when `Lo == Hi`; setting `Lo <- Mid` instead of `Mid + 1` (the range does not shrink); and reading `Xs[Hi]` when `Hi` is the length. Half-open bounds remove the first and third; `DECREASES` catches the second on the first run.', + '- Invariant: the target, if present, is in [Lo, Hi). Start with [0, LENGTH).', + '- Too small: Lo <- Mid + 1. Too large: Hi <- Mid. Equal: found. Empty range: absent.', + '- Each pass halves the range: log N passes. DECREASES Hi - Lo proves it ends.', + ], + example: { + pseudo: 'FUNCTION FIND_SORTED(Xs : LIST OF INTEGER, Target : INTEGER) RETURNS INTEGER\n DECLARE Lo, Hi, Mid : INTEGER\n Lo <- 0\n Hi <- LENGTH(Xs)\n WHILE Lo < Hi\n INVARIANT 0 <= Lo AND Lo <= Hi AND Hi <= LENGTH(Xs)\n DECREASES Hi - Lo\n Mid <- (Lo + Hi) DIV 2\n IF Xs[Mid] == Target THEN\n RETURN Mid\n ELSEIF Xs[Mid] < Target THEN\n Lo <- Mid + 1\n ELSE\n Hi <- Mid\n ENDIF\n ENDWHILE\n RETURN -1\nENDFUNCTION\nDECLARE Primes : LIST OF INTEGER\nPrimes <- [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]\nOUTPUT FIND_SORTED(Primes, 19), " ", FIND_SORTED(Primes, 4), " ", FIND_SORTED(Primes, 2)', + expect: ['7 -1 0'], + watch: 'Lo and Hi close in on the target: [0, 10), [5, 10), [8, 10), then Mid is 7 and found. For 4 the range empties and the WHILE condition fails: absent.', + }, + checks: [ + { question: 'Xs[Mid] is less than the target. What is the new range?', options: ['[Lo, Mid)', '[Mid, Hi)', '[Mid + 1, Hi)'], answer: 2, why: 'Mid itself is too small, so it is excluded; everything after it stays.' }, + { question: 'How many passes for a sorted list of 1,000,000?', options: ['About 20', 'About 1,000', 'About 500,000'], answer: 0, why: 'Each pass halves; 2 to the 20 is about a million.' }, + { question: 'Why does binary search need sorted data?', options: ['So Mid can be computed', 'So one comparison says which half the target is in', 'It does not'], answer: 1, why: 'Only in sorted data does "too small" mean "everything before here is too small too".' }, + ], + flavors: [ + { id: 'cells', hook: 'Gene coordinates are sorted along a chromosome. Finding which gene contains a position is binary search over the start positions, done millions of times per analysis.', task: 'Starts are [100, 450, 900, 1300, 2000]. Write LAST_START_BEFORE(Starts, Pos) returning the position of the largest start <= Pos, or -1 if none, by binary search. Print it for 950, 100, and 50.', solution: { pseudo: 'FUNCTION LAST_START_BEFORE(Starts : LIST OF INTEGER, Pos : INTEGER) RETURNS INTEGER\n DECLARE Lo, Hi, Mid : INTEGER\n Lo <- 0\n Hi <- LENGTH(Starts)\n WHILE Lo < Hi\n INVARIANT 0 <= Lo AND Lo <= Hi AND Hi <= LENGTH(Starts)\n DECREASES Hi - Lo\n Mid <- (Lo + Hi) DIV 2\n IF Starts[Mid] <= Pos THEN\n Lo <- Mid + 1 // ~ this start qualifies; the answer is here or later\n ELSE\n Hi <- Mid // ~ too far along\n ENDIF\n ENDWHILE\n RETURN Lo - 1\nENDFUNCTION\nDECLARE Starts : LIST OF INTEGER\nStarts <- [100, 450, 900, 1300, 2000]\nOUTPUT LAST_START_BEFORE(Starts, 950), " ", LAST_START_BEFORE(Starts, 100), " ", LAST_START_BEFORE(Starts, 50)', expect: ['2 0 -1'] } }, + { id: 'games', hook: 'Guess my number between 1 and 100: the best strategy is binary search, and seven guesses always suffice.', task: 'Write GUESSES(Secret) that plays the halving strategy over [1, 101) and returns how many guesses it took. Print it for 1, 50, 100, then the maximum over all secrets 1 to 100.', solution: { pseudo: 'FUNCTION GUESSES(Secret : INTEGER) RETURNS INTEGER\n DECLARE Lo, Hi, Mid, N : INTEGER\n Lo <- 1\n Hi <- 101\n N <- 0\n WHILE Lo < Hi\n DECREASES Hi - Lo\n Mid <- (Lo + Hi) DIV 2\n N <- N + 1\n IF Mid == Secret THEN\n RETURN N\n ELSEIF Mid < Secret THEN\n Lo <- Mid + 1\n ELSE\n Hi <- Mid\n ENDIF\n ENDWHILE\n RETURN N\nENDFUNCTION\nDECLARE Worst : INTEGER\nWorst <- 0\nFOR S <- 1 TO 101\n IF GUESSES(S) > Worst THEN\n Worst <- GUESSES(S) // ~ the hardest secret so far\n ENDIF\nNEXT S\nOUTPUT GUESSES(1), " ", GUESSES(50), " ", GUESSES(100), " worst ", Worst', expect: ['7 6 6 worst 7'] } }, + ], + }, + { + id: 'sorting', title: 'Putting things in order', stage: 'advanced', minutes: 30, + purpose: 'You will be able to write insertion sort in place, state its invariant (the sorted prefix), and say why it is N² in the worst case.', + teaches: ['sorting'], requires: ['invariant', 'nesting', 'byref', 'list', 'while', 'record'], + explanation: [ + 'Insertion sort is how most people sort a hand of cards: keep the left part sorted, take the next card, and slide it left past everything larger. In pseudo the left part is positions `[0, I)`, the next card is `Xs[I]`, and the slide is a `WHILE` that moves larger values one place right until the card\'s place is found.', + 'The invariant is "positions `[0, I)` are in order", written `FORALL K IN [1, I) : Xs[K - 1] <= Xs[K]`. It holds for `I` = 1 (one value is sorted), each pass extends it by one, and at exit `I` is `LENGTH`, so the whole list is sorted. The sort takes the list `BYREF` and changes it in place; that is a procedure\'s job, which is why this is a `PROCEDURE` and not a `FUNCTION`.', + 'Cost: the inner `WHILE` slides up to `I` places on pass `I`, so a reversed list takes 1 + 2 + … + N, about N²/2 moves. An already sorted list takes N comparisons and no moves. Sorting is a subject of its own (merge sort and quicksort reach N log N), and `SORT(L)`, the built-in, does it for you; this lesson is about seeing the invariant and the cost in the simplest honest sort.', + '- Insertion sort: grow a sorted prefix [0, I) by sliding Xs[I] into place.', + '- Invariant: the prefix is in order. At exit the prefix is the whole list.', + '- Worst case N² moves (reversed input), best case N (already sorted).', + ], + example: { + pseudo: 'PROCEDURE INSERTION_SORT(BYREF Xs : LIST OF INTEGER, BYREF Moves : INTEGER)\n DECLARE J, V : INTEGER\n FOR I <- 1 TO LENGTH(Xs)\n INVARIANT FORALL K IN [1, I) : Xs[K - 1] <= Xs[K]\n V <- Xs[I]\n J <- I\n WHILE J > 0 AND Xs[J - 1] > V\n Xs[J] <- Xs[J - 1]\n J <- J - 1\n Moves <- Moves + 1\n ENDWHILE\n Xs[J] <- V\n NEXT I\nENDPROCEDURE\nDECLARE L : LIST OF INTEGER\nDECLARE Moves : INTEGER\nL <- [5, 2, 9, 1, 7]\nMoves <- 0\nCALL INSERTION_SORT(L, Moves)\nOUTPUT L, " in ", Moves, " moves"\nL <- [5, 4, 3, 2, 1]\nMoves <- 0\nCALL INSERTION_SORT(L, Moves)\nOUTPUT L, " in ", Moves, " moves"', + expect: ['[1, 2, 5, 7, 9] in 5 moves', '[1, 2, 3, 4, 5] in 10 moves'], + watch: 'The list ruler. On each outer pass the prefix up to I is sorted and the rest is untouched. The reversed list costs 1 + 2 + 3 + 4 = 10 moves: the N² shape.', + }, + checks: [ + { question: 'What is true at the top of pass I of insertion sort?', options: ['Positions [0, I) are sorted', 'Positions [I, N) are sorted', 'Xs[I] is the smallest'], answer: 0, why: 'The sorted prefix is the invariant; everything after I has not been looked at.' }, + { question: 'Why is the sort a PROCEDURE?', options: ['Sorting is slow', 'It changes the caller\'s list in place, which needs BYREF', 'Procedures can use WHILE'], answer: 1, why: 'The effect on the caller\'s list is the job. BYREF is only available to a PROCEDURE.' }, + ], + flavors: [ + { id: 'cells', hook: 'Reads are sorted by position before anything else happens to them. Sorting records by one field is the same insertion sort comparing that field.', task: 'Define a Read with Name and Start. Sort a list of three reads by Start with insertion sort and print the names in order.', solution: { pseudo: 'TYPE Read\n DECLARE Name : STRING\n DECLARE Start : INTEGER\nENDTYPE\nPROCEDURE SORT_BY_START(BYREF Rs : LIST OF Read)\n DECLARE J : INTEGER\n DECLARE V : Read\n FOR I <- 1 TO LENGTH(Rs)\n V <- Rs[I]\n J <- I\n WHILE J > 0 AND Rs[J - 1].Start > V.Start // ~ compare the field, move the record\n Rs[J] <- Rs[J - 1]\n J <- J - 1\n ENDWHILE\n Rs[J] <- V\n NEXT I\nENDPROCEDURE\nDECLARE A, B, C : Read\nDECLARE Reads : LIST OF Read\nA.Name <- "r1"\nA.Start <- 900\nB.Name <- "r2"\nB.Start <- 120\nC.Name <- "r3"\nC.Start <- 450\nAPPEND(Reads, A)\nAPPEND(Reads, B)\nAPPEND(Reads, C)\nCALL SORT_BY_START(Reads)\nFOR EACH R IN Reads\n OUTPUT R.Name, " ", R.Start\nNEXT R', expect: ['r2 120', 'r3 450', 'r1 900'] } }, + { id: 'money', hook: 'A statement sorted by amount shows the biggest transactions first. Sorting descending is the same algorithm with the comparison flipped.', task: 'Sort [250, -400, 75, 1200, -30] from largest to smallest with insertion sort and print the result.', solution: { pseudo: 'PROCEDURE SORT_DESC(BYREF Xs : LIST OF INTEGER)\n DECLARE J, V : INTEGER\n FOR I <- 1 TO LENGTH(Xs)\n V <- Xs[I]\n J <- I\n WHILE J > 0 AND Xs[J - 1] < V // ~ flipped: slide past smaller values\n Xs[J] <- Xs[J - 1]\n J <- J - 1\n ENDWHILE\n Xs[J] <- V\n NEXT I\nENDPROCEDURE\nDECLARE Txns : LIST OF INTEGER\nTxns <- [250, -400, 75, 1200, -30]\nCALL SORT_DESC(Txns)\nOUTPUT Txns', expect: ['[1200, 250, 75, -30, -400]'] } }, + ], + }, + { + id: 'graphs', title: 'Networks, and searching them by distance', stage: 'advanced', minutes: 30, + purpose: 'You will be able to represent a graph as a HASHMAP of neighbor lists and run breadth-first search with a queue and a visited set.', + teaches: ['graph'], requires: ['stack-queue', 'hashmap', 'set', 'array'], + explanation: [ + 'A graph is things and the connections between them: genes and the proteins that regulate them, accounts and transfers, rooms and doors, stations and lines. In pseudo the natural representation is `HASHMAP OF STRING : LIST OF STRING`: each node is a key, and its value is the list of nodes it connects to. `Adj["A"]` is everything one step from `A`.', + 'Breadth-first search visits nodes in order of distance from a start: the start, then everything one step away, then two, and so on. It is a queue (oldest first, so nearer nodes come out before farther ones) and a set (so no node is visited twice). Take the front of the queue, add its unseen neighbors to the back and to the set, repeat until the queue is empty. The order nodes leave the queue is the order of their distance.', + 'BFS is the first graph algorithm and the model for the rest: shortest path in steps, "is B reachable from A", connected components, and the layers of a social network are all BFS with a small change. Swap the queue for a stack and you have depth-first search, which dives instead of spreading.', + '- A graph is a HASHMAP from each node to its list of neighbors.', + '- BFS: a queue of nodes to visit and a set of nodes seen; take the front, enqueue unseen neighbors.', + '- Nodes leave the queue in order of distance from the start.', + ], + example: { + pseudo: 'DECLARE Adj : HASHMAP OF STRING : LIST OF STRING\nAdj["A"] <- ["B", "C"]\nAdj["B"] <- ["D"]\nAdj["C"] <- ["D", "E"]\nAdj["D"] <- ["F"]\nAdj["E"] <- []\nAdj["F"] <- []\nDECLARE Queue, Order : LIST OF STRING\nDECLARE Seen : SET OF STRING\nAPPEND(Queue, "A")\nADD(Seen, "A")\nWHILE LENGTH(Queue) > 0\n DECLARE Node : STRING\n Node <- Queue[0]\n REMOVE(Queue, 0)\n APPEND(Order, Node)\n FOR EACH Nb IN Adj[Node]\n IF NOT Nb IN Seen THEN\n ADD(Seen, Nb)\n APPEND(Queue, Nb)\n ENDIF\n NEXT Nb\nENDWHILE\nOUTPUT Order', + expect: ['["A", "B", "C", "D", "E", "F"]'], + watch: 'Queue and Seen together. D is reached from B and again from C; the set lets it in once. F, two steps from A, leaves the queue after every one-step node.', + }, + checks: [ + { question: 'Why does BFS need a set?', options: ['To sort the nodes', 'So a node reachable two ways is visited once', 'To store distances'], answer: 1, why: 'Without it, D would be queued twice and a cycle would loop forever.' }, + { question: 'Replace the queue with a stack. What changes?', options: ['Nothing', 'Nodes are visited deepest-first instead of nearest-first', 'The search fails'], answer: 1, why: 'Newest-first exploration dives down one path: depth-first search.' }, + ], + flavors: [ + { id: 'cells', hook: 'A gene regulatory network is a graph. "Which genes does a knocked-out gene affect, directly or indirectly?" is exactly the set BFS builds.', task: 'Genes: A activates B and C; B activates D; C activates D; D activates nothing; E activates A. Print every gene downstream of A, in BFS order, and whether E is among them.', solution: { pseudo: 'DECLARE Reg : HASHMAP OF STRING : LIST OF STRING\nReg["A"] <- ["B", "C"]\nReg["B"] <- ["D"]\nReg["C"] <- ["D"]\nReg["D"] <- []\nReg["E"] <- ["A"]\nDECLARE Queue, Downstream : LIST OF STRING\nDECLARE Seen : SET OF STRING\nAPPEND(Queue, "A")\nADD(Seen, "A")\nWHILE LENGTH(Queue) > 0\n DECLARE G : STRING\n G <- Queue[0]\n REMOVE(Queue, 0)\n FOR EACH Target IN Reg[G]\n IF NOT Target IN Seen THEN\n ADD(Seen, Target) // ~ seen once\n APPEND(Queue, Target)\n APPEND(Downstream, Target) // ~ affected, directly or indirectly\n ENDIF\n NEXT Target\nENDWHILE\nOUTPUT Downstream, " ", "E" IN Seen', expect: ['["B", "C", "D"] FALSE'] } }, + { id: 'everyday', hook: 'How many changes to get from one station to another? BFS over the station graph counts them, and it is how every journey planner starts.', task: 'Stations: Home connects to Park and Mall; Park to Museum; Mall to Museum and Airport; Museum to Airport. Record the distance in stops from Home to every station in a HASHMAP and print it.', solution: { pseudo: 'DECLARE Lines : HASHMAP OF STRING : LIST OF STRING\nLines["Home"] <- ["Park", "Mall"]\nLines["Park"] <- ["Museum"]\nLines["Mall"] <- ["Museum", "Airport"]\nLines["Museum"] <- ["Airport"]\nLines["Airport"] <- []\nDECLARE Queue : LIST OF STRING\nDECLARE Dist : HASHMAP OF STRING : INTEGER\nAPPEND(Queue, "Home")\nDist["Home"] <- 0\nWHILE LENGTH(Queue) > 0\n DECLARE S : STRING\n S <- Queue[0]\n REMOVE(Queue, 0)\n FOR EACH Nb IN Lines[S]\n IF NOT Nb IN Dist THEN // ~ the map of distances doubles as the seen set\n Dist[Nb] <- Dist[S] + 1 // ~ one stop further than S\n APPEND(Queue, Nb)\n ENDIF\n NEXT Nb\nENDWHILE\nOUTPUT Dist', expect: ['{"Airport": 2, "Home": 0, "Mall": 1, "Museum": 2, "Park": 1}'] } }, + { id: 'games', hook: 'A small, dense graph can also be an adjacency matrix: an ARRAY[N, N] OF BOOL where cell (I, J) says whether I connects to J. A board game\'s squares often come this way.', task: 'Four squares: 0 connects to 1 and 2, 1 to 3, 2 to 3. Fill the matrix both ways (edges go both directions), then print each square\'s number of neighbors.', solution: { pseudo: 'DECLARE Adj : ARRAY[4, 4] OF BOOL\nFOR I <- 0 TO 4\n FOR J <- 0 TO 4\n Adj[I, J] <- FALSE\n NEXT J\nNEXT I\nPROCEDURE LINK(BYREF M : ARRAY[4, 4] OF BOOL, A : INTEGER, B : INTEGER)\n M[A, B] <- TRUE\n M[B, A] <- TRUE // ~ both directions\nENDPROCEDURE\nCALL LINK(Adj, 0, 1)\nCALL LINK(Adj, 0, 2)\nCALL LINK(Adj, 1, 3)\nCALL LINK(Adj, 2, 3)\nFOR I <- 0 TO 4\n DECLARE Degree : INTEGER\n Degree <- 0\n FOR J <- 0 TO 4\n IF Adj[I, J] THEN\n Degree <- Degree + 1 // ~ one more neighbor\n ENDIF\n NEXT J\n OUTPUT I, ": ", Degree\nNEXT I', expect: ['0: 2', '1: 2', '2: 2', '3: 2'] } }, + ], + }, + { + id: 'the-machine', title: 'What the machine actually runs', stage: 'advanced', minutes: 30, + purpose: 'You will be able to read a counted loop in x86-64, match each instruction to a line of pseudo, and say what a register, a flag, and a memory load are.', + teaches: ['machine-model'], requires: ['tracing', 'case'], + explanation: [ + 'Below every language is a processor that knows nothing about loops, strings, or types. It has a few dozen registers (fixed-size boxes for numbers), a block of memory addressed by number, and a list of instructions that each do one small thing: load a byte from memory, add two registers, compare, jump to another instruction if the comparison said so. A `FOR` loop over a string is a compare, a conditional jump, a load, another compare, an add, and a jump back. That is all it ever was.', + 'Open the x86-64 pane of the count-base problem beside this lesson. `rdi` holds the address of the text, `rsi` the base to count, `rcx` the position, `rax` the total. `movzx eax, byte [rdi + rcx]` is `T[I]`; `cmp` and `je` are the `IF`; `inc rcx` is the counter moving; `cmp rcx, rdx` and `jl` are the half-open bound, `I < LENGTH(T)`, as a comparison and a jump. The flags register holds the result of the last compare, which is how the jump knows which way to go.', + 'Why look this far down? Because every performance question and every "undefined behavior" eventually lands here, and because an agent that writes you C or Rust is writing instructions for this machine, with the types and bounds checks that pseudo gave you for free now your responsibility. The machine view shows the cost in cycles too: a memory load is many times the price of an add, which is where the cost lesson\'s constants come from.', + '- A processor has registers, flags, memory, and small instructions. Everything else is translation.', + '- A FOR loop is a compare, a conditional jump, and an increment. The half-open bound is a cmp and a jl.', + '- Memory is slow, registers are fast: that is the constant behind the cost shapes.', + ], + example: { + pseudo: 'FUNCTION COUNT_BASE(B : CHAR, T : STRING) RETURNS INTEGER\n DECLARE Total : INTEGER\n Total <- 0\n FOR I <- 0 TO LENGTH(T)\n IF T[I] == B THEN\n Total <- Total + 1\n ENDIF\n NEXT I\n RETURN Total\nENDFUNCTION\nOUTPUT COUNT_BASE(\'A\', "GATTACA")', + expect: ['3'], + watch: 'The assembly beside this. Step the machine view of the count-base problem with the pseudo trace side by side: each pass of the FOR is about six instructions, and the registers panel shows rcx and rax doing what I and Total do.', + }, + showAsmOf: 'count-base', + problems: ['count-base'], + checks: [ + { question: 'What does a `cmp` instruction leave behind?', options: ['A new value in a register', 'Flags that the next conditional jump reads', 'Nothing'], answer: 1, why: 'cmp subtracts without storing and sets the flags; jl, je and friends read them.' }, + { question: 'In the loop, which instruction is the half-open bound?', options: ['inc rcx', 'cmp rcx, rdx followed by jl', 'movzx eax, byte [rdi + rcx]'], answer: 1, why: 'Continue while the position is less than the length: a compare and a jump-if-less.' }, + ], + flavors: [ + { id: 'machines', hook: 'The machine does not know a type from a hole in the ground. Reading a byte as a number is one instruction; what it means is the program\'s business.', task: 'In pseudo, print the number Unicode gives each character of "GATTACA" with ORD. That number is what the movzx instruction loads into eax.', solution: { pseudo: 'FOR EACH C IN "GATTACA"\n OUTPUT C, " ", ORD(C) // ~ the byte the machine sees\nNEXT C', expect: ['G 71', 'A 65', 'T 84', 'T 84', 'A 65', 'C 67', 'A 65'] } }, + { id: 'cells', hook: 'Four bases fit in two bits. Genomics tools pack them that way, and the packing is a few machine instructions per base.', task: 'Encode A, C, G, T as 0, 1, 2, 3 and pack "GAT" into one whole number, two bits per base (first base most significant). Print the number and its binary digits.', solution: { pseudo: 'FUNCTION CODE(B : CHAR) RETURNS INTEGER\n CASE OF B\n \'A\' : RETURN 0\n \'C\' : RETURN 1\n \'G\' : RETURN 2\n OTHERWISE : RETURN 3\n ENDCASE\nENDFUNCTION\nDECLARE Packed : INTEGER\nPacked <- 0\nFOR EACH B IN "GAT"\n Packed <- Packed * 4 + CODE(B) // ~ shift left two bits, add the code\nNEXT B\nDECLARE Bits : STRING\nDECLARE N : INTEGER\nBits <- ""\nN <- Packed\nFOR K <- 0 TO 6\n Bits <- TO_STRING(N MOD 2) & Bits // ~ the lowest bit, pushed to the front\n N <- N DIV 2\nNEXT K\nOUTPUT Packed, " ", Bits', expect: ['35 100011'] } }, + ], + }, + { + id: 'reading-agent-code', title: 'Reading code you did not write', stage: 'advanced', minutes: 30, + purpose: 'You will have a procedure for checking code from a colleague or an agent: state the contract, trace one input, test the boundary, and only then trust it.', + teaches: ['code-review'], requires: ['specification', 'tracing', 'runtime-stops', 'float-precision'], + explanation: [ + 'Most code you meet from now on will be written by someone or something else, quickly, and handed to you as finished. It usually looks right. The question is not whether it looks right but whether it is, and everything on this platform was built to answer that question in a fixed order.', + 'First, state the contract. What must be true of the inputs, and what is promised about the result? If the author did not write a `REQUIRE` and an `ENSURE`, write them yourself; the act of writing them is where half the bugs appear, because you discover that you cannot say what the code promises. Second, trace one input by hand, with predict mode on, and let the run correct you. Third, test the boundary: the empty input, the one-element input, the input where the answer is at the last position. Agents and people both get the middle right and the edges wrong. Fourth, read the stops: run it on a bad input and see whether it stops with a sentence or carries on with a wrong value.', + 'The example is a plausible agent answer to "return the position of the largest value". It has an `ENSURE`, it runs, and it prints a correct answer. Open the Tests view; the generated inputs include the one that breaks it. Then fix it and watch the proof go through. That sequence, contract, trace, boundary, stop, is the skill this whole platform exists to teach, and it transfers to every language you will read.', + '- Code from anyone, human or agent, is a claim. The contract is how you check it.', + '- Trace one input; test the boundaries; try a bad input and read the stop.', + '- Only then translate, ship, or build on it.', + ], + example: { + pseudo: '// An agent was asked: "return the position of the largest value in Xs". It wrote this.\nFUNCTION ARGMAX(Xs : LIST OF INTEGER) RETURNS INTEGER\n REQUIRE LENGTH(Xs) > 0\n ENSURE FORALL K IN [0, LENGTH(Xs)) : Xs[K] <= Xs[RESULT]\n DECLARE Best : INTEGER\n Best <- 0\n FOR I <- 1 TO LENGTH(Xs) - 1\n IF Xs[I] > Xs[Best] THEN\n Best <- I\n ENDIF\n NEXT I\n RETURN Best\nENDFUNCTION\nOUTPUT ARGMAX([3, 9, 2, 5])', + expect: ['1'], + watch: 'It prints 1, which is right. Open the Tests view: the generated inputs include one where the largest value is last, and the ENSURE fails. The loop bound is LENGTH(Xs) - 1: the last position is never looked at.', + }, + checks: [ + { question: 'An agent\'s function passes the one example you tried. What do you know?', options: ['It is correct', 'It is correct for that input', 'It is probably correct'], answer: 1, why: 'One run is one input. The contract, the boundary tests, and the proof speak for the rest.' }, + { question: 'Which input is most likely to break code that "looks right"?', options: ['A typical one', 'An empty list, or one where the answer is at the last position', 'A very large one'], answer: 1, why: 'Boundaries are where off-by-one and missing-case errors live.' }, + { question: 'The code has no REQUIRE or ENSURE. What is the first thing to do?', options: ['Run it', 'Write them yourself', 'Translate it to Python'], answer: 1, why: 'Until you can state what it promises, you cannot check anything about it.' }, + ], + flavors: [ + { id: 'money', hook: 'An agent wrote a function to total a statement and round to the nearest dollar. It passes the examples. The boundary is the half-dollar.', task: 'TO_DOLLARS(Cents) should round half up: 150 is 2 dollars, 149 is 1. The given version uses INT, which rounds toward zero. Write the ENSURE you would want, find the failing input, and fix it.', starter: 'FUNCTION TO_DOLLARS(Cents : INTEGER) RETURNS INTEGER\n REQUIRE Cents >= 0\n RETURN INT(Cents / 100)\nENDFUNCTION\nOUTPUT TO_DOLLARS(149), " ", TO_DOLLARS(150), " ", TO_DOLLARS(250)', solution: { pseudo: 'FUNCTION TO_DOLLARS(Cents : INTEGER) RETURNS INTEGER\n REQUIRE Cents >= 0\n ENSURE RESULT == (Cents + 50) DIV 100\n RETURN (Cents + 50) DIV 100\nENDFUNCTION\nOUTPUT TO_DOLLARS(149), " ", TO_DOLLARS(150), " ", TO_DOLLARS(250)', expect: ['1 2 3'] } }, + { id: 'cells', hook: 'An agent wrote a k-mer counter. It is right for every read except the ones where the last k-mer matters, which is all of them.', task: 'KMERS(T, K) should return how many k-mers of length K a read of length LENGTH(T) has: LENGTH - K + 1, or 0 when the read is shorter than K. The given version is off at the boundary. Add the ENSURE, find the failing input in Tests, fix it.', starter: 'FUNCTION KMERS(T : STRING, K : INTEGER) RETURNS INTEGER\n REQUIRE K >= 1\n DECLARE N : INTEGER\n N <- 0\n FOR I <- 0 TO LENGTH(T) - K\n N <- N + 1\n NEXT I\n RETURN N\nENDFUNCTION\nOUTPUT KMERS("GATTACA", 3), " ", KMERS("GA", 3), " ", KMERS("GAT", 3)', solution: { pseudo: 'FUNCTION KMERS(T : STRING, K : INTEGER) RETURNS INTEGER\n REQUIRE K >= 1\n ENSURE RESULT == MAX(0, LENGTH(T) - K + 1)\n DECLARE N : INTEGER\n N <- 0\n FOR I <- 0 TO LENGTH(T) - K + 1\n N <- N + 1\n NEXT I\n RETURN N\nENDFUNCTION\nOUTPUT KMERS("GATTACA", 3), " ", KMERS("GA", 3), " ", KMERS("GAT", 3)', expect: ['5 0 1'] } }, + { id: 'machines', hook: 'An agent translated a pseudo loop into a language with inclusive ranges and got the bound wrong. The pseudo contract catches it on the way back.', task: 'This SUM_TO(N) claims to return 0 + 1 + … + N. Its ENSURE says so. Run the Tests view, read which input breaks it, and fix the bound.', starter: 'FUNCTION SUM_TO(N : INTEGER) RETURNS INTEGER\n REQUIRE N >= 0\n ENSURE RESULT == N * (N + 1) DIV 2\n DECLARE S : INTEGER\n S <- 0\n FOR I <- 0 TO N\n S <- S + I\n NEXT I\n RETURN S\nENDFUNCTION\nOUTPUT SUM_TO(0), " ", SUM_TO(4), " ", SUM_TO(10)', solution: { pseudo: 'FUNCTION SUM_TO(N : INTEGER) RETURNS INTEGER\n REQUIRE N >= 0\n ENSURE RESULT == N * (N + 1) DIV 2\n DECLARE S : INTEGER\n S <- 0\n FOR I <- 0 TO N + 1\n S <- S + I\n NEXT I\n RETURN S\nENDFUNCTION\nOUTPUT SUM_TO(0), " ", SUM_TO(4), " ", SUM_TO(10)', expect: ['0 10 55'] } }, + ], + }, +]; diff --git a/src/curriculum/concepts.ts b/src/curriculum/concepts.ts new file mode 100644 index 0000000..375d49f --- /dev/null +++ b/src/curriculum/concepts.ts @@ -0,0 +1,58 @@ +import type { Concept } from './types.js'; + +/** + * Every concept a lesson can teach or require. The ids are the vocabulary of + * the dependency map, so a lesson may only name ids from this list. + */ +export const CONCEPTS: Concept[] = [ + // foundations + { id: 'procedure', name: 'A procedure is a list of steps', summary: 'A computer follows written steps exactly, in order, with no judgment of its own; what it does is entirely what the steps say.' }, + { id: 'value', name: 'Values', summary: 'Every piece of data is a value: 3, 2.5, \'A\', "GATTACA", TRUE.' }, + { id: 'type', name: 'Types', summary: 'Every value has exactly one type (INTEGER, FLOAT, CHAR, STRING, BOOL), and the type decides what can be done with it.' }, + { id: 'variable', name: 'Variables', summary: 'A variable is a named place that holds one value at a time; DECLARE makes the place, and its type never changes.' }, + { id: 'assignment', name: 'Assignment', summary: 'X <- E works out E first, then stores the result in X, replacing what was there; the old value is gone.' }, + { id: 'arithmetic', name: 'Arithmetic', summary: '+ - * ^ work as in school; / always gives a FLOAT; DIV and MOD split a whole-number division into quotient and remainder.' }, + { id: 'io', name: 'Output and input', summary: 'OUTPUT shows values in order with nothing between them; INPUT fills one variable from outside the program.' }, + { id: 'indexing', name: 'Positions', summary: 'Positions count from 0, so a text of length N has positions 0 to N-1, and T[I] is the character at position I.' }, + { id: 'boolean', name: 'TRUE and FALSE', summary: 'A comparison such as X < 10 is a value of type BOOL, and a BOOL is a value like any other.' }, + { id: 'selection', name: 'Choosing a path', summary: 'IF tests a condition and runs one body or another; the first TRUE condition wins; ENDIF closes it.' }, + { id: 'logic', name: 'Combining conditions', summary: 'AND, OR, and NOT build one condition from several; AND stops at the first FALSE, so the guard goes first.' }, + { id: 'for-loop', name: 'Counted loops', summary: 'FOR I <- A TO B runs the body with I = A, A+1, …, B-1; the counter changes between passes and has no value afterwards.' }, + { id: 'half-open', name: 'Half-open bounds', summary: 'A range [A, B) includes A and excludes B, so its length is B - A and ranges [0, K) and [K, N) meet without overlap.' }, + { id: 'accumulator', name: 'Accumulators', summary: 'A running total is a variable set before a loop and updated once per pass: Total <- Total + X.' }, + { id: 'tracing', name: 'Reading a run', summary: 'A trace is the sequence of states a program passes through; predicting the next state and checking is how you find out what the code does, as opposed to what you meant.' }, + // core + { id: 'for-each', name: 'Visiting every value', summary: 'FOR EACH X IN C runs the body once per value of a collection, in order, without naming positions.' }, + { id: 'while', name: 'Loops that test first', summary: 'WHILE tests before every pass and may run zero times; the body must move toward making the condition FALSE.' }, + { id: 'repeat', name: 'Loops that test after', summary: 'REPEAT … UNTIL runs the body at least once and stops when the condition becomes TRUE.' }, + { id: 'early-exit', name: 'Leaving early', summary: 'BREAK ends the loop at once, CONTINUE skips to the next pass, and RETURN ends the subroutine; a search uses one of them to stop at the first hit.' }, + { id: 'function', name: 'Functions', summary: 'A FUNCTION takes parameters, which are copies, and RETURNs one value; a call stands wherever a value stands.' }, + { id: 'procedure-sub', name: 'Procedures', summary: 'A PROCEDURE is a subroutine run for its effect with CALL; it returns nothing.' }, + { id: 'byref', name: 'Passing by reference', summary: 'A BYREF parameter is the caller\'s variable itself, so a change inside the procedure is a change outside it; everything else is a copy.' }, + { id: 'list', name: 'Lists', summary: 'A LIST holds any number of values of one type in order; it starts empty, APPEND grows it, and positions work as in a STRING.' }, + { id: 'nesting', name: 'Loops inside loops', summary: 'An inner loop runs completely for every pass of the outer one, so the body runs rows × columns times.' }, + { id: 'case', name: 'Many-way choice', summary: 'CASE OF picks the first arm whose value or half-open range matches, with OTHERWISE for the rest.' }, + { id: 'hashmap', name: 'Lookup by key', summary: 'A HASHMAP stores a value under a key; reading a missing key stops the run, so test IN first; keys come out smallest first.' }, + { id: 'set', name: 'Sets', summary: 'A SET holds distinct values in no order; ADD and IN are its operations, and adding a member twice changes nothing.' }, + { id: 'record', name: 'Records', summary: 'A TYPE groups named fields into one value; R.Field reaches one, and a record is how a function returns two things.' }, + { id: 'array', name: 'Fixed tables', summary: 'An ARRAY has a fixed length (or two), stated in its type, and is for tables whose shape is part of the problem.' }, + { id: 'runtime-stops', name: 'What stops a run', summary: 'A position that does not exist, a name with no value yet, a missing key, division by zero, and a number too large each stop the run with a sentence; no wrong answer comes out.' }, + { id: 'float-precision', name: 'Floats are approximate', summary: '0.1 + 0.2 is not 0.3; compare floats with a tolerance and keep money in whole cents.' }, + // advanced + { id: 'recursion', name: 'Recursion', summary: 'A function may call itself on a smaller input; a base case stops it, and DECREASES names the thing that shrinks.' }, + { id: 'memoization', name: 'Memoization', summary: 'Remembering answers in a table turns an exponential recursion into a linear one; in pseudo the table travels BYREF through a PROCEDURE.' }, + { id: 'specification', name: 'Specifications', summary: 'REQUIRE says what a caller promises, ENSURE says what the subroutine promises back; together they say what, not how.' }, + { id: 'invariant', name: 'Loop invariants', summary: 'An INVARIANT is true at the top of every pass; with the exit condition it proves the result, and DECREASES proves the loop ends.' }, + { id: 'cost', name: 'Counting cost', summary: 'The number of steps as a function of input size is the cost; a loop is N, a loop in a loop is N², and halving each pass is log N.' }, + { id: 'stack-queue', name: 'Stacks and queues', summary: 'A stack gives back the most recent value, a queue the oldest; both are a LIST used with discipline.' }, + { id: 'binary-search', name: 'Binary search', summary: 'In sorted data, comparing with the middle halves the candidates every step; the invariant is "the answer, if any, is in [Lo, Hi)".' }, + { id: 'sorting', name: 'Sorting', summary: 'Insertion sort keeps a sorted prefix and grows it by one; the invariant is the prefix, the cost is N² in the worst case.' }, + { id: 'graph', name: 'Graphs and breadth-first search', summary: 'A graph is a HASHMAP from each node to its neighbors; BFS visits by distance using a queue and a visited set.' }, + { id: 'machine-model', name: 'What the machine runs', summary: 'A processor has registers, flags, and memory; a FOR loop becomes a compare, a conditional jump, and an add.' }, + { id: 'numerics', name: 'Numbers for science', summary: 'SUM_OF, MEAN_OF, STDEV_OF, DOT, CHOOSE, the trigonometric and logarithmic functions and the random draws; counts stay INTEGER and measurements FLOAT.' }, + { id: 'equations', name: 'Equations as procedures', summary: 'A sum is a loop with an accumulator, a subscript is a position, a case bracket is an IF; the matho pane makes the translation and reports each decision.' }, + { id: 'parallel', name: 'Independent passes', summary: 'PARALLEL FOR promises that its passes do not depend on each other and stops with a data race when two passes touch one location; each pass owns a slot, and a reduction combines the slots afterwards.' }, + { id: 'code-review', name: 'Reading code you did not write', summary: 'Code from a colleague or an agent is a claim; state the specification, trace one input, and test the boundary before trusting it.' }, +]; + +export const CONCEPT_BY_ID: Record = Object.fromEntries(CONCEPTS.map((c) => [c.id, c])); diff --git a/src/curriculum/core.ts b/src/curriculum/core.ts new file mode 100644 index 0000000..2a3fa2f --- /dev/null +++ b/src/curriculum/core.ts @@ -0,0 +1,388 @@ +import type { Lesson } from './types.js'; + +/** Core: the rest of the notation, from FOR EACH to floats and the flavors of code. */ +export const CORE: Lesson[] = [ + { + id: 'for-each', title: 'Visiting every value', stage: 'core', minutes: 15, + purpose: 'You will be able to loop over a collection without naming positions, and know when positions are still needed.', + teaches: ['for-each'], requires: ['for-loop', 'logic'], + explanation: [ + '`FOR EACH B IN T` runs its body once for each value in `T`, in order, with `B` holding the current value. For a `STRING` that is each character. There is no counter and no position; when all you need is each value, this is the loop to write, because it cannot be off by one.', + 'You still need the counted `FOR` when the position matters: to compare a character with its neighbor, to fill a result at the same position, or to visit only part of the collection. A good test: if the body never mentions `I` except inside `T[I]`, the loop wants to be a `FOR EACH`.', + 'As with a `FOR` counter, `B` is made by the loop and has no value after `NEXT B`, and you may not assign to it. The loop variable is a copy of the value; changing it would not change the collection, so the notation does not let you.', + '- `FOR EACH X IN C` visits every value of C in order; X is the value, not a position.', + '- Use a counted FOR when the position itself is needed. Otherwise prefer FOR EACH.', + '- The loop variable cannot be assigned and has no value after NEXT.', + ], + example: { + pseudo: 'DECLARE GC : INTEGER\nGC <- 0\nFOR EACH B IN "GATTACA"\n IF B == \'G\' OR B == \'C\' THEN\n GC <- GC + 1\n ENDIF\nNEXT B\nOUTPUT GC\nDECLARE Vowels : INTEGER\nVowels <- 0\nFOR EACH C IN "procedure"\n IF C == \'a\' OR C == \'e\' OR C == \'i\' OR C == \'o\' OR C == \'u\' THEN\n Vowels <- Vowels + 1\n ENDIF\nNEXT C\nOUTPUT Vowels', + expect: ['2', '4'], + watch: 'B takes each character in turn; there is no I anywhere. Compare with the counted version from the accumulators lesson: same answer, one fewer thing to get wrong.', + }, + checks: [ + { question: 'Which loop needs a counted FOR rather than FOR EACH?', options: ['Count the T in a read', 'Print each word of a list', 'Count positions where T[I] equals T[I + 1]'], answer: 2, why: 'Comparing with a neighbor needs the position I to reach I + 1.' }, + { question: 'What is B inside `FOR EACH B IN "CAT"` on the second pass?', options: ['1', '\'A\'', '"A"'], answer: 1, why: 'B is the value at that point of the collection: the CHAR \'A\'.' }, + ], + flavors: [ + { id: 'cells', hook: 'Counting each kind of base is the start of every composition report. FOR EACH makes it four counters and no positions.', task: 'Count the A, C, G, and T in "GATTACAGC" and print the four counts on one line.', solution: { pseudo: 'DECLARE A, C, G, T : INTEGER\nA <- 0\nC <- 0\nG <- 0\nT <- 0\nFOR EACH B IN "GATTACAGC"\n IF B == \'A\' THEN\n A <- A + 1\n ELSEIF B == \'C\' THEN\n C <- C + 1 // ~ count a C\n ELSEIF B == \'G\' THEN\n G <- G + 1\n ELSE\n T <- T + 1 // ~ anything else here is a T\n ENDIF\nNEXT B\nOUTPUT A, " ", C, " ", G, " ", T', expect: ['3 2 2 2'] } }, + { id: 'words', hook: 'Counting spaces is counting words, nearly. Every text statistic starts with a FOR EACH over characters.', task: 'Count the spaces in "the quick brown fox" and print the number of words (spaces plus one).', solution: { pseudo: 'DECLARE Spaces : INTEGER\nSpaces <- 0\nFOR EACH C IN "the quick brown fox"\n IF C == \' \' THEN\n Spaces <- Spaces + 1 // ~ one more space\n ENDIF\nNEXT C\nOUTPUT Spaces + 1 // ~ words are spaces plus one', expect: ['4'] } }, + { id: 'games', hook: 'A hand of cards is a text of ranks. Scoring it is a FOR EACH with a CASE-like chain of IFs.', task: 'Score the hand "A7KT" where A is 11, K and T (ten) are 10, and a digit is its value.', solution: { pseudo: 'DECLARE Score : INTEGER\nScore <- 0\nFOR EACH R IN "A7KT"\n IF R == \'A\' THEN\n Score <- Score + 11\n ELSEIF R == \'K\' OR R == \'T\' THEN // ~ ten-valued cards\n Score <- Score + 10\n ELSE\n Score <- Score + TO_INTEGER(R) // ~ the digit\'s own value\n ENDIF\nNEXT R\nOUTPUT Score', expect: ['38'] } }, + ], + }, + { + id: 'conditional-loops', title: 'Loops that do not know how many passes', stage: 'core', minutes: 20, + purpose: 'You will be able to write a loop that runs until something becomes true, choose between WHILE and REPEAT, and say why it ends.', + teaches: ['while', 'repeat'], requires: ['for-loop', 'boolean', 'arithmetic'], + explanation: [ + 'A `FOR` knows how many passes before it starts. A `WHILE` does not: it tests a condition before every pass and runs the body only while the condition is `TRUE`. If the condition is `FALSE` at the start, the body runs zero times. `REPEAT … UNTIL` is the other way round: it runs the body, then tests, so the body runs at least once, and the loop stops when the condition becomes `TRUE`.', + 'Which one? Use `WHILE` when there might be nothing to do ("while there are reads left"). Use `REPEAT` when the first pass must happen before you can test ("ask for a number until it is valid"). When the number of passes is known, use `FOR`; a `WHILE` with a hand-made counter is a `FOR` in disguise and one more place to be off by one.', + 'Every `WHILE` carries a promise: something in the body moves toward making the condition `FALSE`. `N <- N DIV 2` halves `N` every pass, so `N > 1` eventually fails. A body that forgets its move loops forever; here the lab stops it after 20,000 steps and tells you which loop, and whether its condition had stopped changing. `DECREASES N` is how you write the promise down so the prover can check it.', + '- WHILE tests first and may run zero times. REPEAT runs first and tests after.', + '- Every conditional loop needs a step that moves toward stopping. Name it, then check it is in the body.', + '- A known number of passes is a FOR.', + ], + example: { + pseudo: 'DECLARE N, Halvings : INTEGER\nN <- 100\nHalvings <- 0\nWHILE N > 1\n N <- N DIV 2\n Halvings <- Halvings + 1\nENDWHILE\nOUTPUT Halvings\nDECLARE Tries : INTEGER\nTries <- 0\nREPEAT\n Tries <- Tries + 1\nUNTIL Tries * Tries > 50\nOUTPUT Tries', + expect: ['6', '8'], + watch: 'N: 100, 50, 25, 12, 6, 3, 1. The condition is tested seven times and TRUE six. Then Tries climbs to 8, the first number whose square passes 50: REPEAT tested after each pass.', + }, + checks: [ + { question: 'A WHILE whose condition is FALSE at the start. How many times does the body run?', options: ['0', '1', 'It depends on the body'], answer: 0, why: 'WHILE tests before the first pass.' }, + { question: 'A REPEAT whose UNTIL condition is TRUE from the start?', options: ['0', '1', 'Forever'], answer: 1, why: 'REPEAT runs the body once before it tests.' }, + { question: '`WHILE N > 0` with a body that never changes N. What happens?', options: ['The run stops with a sentence naming the loop', 'N becomes 0 eventually', 'The loop runs once'], answer: 0, why: 'Nothing moves toward stopping. The lab stops the run and says the condition stopped changing.' }, + ], + flavors: [ + { id: 'money', hook: 'How many years until savings double at 5 percent? Nobody knows the number of passes in advance; that is a WHILE.', task: 'Start with 10000 cents at 5% a year (whole cents, rounded down each year). Count the years until the balance is at least 20000, and print the years and the balance.', solution: { pseudo: 'DECLARE Balance, Years : INTEGER\nBalance <- 10000\nYears <- 0\nWHILE Balance < 20000 // ~ keep going until it has doubled\n Balance <- Balance + Balance * 5 DIV 100\n Years <- Years + 1\nENDWHILE\nOUTPUT Years, " ", Balance', expect: ['15 20780'] } }, + { id: 'cells', hook: 'Trimming low-quality bases from the end of a read is a WHILE: strip while the last base is poor, and stop when it is good or nothing is left.', task: 'Quality letters run from "!" (worst) upward. Strip trailing characters of "GATTNN" that are \'N\' and print what remains and how many were stripped.', solution: { pseudo: 'DECLARE R : STRING\nDECLARE Keep, Stripped : INTEGER\nR <- "GATTNN"\nKeep <- LENGTH(R)\nStripped <- 0\nWHILE Keep > 0 AND R[Keep - 1] == \'N\' // ~ the guard first: Keep > 0\n Keep <- Keep - 1\n Stripped <- Stripped + 1\nENDWHILE\nOUTPUT SLICE(R, 0, Keep), " ", Stripped', expect: ['GATT 2'] } }, + { id: 'games', hook: 'Rolling until you get a six has no fixed length. The REPEAT shape is "roll, then check".', task: 'The rolls arrive as input, one per line. REPEAT reading until a 6 arrives, then print how many rolls it took.', solution: { pseudo: 'DECLARE Roll, Rolls : INTEGER\nRolls <- 0\nREPEAT\n INPUT Roll\n Rolls <- Rolls + 1 // ~ count this roll\nUNTIL Roll == 6 // ~ stop at the first six\nOUTPUT Rolls', expect: ['4'], input: ['3', '1', '5', '6', '2'] } }, + ], + }, + { + id: 'functions', title: 'Naming a procedure so you can use it', stage: 'core', minutes: 25, + purpose: 'You will be able to write a FUNCTION with parameters and a result, call it from an expression, and explain why changing a parameter inside it changes nothing outside.', + teaches: ['function'], requires: ['accumulator', 'for-each'], + explanation: [ + 'A `FUNCTION` is a procedure with a name, inputs, and one answer. `FUNCTION AREA(W : INTEGER, H : INTEGER) RETURNS INTEGER` takes two whole numbers and promises a whole number back; `RETURN W * H` hands it over and ends the function at once. A call, `AREA(3, 4)`, stands wherever a value stands: in an `OUTPUT`, in an assignment, inside another call.', + 'Parameters are copies. When `BUMP(A)` is called, `X` inside `BUMP` starts with `A`\'s value, and `X <- X + 1` changes `X` only. Back outside, `A` is untouched. This is what lets you read a function on its own: nothing it does leaks out except through `RETURN`. In pseudo a `FUNCTION` cannot even ask for a reference parameter; a later lesson shows the `PROCEDURE`, which can.', + 'Why bother? Three reasons that compound. A name says what a block of code does, so the caller reads `GC_FRACTION(R)` instead of eight lines. The same code runs for every input without being copied. And a function with a clear contract can be tested, proved, and translated on its own, which is what the rest of the platform does with it.', + '- A FUNCTION takes parameters (copies) and RETURNs exactly one value of the declared type.', + '- A call stands where a value stands. RETURN ends the function immediately.', + '- Every variable a function uses is its own: declare it inside, or receive it as a parameter.', + ], + example: { + pseudo: 'FUNCTION AREA(W : INTEGER, H : INTEGER) RETURNS INTEGER\n RETURN W * H\nENDFUNCTION\nFUNCTION GC_FRACTION(T : STRING) RETURNS FLOAT\n DECLARE GC : INTEGER\n GC <- 0\n FOR EACH B IN T\n IF B == \'G\' OR B == \'C\' THEN\n GC <- GC + 1\n ENDIF\n NEXT B\n RETURN GC / LENGTH(T)\nENDFUNCTION\nFUNCTION BUMP(X : INTEGER) RETURNS INTEGER\n X <- X + 1\n RETURN X\nENDFUNCTION\nOUTPUT AREA(3, 4)\nOUTPUT AREA(3, 4) + AREA(1, 1)\nOUTPUT GC_FRACTION("GGCA")\nDECLARE A : INTEGER\nA <- 5\nOUTPUT BUMP(A), " ", A', + expect: ['12', '13', '0.75', '6 5'], + watch: 'The call stack. At OUTPUT BUMP(A) a new frame appears with X = 5; X becomes 6; the frame vanishes and A is still 5 in the frame below.', + }, + checks: [ + { question: '`FUNCTION F(X : INTEGER) RETURNS INTEGER` does `X <- 0` then `RETURN X`. After `N <- 7` and `OUTPUT F(N), N`, what prints?', options: ['00', '07', '77'], answer: 1, why: 'X is a copy. F returns 0; N is still 7.' }, + { question: 'Where can a call like `AREA(2, 3)` appear?', options: ['Only on a line of its own', 'Anywhere a value can', 'Only inside OUTPUT'], answer: 1, why: 'A function call is a value of the RETURNS type.' }, + { question: 'A function reaches its ENDFUNCTION without a RETURN. What happens?', options: ['It returns 0', 'The checker refuses it before the run', 'It returns the last variable'], answer: 1, why: 'A FUNCTION promises a value on every path, and the checker holds it to that.' }, + ], + flavors: [ + { id: 'cells', hook: 'The complement of a base, the GC content of a read, the number of codons: every bioinformatics library is a stack of small functions like these.', task: 'Write COMPLEMENT(B) returning the paired base (A-T, C-G), then REVERSE_COMPLEMENT(T) that builds the reverse complement of a read, and print it for "GATTACA".', solution: { pseudo: 'FUNCTION COMPLEMENT(B : CHAR) RETURNS CHAR\n IF B == \'A\' THEN\n RETURN \'T\'\n ELSEIF B == \'T\' THEN\n RETURN \'A\'\n ELSEIF B == \'C\' THEN\n RETURN \'G\' // ~ C pairs with G\n ENDIF\n RETURN \'C\'\nENDFUNCTION\nFUNCTION REVERSE_COMPLEMENT(T : STRING) RETURNS STRING\n DECLARE Out : STRING\n Out <- ""\n FOR EACH B IN T\n Out <- COMPLEMENT(B) & Out // ~ put the complement at the front, so the text reverses\n NEXT B\n RETURN Out\nENDFUNCTION\nOUTPUT REVERSE_COMPLEMENT("GATTACA")', expect: ['TGTAATC'] } }, + { id: 'money', hook: 'Tax, tip, and discount are each a function of an amount. Once written, the checkout calls them and never repeats the arithmetic.', task: 'Write WITH_TAX(Cents, Percent) that adds whole-cent tax rounded down, and DISCOUNT(Cents, Percent) that takes a percentage off. Print 2000 with 8% tax, and 2000 at 25% off then taxed.', solution: { pseudo: 'FUNCTION WITH_TAX(Cents : INTEGER, Percent : INTEGER) RETURNS INTEGER\n RETURN Cents + Cents * Percent DIV 100 // ~ add the tax in whole cents\nENDFUNCTION\nFUNCTION DISCOUNT(Cents : INTEGER, Percent : INTEGER) RETURNS INTEGER\n RETURN Cents - Cents * Percent DIV 100\nENDFUNCTION\nOUTPUT WITH_TAX(2000, 8)\nOUTPUT WITH_TAX(DISCOUNT(2000, 25), 8) // ~ a call inside a call', expect: ['2160', '1620'] } }, + { id: 'games', hook: 'A game needs the same check in many places: is this score a win, is this roll a double. A function is written once and trusted everywhere.', task: 'Write IS_WIN(Score) that is TRUE at 21 or more, and ROLL_VALUE(D1, D2) that returns the sum, doubled when the dice match. Print IS_WIN(ROLL_VALUE(6, 6)) and IS_WIN(ROLL_VALUE(6, 5)).', solution: { pseudo: 'FUNCTION IS_WIN(Score : INTEGER) RETURNS BOOL\n RETURN Score >= 21\nENDFUNCTION\nFUNCTION ROLL_VALUE(D1 : INTEGER, D2 : INTEGER) RETURNS INTEGER\n IF D1 == D2 THEN\n RETURN (D1 + D2) * 2 // ~ doubles count double\n ENDIF\n RETURN D1 + D2\nENDFUNCTION\nOUTPUT IS_WIN(ROLL_VALUE(6, 6)), " ", IS_WIN(ROLL_VALUE(6, 5))', expect: ['TRUE FALSE'] } }, + ], + }, + { + id: 'leaving-early', title: 'Stopping as soon as you know', stage: 'core', minutes: 20, + purpose: 'You will be able to end a loop or a function the moment the answer is known, in three equivalent ways, and say what the answer is when nothing is found.', + teaches: ['early-exit'], requires: ['function', 'while'], + explanation: [ + 'A search can stop at the first hit. There are three ways to write that, and they agree on the answer. Inside a `FUNCTION`, `RETURN I` the moment `T[I]` matches: the loop, the function, everything ends. In a loop that is not in a function, set the answer and `BREAK`, which leaves the innermost loop at once. Or write a `WHILE` whose condition says what you are waiting for: `WHILE J < LENGTH(T) AND T[J] != \'N\'`, with `J <- J + 1` as the body, which stops on its own at the first `N`.', + 'The question every search must answer is: what if nothing is found? Here the answer is `LENGTH(T)`, one past the last position, a value no real position can take, so the caller can tell "not found" from "found at the end". The `FUNCTION` returns it after the loop; the `BREAK` version starts `Found` at it; the `WHILE` version ends with `J` equal to it. Choose a not-found value on purpose and write it down.', + '`CONTINUE` is the smaller cousin of `BREAK`: it skips the rest of this pass and starts the next one. It is for "ignore this item and move on", and it keeps the body flat where an `IF` around everything would nest it.', + '- RETURN ends the function; BREAK ends the loop; CONTINUE ends the pass.', + '- A search that may fail needs a not-found value that no real answer can take. LENGTH is the usual one for positions.', + '- A WHILE whose condition names the stopping event needs no BREAK.', + ], + example: { + pseudo: 'FUNCTION FIRST_N(T : STRING) RETURNS INTEGER\n FOR I <- 0 TO LENGTH(T)\n IF T[I] == \'N\' THEN\n RETURN I\n ENDIF\n NEXT I\n RETURN LENGTH(T)\nENDFUNCTION\nDECLARE T : STRING\nT <- "GATNACA"\nDECLARE Found, J : INTEGER\nFound <- LENGTH(T)\nFOR I <- 0 TO LENGTH(T)\n IF T[I] == \'N\' THEN\n Found <- I\n BREAK\n ENDIF\nNEXT I\nJ <- 0\nWHILE J < LENGTH(T) AND T[J] != \'N\'\n J <- J + 1\nENDWHILE\nOUTPUT FIRST_N(T), " ", Found, " ", J\nOUTPUT FIRST_N("GATTACA")', + expect: ['3 3 3', '7'], + watch: 'Each of the three searches stops at position 3 and never looks at position 4. The last line searches a text with no N and gets 7, the length: the not-found value.', + }, + checks: [ + { question: 'A search over positions finds nothing. Which not-found value is safe?', options: ['0', 'LENGTH(T)', 'The last position'], answer: 1, why: '0 and the last position are real positions. LENGTH(T) is the one value no position can be.' }, + { question: 'Inside two nested loops, BREAK in the inner one does what?', options: ['Leaves both loops', 'Leaves the inner loop only', 'Ends the function'], answer: 1, why: 'BREAK leaves the innermost loop. The outer loop continues with its next pass.' }, + ], + flavors: [ + { id: 'cells', hook: 'Finding the first start codon ATG in a read is the first step of finding a gene. A search that scans the whole read when the answer was at position 2 wastes most of its work.', task: 'Write FIRST_ATG(T) that returns the position of the first "ATG" in T, or LENGTH(T) when there is none, and print it for "CCATGGATG" and "CCCC".', solution: { pseudo: 'FUNCTION FIRST_ATG(T : STRING) RETURNS INTEGER\n FOR I <- 0 TO LENGTH(T) - 2 // ~ the last place a 3-letter codon can start\n IF SLICE(T, I, I + 3) == "ATG" THEN\n RETURN I // ~ stop at the first one\n ENDIF\n NEXT I\n RETURN LENGTH(T)\nENDFUNCTION\nOUTPUT FIRST_ATG("CCATGGATG"), " ", FIRST_ATG("CCCC")', expect: ['2 4'] } }, + { id: 'money', hook: 'Which month does the debt reach zero? The schedule stops there; printing the rest would be wrong.', task: 'A 1000-cent debt is paid at 300 a month. Print the payments month by month and stop with "paid off in month N" as soon as the debt is 0 or less.', solution: { pseudo: 'DECLARE Debt : INTEGER\nDebt <- 1000\nFOR Month <- 1 TO 100\n Debt <- Debt - 300\n IF Debt <= 0 THEN\n OUTPUT "paid off in month ", Month\n BREAK // ~ the schedule ends here\n ENDIF\n OUTPUT "month ", Month, ": ", Debt, " left"\nNEXT Month', expect: ['month 1: 700 left', 'month 2: 400 left', 'month 3: 100 left', 'paid off in month 4'] } }, + { id: 'words', hook: 'Skipping punctuation while counting letters is what CONTINUE is for: ignore this character, move on.', task: 'Count the letters in "a-b, c!" by skipping anything that is not a lowercase letter with CONTINUE.', solution: { pseudo: 'DECLARE Letters : INTEGER\nLetters <- 0\nFOR EACH C IN "a-b, c!"\n IF C < \'a\' OR C > \'z\' THEN\n CONTINUE // ~ not a letter: skip the rest of this pass\n ENDIF\n Letters <- Letters + 1\nNEXT C\nOUTPUT Letters', expect: ['3'] } }, + ], + }, + { + id: 'procedures-and-byref', title: 'Procedures, and changing the caller\'s variable', stage: 'core', minutes: 20, + purpose: 'You will be able to write a PROCEDURE, call it, and say exactly when a change inside it reaches the outside.', + teaches: ['procedure-sub', 'byref'], requires: ['function'], + explanation: [ + 'A `PROCEDURE` is a subroutine run for what it does, not for an answer. It has no `RETURNS` and is run on a line of its own with `CALL`. Printing a report, swapping two values, and sorting a list in place are procedures: the effect is the point.', + 'By default a parameter is a copy, exactly as in a function, so `CALL RESET_COPY(A)` changes nothing outside. `BYREF` changes that: a `BYREF X : INTEGER` parameter is the caller\'s variable itself under another name, so `X <- 0` inside is `A <- 0` outside. Only a `PROCEDURE` may ask for `BYREF`, and only a variable, not a literal or an expression, may be passed to it. The word is there so the reader of the call can see that `A` may come back changed.', + 'This is the first time a subroutine can reach outside itself, and it is the reason `BYREF` is rare on purpose. A function that returns a value is easier to read, test, and prove than a procedure that changes its arguments. Use `BYREF` when the effect on the caller\'s variable is the whole job (swap, sort in place, fill a table), and a `FUNCTION` otherwise.', + '- PROCEDURE … ENDPROCEDURE, run with CALL, returns nothing.', + '- Parameters are copies unless marked BYREF, in which case they are the caller\'s own variables.', + '- Prefer a FUNCTION with a RETURN. Reach for BYREF when changing the caller\'s variable is the job.', + ], + example: { + pseudo: 'PROCEDURE RESET_COPY(X : INTEGER)\n X <- 0\nENDPROCEDURE\nPROCEDURE RESET(BYREF X : INTEGER)\n X <- 0\nENDPROCEDURE\nPROCEDURE SWAP(BYREF P : INTEGER, BYREF Q : INTEGER)\n DECLARE Temp : INTEGER\n Temp <- P\n P <- Q\n Q <- Temp\nENDPROCEDURE\nDECLARE A, B : INTEGER\nA <- 7\nB <- 2\nCALL RESET_COPY(A)\nOUTPUT A\nCALL RESET(A)\nOUTPUT A\nA <- 7\nCALL SWAP(A, B)\nOUTPUT A, " ", B', + expect: ['7', '0', '2 7'], + watch: 'The frames. In RESET_COPY, X is its own box and A outside does not move. In RESET, X and A are one box; the panel marks X as a reference and A changes with it.', + }, + checks: [ + { question: '`PROCEDURE P(X : INTEGER)` sets X to 9. After `N <- 1` and `CALL P(N)`, N is?', options: ['9', '1', 'No value'], answer: 1, why: 'Without BYREF, X is a copy.' }, + { question: 'Which call is refused by the checker?', options: ['CALL SWAP(A, B)', 'CALL SWAP(A, 5)', 'CALL SWAP(B, A)'], answer: 1, why: 'A BYREF parameter must be a variable: there is no variable 5 to change.' }, + { question: 'What is the best shape for "the largest of a list"?', options: ['A PROCEDURE with a BYREF result', 'A FUNCTION that RETURNs it', 'Either'], answer: 1, why: 'A value that comes back is a function\'s job. BYREF is for changing the caller\'s variable.' }, + ], + flavors: [ + { id: 'cells', hook: 'Normalizing a read in place (uppercase, N for anything unknown) changes the caller\'s variable. That is the shape of a cleaning step in a pipeline.', task: 'Write CLEAN(BYREF R) that rewrites R with every character uppercased and anything other than A, C, G, T replaced by N. Clean "gatxaca" and print it.', solution: { pseudo: 'PROCEDURE CLEAN(BYREF R : STRING)\n DECLARE Out : STRING\n Out <- ""\n FOR EACH C IN UPPER(R)\n IF C == \'A\' OR C == \'C\' OR C == \'G\' OR C == \'T\' THEN\n Out <- Out & C\n ELSE\n Out <- Out & \'N\' // ~ anything else becomes N\n ENDIF\n NEXT C\n R <- Out // ~ the change reaches the caller through BYREF\nENDPROCEDURE\nDECLARE Read : STRING\nRead <- "gatxaca"\nCALL CLEAN(Read)\nOUTPUT Read', expect: ['GATNACA'] } }, + { id: 'money', hook: 'Transferring money changes two balances at once. A function returning one value cannot do that; a procedure with two BYREF parameters can.', task: 'Write TRANSFER(BYREF Payer, BYREF Payee, Amount) that moves Amount only when From can cover it. Transfer 300 from a 500 balance to a 100 balance, then try 900, and print both balances after each.', solution: { pseudo: 'PROCEDURE TRANSFER(BYREF Payer : INTEGER, BYREF Payee : INTEGER, Amount : INTEGER)\n IF Payer >= Amount THEN // ~ only when covered\n Payer <- Payer - Amount\n Payee <- Payee + Amount\n ENDIF\nENDPROCEDURE\nDECLARE Alice, Bob : INTEGER\nAlice <- 500\nBob <- 100\nCALL TRANSFER(Alice, Bob, 300)\nOUTPUT Alice, " ", Bob\nCALL TRANSFER(Alice, Bob, 900)\nOUTPUT Alice, " ", Bob', expect: ['200 400', '200 400'] } }, + { id: 'games', hook: 'A turn changes the score and the position of the player at once: two effects, one procedure.', task: 'Write TAKE_TURN(BYREF Square, BYREF Score, Roll) that moves the player Roll squares and adds 10 to the score if the new square is a multiple of 5. Take turns with rolls 3 and 2 from square 0 and print the state after each.', solution: { pseudo: 'PROCEDURE TAKE_TURN(BYREF Square : INTEGER, BYREF Score : INTEGER, Roll : INTEGER)\n Square <- Square + Roll\n IF Square MOD 5 == 0 THEN // ~ a bonus square\n Score <- Score + 10\n ENDIF\nENDPROCEDURE\nDECLARE Sq, Pts : INTEGER\nSq <- 0\nPts <- 0\nCALL TAKE_TURN(Sq, Pts, 3)\nOUTPUT Sq, " ", Pts\nCALL TAKE_TURN(Sq, Pts, 2)\nOUTPUT Sq, " ", Pts', expect: ['3 0', '5 10'] } }, + ], + }, + { + id: 'lists', title: 'Collections that grow', stage: 'core', minutes: 25, + purpose: 'You will be able to build a LIST one value at a time, reach its positions, and change it with APPEND, INSERT, and REMOVE.', + teaches: ['list'], requires: ['for-each', 'indexing'], + explanation: [ + 'A `LIST OF STRING` holds any number of texts, in order, and starts empty. `APPEND(L, V)` adds `V` at the end. Positions work exactly as in a `STRING`: `L[0]` is the first value, `L[LENGTH(L) - 1]` the last, and `L[LENGTH(L)]` does not exist. `FOR EACH X IN L` visits every value; `FOR I <- 0 TO LENGTH(L)` visits every position.', + '`INSERT(L, I, V)` puts `V` at position `I` and moves everything from `I` onward one place to the right. `REMOVE(L, I)` takes the value at `I` out and moves everything after it one place left. Both change the list in place, and both shift every later value, which the trace shows and a later lesson measures as cost.', + 'A list is how a program keeps "all of them" rather than one at a time. Where the accumulators lesson used four `IF`s to pretend it had a list of prices, you now write the prices once and loop over them. Every value in a list has the same type; a list of records (later) is how mixed data travels together.', + '- A LIST starts empty, grows with APPEND, and is indexed like a STRING.', + '- INSERT and REMOVE shift every later value by one position.', + '- `[1, 2, 3]` is a list literal, handy for examples and tests.', + '- Text and lists convert both ways: `SPLIT(S, ",")` cuts a text into a LIST OF STRING at every comma, and `JOIN(L, ",")` glues one back together.', + ], + example: { + pseudo: 'DECLARE Reads : LIST OF STRING\nAPPEND(Reads, "GAT")\nAPPEND(Reads, "TAC")\nAPPEND(Reads, "ACA")\nOUTPUT LENGTH(Reads)\nOUTPUT Reads[1]\nINSERT(Reads, 0, "AAAA")\nREMOVE(Reads, 2)\nOUTPUT Reads\nDECLARE Longest : INTEGER\nLongest <- 0\nFOR EACH R IN Reads\n IF LENGTH(R) > Longest THEN\n Longest <- LENGTH(R)\n ENDIF\nNEXT R\nOUTPUT Longest\nOUTPUT JOIN(Reads, ",")\nOUTPUT SPLIT("A,B,C", ",")', + expect: ['3', 'TAC', '["AAAA", "GAT", "ACA"]', '4', 'AAAA,GAT,ACA', '["A", "B", "C"]'], + watch: 'The list ruler. INSERT at 0 pushes every read one position right; REMOVE at 2 pulls the last one back. Watch which value sits at position 1 before and after.', + }, + checks: [ + { question: 'After `APPEND(L, 5)` on an empty list, what is `L[1]`?', options: ['5', 'The run stops', '0'], answer: 1, why: 'The list has one value, at position 0. Position 1 does not exist.' }, + { question: '`L` is `[10, 20, 30]`. After `REMOVE(L, 0)`, what is `L[0]`?', options: ['10', '20', 'No value'], answer: 1, why: 'Removing position 0 shifts everything left: 20 is now first.' }, + ], + flavors: [ + { id: 'cells', hook: 'A FASTQ file is a list of reads. Filtering it by length is the first step of every pipeline: build a new list from the ones that pass.', task: 'From the reads ["GATTACA", "GA", "TTTTCC", "C"], build a list of those at least 4 long and print it with its length.', solution: { pseudo: 'DECLARE Kept : LIST OF STRING\nFOR EACH R IN ["GATTACA", "GA", "TTTTCC", "C"]\n IF LENGTH(R) >= 4 THEN\n APPEND(Kept, R) // ~ keep the long ones\n ENDIF\nNEXT R\nOUTPUT Kept\nOUTPUT LENGTH(Kept)', expect: ['["GATTACA", "TTTTCC"]', '2'] } }, + { id: 'money', hook: 'A basket is a list of prices. The total, the dearest item, and removing one item are all list operations.', task: 'Prices are [299, 1250, 499, 875]. Remove the dearest, then print the remaining list and its total.', solution: { pseudo: 'DECLARE Basket : LIST OF INTEGER\nDECLARE Dearest, Total : INTEGER\nBasket <- [299, 1250, 499, 875]\nDearest <- 0\nFOR I <- 0 TO LENGTH(Basket)\n IF Basket[I] > Basket[Dearest] THEN\n Dearest <- I // ~ remember the position, not the price\n ENDIF\nNEXT I\nREMOVE(Basket, Dearest)\nTotal <- 0\nFOR EACH P IN Basket\n Total <- Total + P\nNEXT P\nOUTPUT Basket\nOUTPUT Total', expect: ['[299, 499, 875]', '1673'] } }, + { id: 'games', hook: 'A deck is a list; dealing is REMOVE from the top; a discard pile is APPEND. A card game is lists all the way down.', task: 'The deck is ["A", "K", "Q", "J"]. Deal two cards from the front into a hand, then print the hand and the deck.', solution: { pseudo: 'DECLARE Deck, Hand : LIST OF STRING\nDeck <- ["A", "K", "Q", "J"]\nFOR N <- 0 TO 2\n APPEND(Hand, Deck[0]) // ~ the top card goes to the hand\n REMOVE(Deck, 0) // ~ and leaves the deck\nNEXT N\nOUTPUT Hand\nOUTPUT Deck', expect: ['["A", "K"]', '["Q", "J"]'] } }, + ], + }, + { + id: 'nested-loops', title: 'Loops inside loops', stage: 'core', minutes: 20, + purpose: 'You will be able to say how many times the body of a nested loop runs and in what order, and write a loop over pairs that visits each pair once.', + teaches: ['nesting'], requires: ['accumulator', 'half-open'], + explanation: [ + 'When one loop sits inside another, the inner loop runs to completion for every pass of the outer one. `FOR R <- 0 TO 3` around `FOR C <- 0 TO 4` runs the body 3 × 4 = 12 times, in the order (0,0), (0,1), (0,2), (0,3), (1,0), …: the inner counter changes fastest. This is how you visit every cell of a grid, every pair of items, every position of every read.', + 'The pattern for pairs is worth learning as a unit. `FOR I <- 0 TO N` around `FOR J <- I + 1 TO N` visits each unordered pair once: (0,1), (0,2), (0,3), (1,2), (1,3), (2,3). Starting `J` at `I + 1` skips both the pair of an item with itself and the pair already seen the other way round. The count is N(N-1)/2, which for N = 4 is 6.', + 'Nesting multiplies work. A body that runs N times inside a loop that runs N times runs N² times; for N = 1000 that is a million. The cost lesson measures this; for now, notice it, and notice that the trace\'s loop-pass jump (Shift+→) steps the outer loop while the inner one runs invisibly.', + '- The inner loop runs completely for every pass of the outer loop. Passes multiply.', + '- Each pair once: inner counter starts at the outer counter plus one.', + '- Close loops in the right order: the inner NEXT comes first.', + ], + example: { + pseudo: 'DECLARE Pairs : INTEGER\nPairs <- 0\nFOR I <- 0 TO 4\n FOR J <- I + 1 TO 4\n OUTPUT I, "-", J\n Pairs <- Pairs + 1\n NEXT J\nNEXT I\nOUTPUT Pairs', + expect: ['0-1', '0-2', '0-3', '1-2', '1-3', '2-3', '6'], + watch: 'I stays at 0 while J runs 1, 2, 3; then I becomes 1 and J runs 2, 3. Every pair appears once and no pair appears with itself.', + }, + checks: [ + { question: '`FOR I <- 0 TO 5` around `FOR J <- 0 TO 3`: how many times does the body run?', options: ['8', '15', '18'], answer: 1, why: '5 outer passes × 3 inner passes = 15.' }, + { question: 'To visit each pair of 6 items once, the inner loop is?', options: ['FOR J <- 0 TO 6', 'FOR J <- I TO 6', 'FOR J <- I + 1 TO 6'], answer: 2, why: 'Starting at I + 1 skips the self-pair and the reverse of pairs already seen.' }, + ], + flavors: [ + { id: 'cells', hook: 'Comparing every read with every other read is the shape of an all-against-all alignment, and the reason those programs take so long.', task: 'For the reads ["GAT", "GAC", "TTT"], count how many pairs share the same first base, visiting each pair once.', solution: { pseudo: 'DECLARE Reads : LIST OF STRING\nDECLARE Alike : INTEGER\nReads <- ["GAT", "GAC", "TTT"]\nAlike <- 0\nFOR I <- 0 TO LENGTH(Reads)\n FOR J <- I + 1 TO LENGTH(Reads) // ~ each pair once\n IF Reads[I][0] == Reads[J][0] THEN\n Alike <- Alike + 1\n ENDIF\n NEXT J\nNEXT I\nOUTPUT Alike', expect: ['1'] } }, + { id: 'games', hook: 'A board is rows of columns. Drawing it, counting pieces, and checking lines are all nested loops.', task: 'Print a 3-row, 4-column board of dots, then the number of squares.', solution: { pseudo: 'DECLARE Squares : INTEGER\nSquares <- 0\nFOR R <- 0 TO 3\n DECLARE Row : STRING\n Row <- ""\n FOR C <- 0 TO 4 // ~ the columns of this row\n Row <- Row & "."\n Squares <- Squares + 1\n NEXT C\n OUTPUT Row\nNEXT R\nOUTPUT Squares', expect: ['....', '....', '....', '12'] } }, + { id: 'everyday', hook: 'A round-robin tournament, a seating plan, a times table: anything with two independent choices is a nested loop.', task: 'Print the times table for 1 to 3 by 1 to 3, one row per line, with values separated by spaces.', solution: { pseudo: 'FOR A <- 1 TO 4\n DECLARE Row : STRING\n Row <- ""\n FOR B <- 1 TO 4\n Row <- Row & TO_STRING(A * B) & " " // ~ one product\n NEXT B\n OUTPUT Row\nNEXT A', expect: ['1 2 3 ', '2 4 6 ', '3 6 9 '] } }, + ], + }, + { + id: 'many-way-choice', title: 'Choosing among many with CASE', stage: 'core', minutes: 15, + purpose: 'You will be able to replace a chain of IFs on one value with a CASE, including half-open ranges and an OTHERWISE.', + teaches: ['case'], requires: ['selection', 'function', 'half-open'], + explanation: [ + 'When one value decides among several paths, `CASE OF` says so in one place. Each arm is a label and a body: a single value (`\'A\' :`), several values (`\'G\', \'C\' :`), or a half-open range (`90 TO 101 :`, which matches 90 up to and including 100). The first arm that matches runs; `OTHERWISE` runs when none does and must come last. `ENDCASE` closes it.', + 'Ranges in `CASE` are half-open, as everywhere in the notation. `80 TO 90` covers 80 to 89, and `90 TO 101` covers 90 to 100, so the two meet without gap or overlap. This is the same reason `FOR` uses half-open bounds: adjacent ranges fit together by sharing their end and start.', + 'A `CASE` is not more powerful than `IF`; it is clearer when every test is "is X this". If the tests compare different variables, or need `AND`, keep the `IF` chain. The checker will tell you when an arm can never run because an earlier one covers it.', + '- CASE OF X picks the first matching arm: a value, a list of values, or a half-open range.', + '- OTHERWISE catches the rest and comes last.', + '- Use CASE when every test is on the same value.', + ], + example: { + pseudo: 'FUNCTION GRADE(Score : INTEGER) RETURNS CHAR\n CASE OF Score\n 90 TO 101 : RETURN \'A\'\n 80 TO 90 : RETURN \'B\'\n 70 TO 80 : RETURN \'C\'\n OTHERWISE : RETURN \'F\'\n ENDCASE\nENDFUNCTION\nOUTPUT GRADE(90), GRADE(89), GRADE(70), GRADE(12)\nDECLARE Base : CHAR\nBase <- \'C\'\nCASE OF Base\n \'A\', \'T\' : OUTPUT "weak pair"\n \'G\', \'C\' : OUTPUT "strong pair"\n OTHERWISE : OUTPUT "not a base"\nENDCASE', + expect: ['ABCF', 'strong pair'], + watch: 'GRADE(90): the first arm, 90 TO 101, matches because 90 is included. GRADE(89) falls through to 80 TO 90 because 90 is excluded from that arm. The two arms meet at 90 exactly once.', + }, + checks: [ + { question: 'Which values does the arm `10 TO 20 :` match?', options: ['10 to 20', '10 to 19', '11 to 20'], answer: 1, why: 'Ranges are half-open: the end is excluded.' }, + { question: 'Where must OTHERWISE go?', options: ['First', 'Anywhere', 'Last'], answer: 2, why: 'OTHERWISE catches what nothing above matched, so it must be last.' }, + ], + flavors: [ + { id: 'cells', hook: 'Translating a codon to an amino acid is a 64-way CASE. Here are four arms of it.', task: 'Write AMINO(Codon) with arms for "ATG" (M), "TGG" (W), the three stop codons "TAA", "TAG", "TGA" (*), and X otherwise. Print it for "ATG", "TAG", "CCC".', solution: { pseudo: 'FUNCTION AMINO(Codon : STRING) RETURNS CHAR\n CASE OF Codon\n "ATG" : RETURN \'M\'\n "TGG" : RETURN \'W\'\n "TAA", "TAG", "TGA" : RETURN \'*\' // ~ the three stop codons share one arm\n OTHERWISE : RETURN \'X\'\n ENDCASE\nENDFUNCTION\nOUTPUT AMINO("ATG"), AMINO("TAG"), AMINO("CCC")', expect: ['M*X'] } }, + { id: 'money', hook: 'Tax bands are half-open ranges of income, and a CASE states them the way the tax table does.', task: 'Write RATE(Income) returning 0 below 10000, 20 from 10000 below 50000, and 40 at 50000 or more. Print the rate for 9999, 10000, 50000.', solution: { pseudo: 'FUNCTION RATE(Income : INTEGER) RETURNS INTEGER\n CASE OF Income\n 0 TO 10000 : RETURN 0\n 10000 TO 50000 : RETURN 20 // ~ the middle band, half-open\n OTHERWISE : RETURN 40\n ENDCASE\nENDFUNCTION\nOUTPUT RATE(9999), " ", RATE(10000), " ", RATE(50000)', expect: ['0 20 40'] } }, + { id: 'games', hook: 'What a die roll means in a board game is a CASE: a value, a few values, or a range.', task: 'Write MOVE(Roll) returning "lose a turn" for 1, "move 2" for 2 or 3, "move 4" for 4 and 5, and "roll again" for 6. Print MOVE(3) and MOVE(6).', solution: { pseudo: 'FUNCTION MOVE(Roll : INTEGER) RETURNS STRING\n CASE OF Roll\n 1 : RETURN "lose a turn"\n 2, 3 : RETURN "move 2" // ~ two values, one arm\n 4 TO 6 : RETURN "move 4"\n OTHERWISE : RETURN "roll again"\n ENDCASE\nENDFUNCTION\nOUTPUT MOVE(3), ", ", MOVE(6)', expect: ['move 2, roll again'] } }, + ], + }, + { + id: 'hashmaps', title: 'Looking things up by key', stage: 'core', minutes: 25, + purpose: 'You will be able to tally values with a HASHMAP, avoid the missing-key stop, and loop over keys.', + teaches: ['hashmap'], requires: ['list', 'logic'], + explanation: [ + 'A `HASHMAP OF CHAR : INTEGER` stores a whole number under each character. `Tally[\'A\'] <- 3` stores; `Tally[\'A\']` reads. There is no position; the key is the address. Where a `LIST` answers "what is at position 7", a `HASHMAP` answers "what is stored under `\'A\'`", and it does so in one step however many keys it holds.', + 'Reading a key that is not there stops the run: there is no value to return and the notation does not invent a 0. So the tally idiom is an `IF`: when `K IN Tally`, add one to `Tally[K]`; otherwise store 1, the first sighting. `IN` tests for a key. `DELETE(Tally, K)` removes one. `FOR EACH K IN Tally` visits the keys, smallest first, so a printed tally always comes out in a fixed order and two runs can be compared line by line.', + 'A hashmap is the right tool when the thing you look up by is a value, not a count: a word, a codon, a name, an account number. It is the structure behind word counts, codon tables, and every index you have ever used.', + '- H[K] <- V stores under the key; H[K] reads. The key is the address.', + '- A missing key stops the run. Test `K IN H` first, or store a starting value.', + '- FOR EACH K IN H visits the keys smallest first.', + ], + example: { + pseudo: 'DECLARE Tally : HASHMAP OF CHAR : INTEGER\nFOR EACH B IN "GATTACA"\n IF B IN Tally THEN\n Tally[B] <- Tally[B] + 1\n ELSE\n Tally[B] <- 1\n ENDIF\nNEXT B\nOUTPUT Tally\nFOR EACH K IN Tally\n OUTPUT K, " ", Tally[K]\nNEXT K\nOUTPUT \'N\' IN Tally', + expect: ['{\'A\': 3, \'C\': 1, \'G\': 1, \'T\': 2}', 'A 3', 'C 1', 'G 1', 'T 2', 'FALSE'], + watch: 'Tally gains a key with the value 1 the first time each base appears, and counts up after that. Replace the IF with the bare Tally[B] <- Tally[B] + 1 and run again: the first read stops the run with "missing key".', + }, + checks: [ + { question: 'Reading `H["x"]` when "x" was never stored does what?', options: ['Gives 0', 'Gives an empty value', 'Stops the run'], answer: 2, why: 'There is nothing to return. Test with IN first.' }, + { question: 'In what order does `FOR EACH K IN H` visit keys?', options: ['The order they were added', 'Smallest first', 'Random'], answer: 1, why: 'Keys come out sorted, so output is the same on every run.' }, + ], + flavors: [ + { id: 'cells', hook: 'Counting k-mers, the short words of a genome, is the hashmap in bioinformatics. A 3-mer count is the same tally with a SLICE as the key.', task: 'Count every 3-mer of "GATGATGAC" and print the tally.', solution: { pseudo: 'DECLARE T : STRING\nDECLARE Kmers : HASHMAP OF STRING : INTEGER\nT <- "GATGATGAC"\nFOR I <- 0 TO LENGTH(T) - 2 // ~ the last 3-mer starts at LENGTH - 3\n DECLARE K : STRING\n K <- SLICE(T, I, I + 3)\n IF K IN Kmers THEN\n Kmers[K] <- Kmers[K] + 1\n ELSE\n Kmers[K] <- 1 // ~ the first sighting of this 3-mer\n ENDIF\nNEXT I\nOUTPUT Kmers', expect: ['{"ATG": 2, "GAC": 1, "GAT": 2, "TGA": 2}'] } }, + { id: 'money', hook: 'Spending by category is a hashmap from category to total: every budgeting app is this loop.', task: 'Purchases arrive as pairs of category and cents: food 1200, rent 50000, food 800, fun 2500. Total each category and print the map.', solution: { pseudo: 'DECLARE Cats : LIST OF STRING\nDECLARE Cents : LIST OF INTEGER\nDECLARE Totals : HASHMAP OF STRING : INTEGER\nCats <- ["food", "rent", "food", "fun"]\nCents <- [1200, 50000, 800, 2500]\nFOR I <- 0 TO LENGTH(Cats)\n IF Cats[I] IN Totals THEN\n Totals[Cats[I]] <- Totals[Cats[I]] + Cents[I] // ~ add this purchase to its category\n ELSE\n Totals[Cats[I]] <- Cents[I]\n ENDIF\nNEXT I\nOUTPUT Totals', expect: ['{"food": 2000, "fun": 2500, "rent": 50000}'] } }, + { id: 'words', hook: 'A word count is the oldest hashmap program there is. Letters first; words are the same loop with a longer key.', task: 'Count the letters of "banana" and print the letter that appears most, with its count.', solution: { pseudo: 'DECLARE Counts : HASHMAP OF CHAR : INTEGER\nDECLARE Top : CHAR\nDECLARE Most : INTEGER\nFOR EACH C IN "banana"\n IF C IN Counts THEN\n Counts[C] <- Counts[C] + 1\n ELSE\n Counts[C] <- 1\n ENDIF\nNEXT C\nTop <- \' \'\nMost <- 0\nFOR EACH K IN Counts\n IF Counts[K] > Most THEN // ~ a new most-frequent letter\n Most <- Counts[K]\n Top <- K\n ENDIF\nNEXT K\nOUTPUT Top, " ", Most', expect: ['a 3'] } }, + ], + }, + { + id: 'sets', title: 'Distinct things', stage: 'core', minutes: 15, + purpose: 'You will be able to use a SET to answer "have I seen this" and "how many different", and know when a set beats a list.', + teaches: ['set'], requires: ['list', 'logic'], + explanation: [ + 'A `SET OF STRING` holds distinct texts in no particular order. `ADD(S, V)` puts a value in; adding one already there changes nothing. `V IN S` asks whether it is there, in one step. `LENGTH(S)` is how many distinct values it holds. There is no `S[0]`: a set has no positions, because it has no order.', + 'Two questions make a set the right tool: "have I seen this before" and "how many different ones are there". With a list you would scan every value to answer the first, and the scan grows with the list. With a set the answer is one step whatever the size. The duplicates program in the example is the shape: `IF W IN Seen` then it is a repeat, else `ADD`.', + 'A fixed set with a name is written in two lines: `TYPE BaseSet = SET OF CHAR` names the kind, and `DEFINE Purines (\'A\', \'G\') : BaseSet` makes one. Membership tests against a named set read like the biology: `IF B IN Purines`.', + '- A SET holds distinct values; ADD twice is ADD once; IN is one step.', + '- No positions and no order: a set answers "is it there" and "how many different", nothing else.', + '- TYPE … = SET OF … and DEFINE make a fixed, named set.', + ], + example: { + pseudo: 'DECLARE Seen : SET OF STRING\nDECLARE Repeats : INTEGER\nRepeats <- 0\nFOR EACH W IN ["cat", "dog", "cat", "emu", "dog", "cat"]\n IF W IN Seen THEN\n Repeats <- Repeats + 1\n ELSE\n ADD(Seen, W)\n ENDIF\nNEXT W\nOUTPUT LENGTH(Seen), " distinct, ", Repeats, " repeats"\nTYPE BaseSet = SET OF CHAR\nDEFINE Purines (\'A\', \'G\') : BaseSet\nOUTPUT \'A\' IN Purines, " ", \'C\' IN Purines', + expect: ['3 distinct, 3 repeats', 'TRUE FALSE'], + watch: 'Seen gains cat, dog, emu and then stops growing. The three later words are repeats; the set does not change when they arrive.', + }, + checks: [ + { question: 'After `ADD(S, 5)` three times on an empty set, `LENGTH(S)` is?', options: ['3', '1', '0'], answer: 1, why: 'A set holds distinct values. Adding 5 again changes nothing.' }, + { question: 'Which question is a set for?', options: ['What was the second value added?', 'Have I seen this value before?', 'What is the largest value?'], answer: 1, why: 'Sets have no order and no positions; membership is their one fast question.' }, + ], + flavors: [ + { id: 'cells', hook: 'How many distinct k-mers does a genome have, out of all the possible ones? That number says how repetitive it is, and it is a set\'s LENGTH.', task: 'Add every 2-mer of "ATATGC" to a set and print how many distinct ones there are, and whether "TA" is one.', solution: { pseudo: 'DECLARE T : STRING\nDECLARE Twos : SET OF STRING\nT <- "ATATGC"\nFOR I <- 0 TO LENGTH(T) - 1 // ~ the last 2-mer starts at LENGTH - 2\n ADD(Twos, SLICE(T, I, I + 2))\nNEXT I\nOUTPUT LENGTH(Twos), " ", "TA" IN Twos', expect: ['4 TRUE'] } }, + { id: 'games', hook: 'Which squares has a player already visited? A set answers in one step whether a square is new, however long the game has run.', task: 'A player lands on squares 3, 7, 3, 12, 7, 20. Print how many distinct squares were visited and how many landings were on an already-visited square.', solution: { pseudo: 'DECLARE Visited : SET OF INTEGER\nDECLARE Again : INTEGER\nAgain <- 0\nFOR EACH Sq IN [3, 7, 3, 12, 7, 20]\n IF Sq IN Visited THEN\n Again <- Again + 1 // ~ already been here\n ELSE\n ADD(Visited, Sq)\n ENDIF\nNEXT Sq\nOUTPUT LENGTH(Visited), " ", Again', expect: ['4 2'] } }, + { id: 'words', hook: 'The vowels are a fixed set. Naming it makes "is this a vowel" read like English and keeps the five letters in one place.', task: 'Define a set Vowels of a, e, i, o, u, then count the vowels in "sequence".', solution: { pseudo: 'TYPE Letters = SET OF CHAR\nDEFINE Vowels (\'a\', \'e\', \'i\', \'o\', \'u\') : Letters\nDECLARE N : INTEGER\nN <- 0\nFOR EACH C IN "sequence"\n IF C IN Vowels THEN // ~ membership in the named set\n N <- N + 1\n ENDIF\nNEXT C\nOUTPUT N', expect: ['4'] } }, + ], + }, + { + id: 'records', title: 'Values that travel together', stage: 'core', minutes: 20, + purpose: 'You will be able to define a TYPE with named fields, build and read records, and use one to return two values from a function.', + teaches: ['record'], requires: ['function', 'list'], + explanation: [ + 'A `TYPE` groups several named fields into one value. `TYPE Read` with `DECLARE Name : STRING` and `DECLARE Bases : STRING` makes a kind of value that holds both; `DECLARE R : Read` makes one; `R.Name <- "read_1"` fills a field; `R.Bases` reads one. A list of records, `LIST OF Read`, is how a table of data travels: one record per row, one field per column, every row the same shape.', + 'Records also solve a problem the functions lesson left open: a `FUNCTION` returns one value, so how do you return two? Return a record. `RANGE_OF` returns an `Extremes` whose `Low` and `High` fields are both filled, and the caller reads `R.Low` and `R.High`. The names say which is which, where a two-element list would leave the reader guessing.', + 'A record is copied whole when assigned or passed, like every other value. Change the copy and the original is untouched; the trace shows two separate records.', + '- TYPE … DECLARE field … ENDTYPE defines a record kind; R.Field reaches a field.', + '- Return two values by returning one record with two named fields.', + '- A LIST OF a record type is a table.', + ], + example: { + pseudo: 'TYPE Extremes\n DECLARE Low : INTEGER\n DECLARE High : INTEGER\nENDTYPE\nFUNCTION RANGE_OF(Xs : LIST OF INTEGER) RETURNS Extremes\n DECLARE E : Extremes\n E.Low <- Xs[0]\n E.High <- Xs[0]\n FOR EACH X IN Xs\n IF X < E.Low THEN\n E.Low <- X\n ENDIF\n IF X > E.High THEN\n E.High <- X\n ENDIF\n NEXT X\n RETURN E\nENDFUNCTION\nDECLARE R : Extremes\nR <- RANGE_OF([7, 2, 9, 4])\nOUTPUT R.Low, " to ", R.High', + expect: ['2 to 9'], + watch: 'E in the function\'s frame has two fields that change independently. One RETURN carries both back, and R outside has the same two fields.', + }, + checks: [ + { question: 'How does a FUNCTION return two values?', options: ['Two RETURN lines', 'A record with two fields', 'It cannot'], answer: 1, why: 'One value comes back; make that value a record whose fields are the two answers.' }, + { question: '`S <- R` copies a record. After `S.Name <- "x"`, what is R.Name?', options: ['"x"', 'Unchanged', 'No value'], answer: 1, why: 'Records copy like every other value. S is a separate record.' }, + ], + flavors: [ + { id: 'cells', hook: 'A read is a name, a sequence, and a quality. Keeping them in one record means they cannot drift apart in three parallel lists.', task: 'Define a Read type with Name, Bases, and Quality. Make a list of two reads and print the name of the one with the higher quality.', solution: { pseudo: 'TYPE Read\n DECLARE Name : STRING\n DECLARE Bases : STRING\n DECLARE Quality : INTEGER\nENDTYPE\nDECLARE A, B : Read\nDECLARE Reads : LIST OF Read\nA.Name <- "r1"\nA.Bases <- "GATT"\nA.Quality <- 28\nB.Name <- "r2"\nB.Bases <- "ACAC"\nB.Quality <- 35 // ~ the better read\nAPPEND(Reads, A)\nAPPEND(Reads, B)\nDECLARE Best : Read\nBest <- Reads[0]\nFOR EACH R IN Reads\n IF R.Quality > Best.Quality THEN\n Best <- R // ~ a better read so far\n ENDIF\nNEXT R\nOUTPUT Best.Name', expect: ['r2'] } }, + { id: 'money', hook: 'A transaction is a date, an amount, and a category. Returning a summary of a list of transactions means two numbers, total in and total out: one record.', task: 'Define a Flow type with Inflow and Outflow fields. Write SUMMARIZE(Amounts) that totals positive amounts into Inflow and negative ones into Outflow (as a positive number). Print both for [500, -120, 300, -80].', solution: { pseudo: 'TYPE Flow\n DECLARE Inflow : INTEGER\n DECLARE Outflow : INTEGER\nENDTYPE\nFUNCTION SUMMARIZE(Amounts : LIST OF INTEGER) RETURNS Flow\n DECLARE F : Flow\n F.Inflow <- 0\n F.Outflow <- 0\n FOR EACH A IN Amounts\n IF A >= 0 THEN\n F.Inflow <- F.Inflow + A\n ELSE\n F.Outflow <- F.Outflow - A // ~ store the outflow as a positive number\n ENDIF\n NEXT A\n RETURN F\nENDFUNCTION\nDECLARE F : Flow\nF <- SUMMARIZE([500, -120, 300, -80])\nOUTPUT F.Inflow, " in, ", F.Outflow, " out"', expect: ['800 in, 200 out'] } }, + { id: 'games', hook: 'A player has a name, a score, and a position. A list of players is the whole game state.', task: 'Define a Player with Name and Score. Make two players, give the second 5 points, and print each player\'s name and score.', solution: { pseudo: 'TYPE Player\n DECLARE Name : STRING\n DECLARE Score : INTEGER\nENDTYPE\nDECLARE P1, P2 : Player\nP1.Name <- "Ada"\nP1.Score <- 0\nP2.Name <- "Lin"\nP2.Score <- 0\nP2.Score <- P2.Score + 5 // ~ the points go to the second player\nFOR EACH P IN [P1, P2]\n OUTPUT P.Name, " ", P.Score\nNEXT P', expect: ['Ada 0', 'Lin 5'] } }, + ], + }, + { + id: 'arrays', title: 'Tables with a fixed shape', stage: 'core', minutes: 20, + purpose: 'You will be able to declare and fill a two-dimensional ARRAY, loop over its rows and columns, and choose between ARRAY and LIST.', + teaches: ['array'], requires: ['nesting', 'list'], + explanation: [ + 'An `ARRAY[3, 3] OF INTEGER` is a grid of nine whole numbers whose shape is fixed when it is declared. `Grid[R, C]` reaches row `R`, column `C`, both counted from 0. `ROWS(Grid)` and `COLUMNS(Grid)` give the two lengths. A one-dimensional `ARRAY[4] OF INTEGER` has four slots at positions 0 to 3; the number in the brackets is a length, not a last position.', + 'Where a `LIST` grows, an `ARRAY` does not: there is no `APPEND`, and every slot exists from the start, empty until stored into. That is the point. A chessboard is 8 by 8, a scoring matrix is 4 by 4, a Sudoku is 9 by 9; the shape is part of the problem, and an `ARRAY` states it.', + 'The cost of the fixed shape is that an `ARRAY` type carries its lengths, so a function over "arrays of any size" cannot be written: `FUNCTION TOTAL(A : ARRAY OF INTEGER)` is refused. Anything a subroutine takes is a `LIST`. The rule: `ARRAY` for a table whose shape is fixed by the problem, `LIST` for everything else, and especially for parameters.', + '- ARRAY[R, C] has a fixed shape; Grid[R, C] reaches a cell; ROWS and COLUMNS give the lengths.', + '- Visit a grid with a loop over rows around a loop over columns.', + '- A subroutine takes a LIST, never an ARRAY.', + ], + example: { + pseudo: 'DECLARE Grid : ARRAY[3, 3] OF INTEGER\nFOR R <- 0 TO ROWS(Grid)\n FOR C <- 0 TO COLUMNS(Grid)\n Grid[R, C] <- R * 3 + C\n NEXT C\nNEXT R\nOUTPUT Grid[1, 2]\nDECLARE Diagonal : INTEGER\nDiagonal <- 0\nFOR K <- 0 TO ROWS(Grid)\n Diagonal <- Diagonal + Grid[K, K]\nNEXT K\nOUTPUT Diagonal', + expect: ['5', '12'], + watch: 'The grid fills row by row: the inner counter C moves fastest. Grid[1, 2] is the second row, third column. The diagonal adds Grid[0,0], Grid[1,1], Grid[2,2].', + }, + checks: [ + { question: '`DECLARE A : ARRAY[4] OF INTEGER`. Which positions exist?', options: ['1 to 4', '0 to 4', '0 to 3'], answer: 2, why: '4 is the length. Positions count from 0.' }, + { question: 'Which should be a LIST?', options: ['A 9 by 9 Sudoku grid', 'The reads a function takes as a parameter', 'An 8 by 8 chessboard'], answer: 1, why: 'A parameter must work for any size, so it is a LIST. The two boards have fixed shapes.' }, + ], + flavors: [ + { id: 'cells', hook: 'A substitution matrix scores every pair of bases. It is 4 by 4 and never changes shape: an ARRAY.', task: 'Fill a 4 by 4 matrix with 1 on the diagonal (match) and -1 elsewhere (mismatch), then print the score for positions (0,0) and (0,3) and the sum of the whole matrix.', solution: { pseudo: 'DECLARE M : ARRAY[4, 4] OF INTEGER\nDECLARE Total : INTEGER\nFOR R <- 0 TO 4\n FOR C <- 0 TO 4\n IF R == C THEN\n M[R, C] <- 1 // ~ a match\n ELSE\n M[R, C] <- -1\n ENDIF\n NEXT C\nNEXT R\nTotal <- 0\nFOR R <- 0 TO ROWS(M)\n FOR C <- 0 TO COLUMNS(M)\n Total <- Total + M[R, C]\n NEXT C\nNEXT R\nOUTPUT M[0, 0], " ", M[0, 3], " ", Total', expect: ['1 -1 -8'] } }, + { id: 'games', hook: 'A tic-tac-toe board is 3 by 3. Checking a row for a win is a loop over its three columns.', task: 'Fill a 3 by 3 board of CHAR with \'.\', place X at (1,0), (1,1), (1,2), and print whether row 1 is three X in a row.', solution: { pseudo: 'DECLARE Board : ARRAY[3, 3] OF CHAR\nFOR R <- 0 TO 3\n FOR C <- 0 TO 3\n Board[R, C] <- \'.\'\n NEXT C\nNEXT R\nBoard[1, 0] <- \'X\'\nBoard[1, 1] <- \'X\'\nBoard[1, 2] <- \'X\'\nDECLARE Win : BOOL\nWin <- TRUE\nFOR C <- 0 TO 3\n IF Board[1, C] != \'X\' THEN\n Win <- FALSE // ~ one non-X breaks the row\n ENDIF\nNEXT C\nOUTPUT Win', expect: ['TRUE'] } }, + { id: 'everyday', hook: 'A weekly timetable is 7 days by 24 hours. The shape never changes, and a cell is an hour of a day.', task: 'Make a 7 by 3 array of hours (morning, afternoon, evening) filled with 0, mark 2 hours of study every evening, and print the total hours.', solution: { pseudo: 'DECLARE Week : ARRAY[7, 3] OF INTEGER\nDECLARE Total : INTEGER\nFOR D <- 0 TO 7\n FOR S <- 0 TO 3\n Week[D, S] <- 0\n NEXT S\n Week[D, 2] <- 2 // ~ two evening hours each day\nNEXT D\nTotal <- 0\nFOR D <- 0 TO ROWS(Week)\n FOR S <- 0 TO COLUMNS(Week)\n Total <- Total + Week[D, S]\n NEXT S\nNEXT D\nOUTPUT Total', expect: ['14'] } }, + ], + }, + { + id: 'what-stops-a-run', title: 'What stops a run, and why that is good', stage: 'core', minutes: 15, + purpose: 'You will be able to name the five things that stop a run, predict which one a line will hit, and explain why stopping beats a wrong answer.', + teaches: ['runtime-stops'], requires: ['list', 'hashmap', 'half-open'], + explanation: [ + 'Some mistakes cannot be caught before the run because they depend on the data: a position that does not exist for this text, a key not in this map, a division by this zero. In many languages these produce a wrong value, a crash a hundred lines later, or nothing visible at all. In pseudo each one stops the run at the line, with a sentence that names the value, the limit, and the usual cause. No wrong answer comes out.', + 'The five you will meet: a position that does not exist (`T[LENGTH(T)]`, the length used as a position); a name with no value yet (reading before storing, or a `FOR` counter after its loop); a missing key in a hashmap; division by zero with `/`, `DIV`, or `MOD`; and a whole number too large to hold exactly (past about nine million billion). Two more are about running rather than data: a loop that never ends is stopped after 20,000 steps, and recursion deeper than 400 calls.', + 'Read the stop as a question. "Position 7 is one past the end" asks whether a length was used where a position belongs. "No value yet" asks where the first store should have been. The example is written to stop on purpose; predict which line, and what the sentence will say, before you run it.', + '- A run stops, with a sentence, instead of continuing with a wrong value.', + '- The usual five: bad position, no value yet, missing key, division by zero, too large.', + '- Each stop names the usual cause. The fix is nearly always one line earlier.', + ], + example: { + pseudo: 'DECLARE T : STRING\nDECLARE L : LIST OF INTEGER\nT <- "GAT"\nL <- [4, 8]\nOUTPUT T[2]\nOUTPUT L[1]\nOUTPUT L[0] DIV 2\nOUTPUT T[LENGTH(T)]\nOUTPUT "this line never runs"', + expect: ['T', '8', '2'], stops: 'index', + watch: 'Three outputs, then the stop at T[LENGTH(T)]: the Problems view names position 3 and says it is one past the end. Change it to T[LENGTH(T) - 1] and the last line runs.', + }, + checks: [ + { question: '`DECLARE N : INTEGER` then `OUTPUT N + 1`. What happens?', options: ['Prints 1', 'Prints 0', 'Stops: N has no value yet'], answer: 2, why: 'DECLARE makes the place but stores nothing. Reading it stops the run.' }, + { question: '`OUTPUT 7 MOD 0`?', options: ['7', '0', 'Stops: division by zero'], answer: 2, why: 'MOD divides, and dividing by zero has no answer.' }, + { question: 'Why stop rather than return a default like 0?', options: ['Stopping is faster', 'A default would be a wrong answer that looks right', 'Defaults are not possible'], answer: 1, why: 'A quiet 0 is indistinguishable from a real 0. A stop with a sentence is a question you can answer.' }, + ], + flavors: [ + { id: 'cells', hook: 'The last codon of a read is where off-by-one errors live. A stop there is better than a codon of two bases silently scored as a match.', task: 'This program is meant to print the three codons of "GATTACAGG". It stops instead. Predict which stop, run to confirm, and fix the loop bound.', starter: 'DECLARE R : STRING\nR <- "GATTACAGG"\nFOR I <- 0 TO LENGTH(R) STEP 3\n OUTPUT SLICE(R, I, I + 3)\nNEXT I', solution: { pseudo: 'DECLARE R : STRING\nR <- "GATTACAGG"\nFOR I <- 0 TO LENGTH(R) - 2 STEP 3\n OUTPUT SLICE(R, I, I + 3)\nNEXT I', expect: ['GAT', 'TAC', 'AGG'] } }, + { id: 'money', hook: 'An average over zero transactions is a division by zero. A program that prints 0 for the average of nothing is lying; one that stops is asking what you want.', task: 'This program averages a list of amounts and stops on the empty list. Make it print "no transactions" for an empty list and the average otherwise.', starter: 'FUNCTION AVERAGE(Xs : LIST OF INTEGER) RETURNS FLOAT\n DECLARE Total : INTEGER\n Total <- 0\n FOR EACH X IN Xs\n Total <- Total + X\n NEXT X\n RETURN Total / LENGTH(Xs)\nENDFUNCTION\nDECLARE Empty : LIST OF INTEGER\nOUTPUT AVERAGE([200, 400])\nOUTPUT AVERAGE(Empty)', solution: { pseudo: 'FUNCTION AVERAGE(Xs : LIST OF INTEGER) RETURNS FLOAT\n DECLARE Total : INTEGER\n Total <- 0\n FOR EACH X IN Xs\n Total <- Total + X\n NEXT X\n RETURN Total / LENGTH(Xs)\nENDFUNCTION\nDECLARE Empty : LIST OF INTEGER\nOUTPUT AVERAGE([200, 400])\nIF LENGTH(Empty) == 0 THEN\n OUTPUT "no transactions"\nELSE\n OUTPUT AVERAGE(Empty)\nENDIF', expect: ['300.0', 'no transactions'] } }, + { id: 'games', hook: 'Looking up a player who never scored is a missing key. The scoreboard should show 0 for them on purpose, not by accident.', task: 'This program stops looking up "Lin", who has no entry. Make it print 0 for a missing player without storing anything.', starter: 'DECLARE Scores : HASHMAP OF STRING : INTEGER\nScores["Ada"] <- 3\nOUTPUT Scores["Ada"]\nOUTPUT Scores["Lin"]', solution: { pseudo: 'DECLARE Scores : HASHMAP OF STRING : INTEGER\nScores["Ada"] <- 3\nOUTPUT Scores["Ada"]\nIF "Lin" IN Scores THEN\n OUTPUT Scores["Lin"]\nELSE\n OUTPUT 0\nENDIF', expect: ['3', '0'] } }, + ], + }, + { + id: 'floats', title: 'Floats are approximate', stage: 'core', minutes: 15, + purpose: 'You will know why 0.1 + 0.2 is not 0.3, how to compare floats, and why money is kept in whole cents.', + teaches: ['float-precision'], requires: ['arithmetic', 'logic', 'for-loop', 'function'], + explanation: [ + 'A `FLOAT` is stored in binary with a fixed number of digits, and most decimal fractions do not fit exactly, just as 1/3 does not fit in a fixed number of decimal digits. So `0.1 + 0.2` is `0.30000000000000004`, and `0.1 + 0.2 == 0.3` is `FALSE`. This is not a bug in the machine; it is what a fixed-width measurement is. The checker warns whenever you compare two floats with `==`, because the answer is nearly always a surprise.', + 'The rule for comparing: ask whether the difference is small, `ABS(A - B) < 0.000001`, with a tolerance that suits the measurement. The rule for money: do not use floats at all. Keep amounts as whole cents in an `INTEGER`, where every addition is exact, and put the decimal point in only when printing, with `DIV 100` and `MOD 100`.', + 'Floats are still the right type for measurements: a GC fraction, a temperature, a probability. The habit to build is noticing which quantities are counts (exact, `INTEGER`) and which are measurements (approximate, `FLOAT`), and never letting a count become a float by accident, which `/` will do.', + '- Most decimals are not exact in binary. 0.1 + 0.2 is not 0.3.', + '- Compare floats with a tolerance: ABS(A - B) < small.', + '- Money is whole cents in an INTEGER. Counts are INTEGER. Measurements are FLOAT.', + ], + example: { + pseudo: 'OUTPUT 0.1 + 0.2\nOUTPUT 1.1 * 3\nOUTPUT ABS((0.1 + 0.2) - 0.3) < 0.000001\nDECLARE Cents : INTEGER\nCents <- 1999 * 3\nOUTPUT Cents DIV 100, ".", Cents MOD 100\nOUTPUT 10 / 4\nOUTPUT 10 DIV 4', + expect: ['0.30000000000000004', '3.3000000000000003', 'TRUE', '59.97', '2.5', '2'], + watch: 'The first two lines. Neither answer is what a calculator shows, and both are correct for what a FLOAT is. The cents line is exact because it never left whole numbers.', + }, + checks: [ + { question: 'Why is `0.1 + 0.2 == 0.3` FALSE?', options: ['A bug in the interpreter', '0.1 and 0.2 are not exact in binary', '== does not work on FLOAT'], answer: 1, why: 'Binary fractions cannot represent most decimals exactly, so the sum is off in the last digit.' }, + { question: 'How should a price of $12.34 be stored?', options: ['12.34 as a FLOAT', '1234 as an INTEGER of cents', '"12.34" as a STRING'], answer: 1, why: 'Whole cents add exactly. The decimal point is for printing.' }, + ], + flavors: [ + { id: 'money', hook: 'Adding $0.10 ten times should give $1.00. In floats it does not, and the bank\'s books will not balance by a fraction of a cent.', task: 'Add 0.1 ten times as a FLOAT and print whether it equals 1.0 with a tolerance. Then do the same in cents and print the exact total.', solution: { pseudo: 'DECLARE F : FLOAT\nDECLARE Cents : INTEGER\nF <- 0.0\nCents <- 0\nFOR I <- 0 TO 10\n F <- F + 0.1\n Cents <- Cents + 10 // ~ ten cents, exactly\nNEXT I\nOUTPUT ABS(F - 1.0) < 0.000001 // ~ close enough, with a tolerance\nOUTPUT Cents', expect: ['TRUE', '100'] } }, + { id: 'cells', hook: 'A GC fraction is a measurement, so it is a FLOAT; the count it came from is exact. Keep both, and compare fractions with a tolerance.', task: 'Compute the GC fraction of "GGCC" and "GCGC" and print whether they are equal, using a tolerance.', solution: { pseudo: 'FUNCTION GC(T : STRING) RETURNS FLOAT\n DECLARE N : INTEGER\n N <- 0\n FOR EACH B IN T\n IF B == \'G\' OR B == \'C\' THEN\n N <- N + 1\n ENDIF\n NEXT B\n RETURN N / LENGTH(T)\nENDFUNCTION\nOUTPUT ABS(GC("GGCC") - GC("GCGC")) < 0.000001 // ~ compare with a tolerance, not ==', expect: ['TRUE'] } }, + { id: 'everyday', hook: 'A recipe scaled by 1.5 three times is not quite scaled by 3.375. For a kitchen it does not matter; knowing why is what matters.', task: 'Scale 200 grams by 1.5 three times in a FLOAT and print the result rounded to a whole gram with ROUND.', solution: { pseudo: 'DECLARE Grams : FLOAT\nGrams <- 200.0\nFOR I <- 0 TO 3\n Grams <- Grams * 1.5\nNEXT I\nOUTPUT ROUND(Grams) // ~ the nearest whole gram', expect: ['675'] } }, + ], + }, +]; diff --git a/src/curriculum/foundations.ts b/src/curriculum/foundations.ts new file mode 100644 index 0000000..01f79e3 --- /dev/null +++ b/src/curriculum/foundations.ts @@ -0,0 +1,311 @@ +import type { Lesson } from './types.js'; + +/** + * Foundations: from "what is a procedure" to reading a run. Nothing here + * assumes the reader has programmed before. Every example and every flavor + * solution runs in `tests/curriculum.test.mjs`. + */ +export const FOUNDATIONS: Lesson[] = [ + { + id: 'what-a-procedure-is', title: 'A procedure is a list of steps', stage: 'foundations', minutes: 10, + purpose: 'You will understand that a computer does exactly what the written steps say, in order, and nothing else.', + teaches: ['procedure'], requires: [], + explanation: [ + 'A procedure is a list of steps that someone else, or something else, can follow without you there. A recipe is a procedure. So is a lab protocol, a set of directions, and the rules of a card game. Programming is writing procedures for a machine that will follow them exactly: it reads the first step, does it, reads the next step, does it, and goes on until there are no steps left.', + 'The machine brings no judgment. If a step is missing, it is not done. If two steps are in the wrong order, they happen in the wrong order. If a step says something slightly different from what you meant, the machine does what the step says. That is not a weakness to work around; it is the whole point. A procedure a machine can follow is a procedure whose meaning does not depend on who reads it.', + 'In Programming+ every procedure is written in `pseudo`, one notation with one meaning per word. The simplest step is `OUTPUT`, which shows a value. Run the example and watch the current line move: one step, one line, top to bottom, no exceptions.', + '- Each line is one step. The machine finishes a step before it reads the next.', + '- Order is meaning. Swap two lines and you have a different procedure.', + '- Text between double quotes is shown exactly as written, without the quotes.', + ], + example: { + pseudo: '// A protocol, as a machine would follow it.\nOUTPUT "Take the tube out of the freezer."\nOUTPUT "Wait until it thaws."\nOUTPUT "Add 2 microlitres of enzyme."\nOUTPUT "Mix gently, then put it on ice."', + expect: ['Take the tube out of the freezer.', 'Wait until it thaws.', 'Add 2 microlitres of enzyme.', 'Mix gently, then put it on ice.'], + watch: 'Press Step. The highlighted line is the step being done now. Nothing happens except what that line says, and the lines run in the order written.', + }, + checks: [ + { question: 'A program has five OUTPUT lines. Which lines run?', options: ['Only the first', 'All five, top to bottom', 'Whichever the machine judges important'], answer: 1, why: 'A machine runs every step in order. It has no notion of important.' }, + { question: 'You swap lines 2 and 3 of a procedure. What happens?', options: ['Nothing; the steps are the same', 'The machine does step 3 before step 2', 'The machine asks which order you meant'], answer: 1, why: 'Order is part of the meaning. The machine follows the order on the page.' }, + ], + flavors: [ + { id: 'everyday', hook: 'You already write procedures: a recipe is one. Writing one for a machine shows you how many steps you usually leave to the reader.', task: 'Write the four steps for making a cup of tea so that each OUTPUT is one step and the order would work in a kitchen.', solution: { pseudo: 'OUTPUT "Boil water."\nOUTPUT "Put a tea bag in the cup." // ~ the step before pouring\nOUTPUT "Pour the water over the bag."\nOUTPUT "Wait three minutes, then take the bag out." // ~ the last step', expect: ['Boil water.', 'Put a tea bag in the cup.', 'Pour the water over the bag.', 'Wait three minutes, then take the bag out.'] } }, + { id: 'cells', hook: 'A PCR cycle is a procedure a thermocycler follows thousands of times without a person present. Each line is one temperature step.', task: 'Write the three steps of one PCR cycle in the order a thermocycler runs them.', solution: { pseudo: 'OUTPUT "Heat to 95 degrees: the two strands separate."\nOUTPUT "Cool to 55 degrees: primers bind." // ~ the middle step\nOUTPUT "Hold at 72 degrees: the polymerase copies."', expect: ['Heat to 95 degrees: the two strands separate.', 'Cool to 55 degrees: primers bind.', 'Hold at 72 degrees: the polymerase copies.'] } }, + { id: 'games', hook: 'A referee follows the rules exactly. Writing a turn as steps is the first thing a game program needs.', task: 'Write one turn of a dice game as three steps: roll, move, and check for the winning square.', solution: { pseudo: 'OUTPUT "Roll the die."\nOUTPUT "Move forward that many squares." // ~ what happens with the roll\nOUTPUT "If you are on square 20, you have won."', expect: ['Roll the die.', 'Move forward that many squares.', 'If you are on square 20, you have won.'] } }, + ], + }, + { + id: 'values-and-types', title: 'Values have types', stage: 'foundations', minutes: 15, + purpose: 'You will be able to name the type of any value and say what that type lets you do with it.', + teaches: ['value', 'type'], requires: ['procedure'], + explanation: [ + 'Everything a program works with is a value: the number `3`, the number `2.5`, the single character `\'A\'`, the text `"GATTACA"`, the truth `TRUE`. Each value has exactly one type, and the type decides what the value can do. You can add two numbers. You can join two texts. You cannot add a text to a number, because there is no one answer to what that means, and pseudo refuses to guess.', + 'There are five basic types. `INTEGER` is a whole number: `3`, `-7`, `0`. `FLOAT` is a number with a fractional part: `2.5`, `0.001`, `3.0`. `CHAR` is exactly one character, in single quotes: `\'A\'`. `STRING` is any amount of text, in double quotes: `"GATTACA"`, `""`. `BOOL` is `TRUE` or `FALSE`.', + 'The types that look most alike are the most important to tell apart. `3` and `3.0` are different types; the first is exact, the second is a measurement. `\'A\'` and `"A"` are different types; one is a character, the other is a text that happens to be one character long. `"3"` is text, not a number: `3 + 4` is `7`, but `"3" & "4"` is `"34"`, because `&` joins text.', + '- A value has one type, and the type never changes.', + '- Numbers add; texts join with `&`. A `/` between two whole numbers gives a `FLOAT`, always.', + '- When pseudo refuses to mix two types, that is a question for you, not an obstacle: which did you mean?', + ], + example: { + pseudo: 'OUTPUT 3\nOUTPUT 2.5\nOUTPUT \'A\'\nOUTPUT "GATTACA"\nOUTPUT TRUE\nOUTPUT 3 + 4\nOUTPUT "3" & "4"\nOUTPUT 6 / 2', + expect: ['3', '2.5', 'A', 'GATTACA', 'TRUE', '7', '34', '3.0'], + watch: 'The last three lines. 3 + 4 adds numbers; "3" & "4" joins texts; 6 / 2 is 3.0, a FLOAT, because / always measures rather than counts.', + }, + checks: [ + { question: 'What type is `"7"`?', options: ['INTEGER', 'STRING', 'CHAR'], answer: 1, why: 'Double quotes make a STRING, whatever is inside them.' }, + { question: 'What is `6 / 3`?', options: ['2', '2.0', 'An error'], answer: 1, why: '/ always produces a FLOAT. Whole-number division is DIV.' }, + { question: 'Which pair are the same type?', options: ['\'A\' and "A"', '3 and 3.0', 'TRUE and FALSE'], answer: 2, why: 'TRUE and FALSE are both BOOL. The other pairs mix CHAR with STRING and INTEGER with FLOAT.' }, + ], + flavors: [ + { id: 'money', hook: 'Money programs break when a price is a FLOAT. $19.99 is 1999 cents, a whole number that adds exactly.', task: 'Show a price three ways: as a FLOAT in dollars, as an INTEGER in cents, and as text with a currency sign.', solution: { pseudo: 'OUTPUT 19.99\nOUTPUT 1999 // ~ the same price as whole cents\nOUTPUT "$" & "19.99" // ~ join a sign to the text of the price', expect: ['19.99', '1999', '$19.99'] } }, + { id: 'cells', hook: 'A single base is a CHAR; a read is a STRING. Keeping them apart is what lets you ask "is this base a G" without ambiguity.', task: 'Output one base as a CHAR, a read as a STRING, the read\'s length as an INTEGER, and TRUE as a BOOL to mark the read as passing.', solution: { pseudo: 'OUTPUT \'G\'\nOUTPUT "GATTACA" // ~ the read, as text\nOUTPUT LENGTH("GATTACA")\nOUTPUT TRUE // ~ a BOOL', expect: ['G', 'GATTACA', '7', 'TRUE'] } }, + { id: 'words', hook: 'Joining text and adding numbers look alike on the page and do opposite things. Writers of text programs meet this on day one.', task: 'Output the two results of putting 12 and 34 together: as numbers and as texts.', solution: { pseudo: 'OUTPUT 12 + 34\nOUTPUT "12" & "34" // ~ the texts joined', expect: ['46', '1234'] } }, + ], + }, + { + id: 'names-and-assignment', title: 'Names hold one value at a time', stage: 'foundations', minutes: 20, + purpose: 'You will be able to say what every variable holds after each line of a short program.', + teaches: ['variable', 'assignment'], requires: ['value', 'type'], + explanation: [ + 'A variable is a named place that holds one value. `DECLARE Score : INTEGER` makes a place called `Score` that can hold a whole number. It holds nothing yet; reading it now would stop the run with "no value yet". `Score <- 10` stores 10 there. The arrow is the whole story of programming in one symbol: the right side is worked out first, and the result travels into the name on the left.', + 'The one rule people get wrong: a name holds one value at a time, and storing a new value destroys the old one. After `Score <- 3`, the 15 that was there is gone, with no record of it. The trace keeps a history for you ("was 15; changed at step 4"), but the program does not.', + '`Score <- Score + 5` is not an equation. It reads the current value of `Score`, adds 5, and stores the result back in `Score`. Read it aloud as "Score becomes Score plus five". The same name on both sides refers to two moments: the old value on the right, the new value on the left.', + '- `DECLARE` makes the place and fixes its type. The type of the place never changes.', + '- `<-` works out the right side, then stores. The old value is gone.', + '- `B <- A` copies the value. Changing `A` afterwards does not change `B`.', + ], + example: { + pseudo: 'DECLARE Score : INTEGER\nScore <- 10\nOUTPUT Score\nScore <- Score + 5\nOUTPUT Score\nDECLARE Saved : INTEGER\nSaved <- Score\nScore <- 3\nOUTPUT Score, " ", Saved', + expect: ['10', '15', '3 15'], + watch: 'The variables panel. At Score <- Score + 5 the right side reads 10 before the left side stores 15. After Score <- 3, Saved still holds 15: it was a copy.', + }, + checks: [ + { question: 'After `X <- 4` then `X <- X * 2` then `X <- 1`, what does X hold?', options: ['8', '1', '4, 8, and 1'], answer: 1, why: 'A name holds one value. The last store wins and the others are gone.' }, + { question: '`A <- 5`, `B <- A`, `A <- 6`. What is B?', options: ['6', '5', 'No value yet'], answer: 1, why: 'B <- A copied the 5. Later changes to A do not reach B.' }, + { question: 'What does `DECLARE N : INTEGER` store in N?', options: ['0', 'Nothing yet', 'INTEGER'], answer: 1, why: 'DECLARE makes the place. Reading N before a store stops the run.' }, + ], + flavors: [ + { id: 'money', hook: 'A bank balance is a variable: one number, replaced by every deposit and withdrawal. The history lives in the statement, not in the balance.', task: 'Start a balance at 500 cents, deposit 250, withdraw 100, and show the balance after each change.', solution: { pseudo: 'DECLARE Balance : INTEGER\nBalance <- 500\nBalance <- Balance + 250 // ~ the deposit\nOUTPUT Balance\nBalance <- Balance - 100 // ~ the withdrawal\nOUTPUT Balance', expect: ['750', '650'] } }, + { id: 'games', hook: 'A score is the simplest variable there is. Every point scored is `Score <- Score + 1`, and a reset is a store of 0.', task: 'Give a player 3 points, then 4 more, show the score, then reset it to 0 and show it again.', solution: { pseudo: 'DECLARE Score : INTEGER\nScore <- 0\nScore <- Score + 3\nScore <- Score + 4 // ~ four more points\nOUTPUT Score\nScore <- 0 // ~ the reset\nOUTPUT Score', expect: ['7', '0'] } }, + { id: 'cells', hook: 'Reading a genome is a running count: how many bases seen so far. The count is one variable that only ever goes up by one.', task: 'Count four bases one at a time with a variable called Seen, showing it after the second and the fourth.', solution: { pseudo: 'DECLARE Seen : INTEGER\nSeen <- 0\nSeen <- Seen + 1\nSeen <- Seen + 1\nOUTPUT Seen\nSeen <- Seen + 1 // ~ the third base\nSeen <- Seen + 1 // ~ the fourth base\nOUTPUT Seen', expect: ['2', '4'] } }, + ], + }, + { + id: 'arithmetic', title: 'Arithmetic, and the two kinds of division', stage: 'foundations', minutes: 15, + purpose: 'You will know what every arithmetic symbol does, which ones give whole numbers, and when to use parentheses.', + teaches: ['arithmetic'], requires: ['assignment'], + explanation: [ + '`+`, `-`, and `*` do what they did at school. `^` is a power: `2 ^ 10` is 1024. Then there are three divisions, and telling them apart is half of early programming.', + '`/` is measuring division and always gives a `FLOAT`: `7 / 2` is `3.5`, and `6 / 2` is `3.0`, not `3`. `DIV` is whole-number division, discarding the remainder: `7 DIV 2` is `3`. `MOD` is the remainder itself: `7 MOD 2` is `1`. `DIV` and `MOD` together split a count into groups and leftovers: 17 eggs are `17 DIV 12` boxes and `17 MOD 12` loose. `MOD` is also the way to ask "is it even" (`N MOD 2 == 0`) and "where does it wrap around" (the twelfth hour after 9 is `(9 + 12) MOD 12`).', + 'Precedence is the order in which the symbols are worked out. `*` and `/` go before `+` and `-`, so `1 + 2 * 3` is 7. You do not need to memorise the rest: put parentheses wherever a reader might have to look it up. Parentheses cost nothing and remove a whole class of mistakes.', + '- `/` gives a FLOAT, always. Whole-number division is `DIV`; the remainder is `MOD`.', + '- Dividing by zero stops the run. The program does not continue with a wrong number.', + '- Whole numbers are exact up to about nine million billion. Past that, the run stops rather than rounding.', + ], + example: { + pseudo: 'OUTPUT 7 + 2, " ", 7 - 2, " ", 7 * 2\nOUTPUT 7 / 2\nOUTPUT 7 DIV 2, " ", 7 MOD 2\nOUTPUT 6 / 2\nOUTPUT 2 ^ 10\nOUTPUT 1 + 2 * 3, " ", (1 + 2) * 3\nOUTPUT 17 DIV 12, " boxes and ", 17 MOD 12, " loose"', + expect: ['9 5 14', '3.5', '3 1', '3.0', '1024', '7 9', '1 boxes and 5 loose'], + watch: 'Line 4: 6 / 2 prints 3.0. The slash measured; it did not count. Line 6 shows the parentheses changing the answer.', + }, + checks: [ + { question: 'What is `9 DIV 4` and `9 MOD 4`?', options: ['2 and 1', '2.25 and 0', '1 and 2'], answer: 0, why: '9 is two groups of 4 with 1 left over.' }, + { question: 'Which asks whether N is even?', options: ['N / 2 == 0', 'N MOD 2 == 0', 'N DIV 2 == 0'], answer: 1, why: 'Even means a remainder of 0 when divided by 2.' }, + { question: 'What is `2 + 3 * 4`?', options: ['20', '14', '24'], answer: 1, why: '* is worked out before +. Write (2 + 3) * 4 if you mean 20.' }, + ], + flavors: [ + { id: 'money', hook: 'Splitting a bill is DIV and MOD: how much each person pays, and how many cents are left over for someone to cover.', task: 'Split 10,000 cents among 3 people. Show each share and the leftover cents.', solution: { pseudo: 'DECLARE Bill, People : INTEGER\nBill <- 10000\nPeople <- 3\nOUTPUT Bill DIV People, " each" // ~ the whole-cent share\nOUTPUT Bill MOD People, " left over" // ~ the remainder', expect: ['3333 each', '1 left over'] } }, + { id: 'cells', hook: 'Codons are three bases long. How many whole codons a read holds, and how many trailing bases do not form one, is DIV and MOD.', task: 'A read is 20 bases long. Show how many whole codons it holds and how many bases are left over.', solution: { pseudo: 'DECLARE Bases : INTEGER\nBases <- 20\nOUTPUT Bases DIV 3, " codons" // ~ whole codons\nOUTPUT Bases MOD 3, " bases left over" // ~ bases that do not make a codon', expect: ['6 codons', '2 bases left over'] } }, + { id: 'everyday', hook: 'Clock arithmetic wraps around. Fourteen hours after 9 o\'clock is not 23 on a 12-hour clock.', task: 'It is 9 on a 12-hour clock. Show what the clock reads 14 hours later, and how many minutes 150 seconds is with the seconds left over.', solution: { pseudo: 'OUTPUT (9 + 14) MOD 12 // ~ hours wrap at 12\nOUTPUT 150 DIV 60, " minutes ", 150 MOD 60, " seconds"', expect: ['11', '2 minutes 30 seconds'] } }, + ], + }, + { + id: 'output-and-input', title: 'Talking to the outside', stage: 'foundations', minutes: 10, + purpose: 'You will be able to show several values on one line and take a value in from outside the program.', + teaches: ['io'], requires: ['assignment', 'variable'], + explanation: [ + '`OUTPUT` shows values. Several values on one line, separated by commas, print one after another with nothing between them, so you put the spaces and punctuation in yourself: `OUTPUT "Found ", Count, " matches."`. Text prints without its quotes; a number prints as digits; `TRUE` prints as `TRUE`.', + '`INPUT Name` waits for a value from outside the program and stores it in `Name`. The variable must already be declared, and the text that arrives must fit the type: typing `abc` into an `INTEGER` stops the run. In the playground the inputs are typed in a box beside the Run button, one per line.', + 'A program that takes input is a procedure that works for many cases at once. The same lines greet Ada and Grace; only the input changed. That is the first step from a calculation to a program.', + '- Commas in OUTPUT separate what to show; they do not print.', + '- INPUT fills one declared variable. Its type decides what text is acceptable.', + ], + example: { + pseudo: 'DECLARE Name : STRING\nDECLARE Age : INTEGER\nINPUT Name\nINPUT Age\nOUTPUT "Hello, ", Name, "."\nOUTPUT "Next year you will be ", Age + 1, "."', + expect: ['Hello, Ada.', 'Next year you will be 37.'], input: ['Ada', '36'], + watch: 'The inputs box holds two lines, Ada and 36. Each INPUT takes the next line. Change 36 to 40 and run again: the procedure did not change, the case did.', + }, + checks: [ + { question: 'What does `OUTPUT "a", "b", 3` print?', options: ['a b 3', 'ab3', 'a, b, 3'], answer: 1, why: 'Values print one after another. Spaces must be written as text.' }, + { question: 'A program does `INPUT N` with N an INTEGER and the input is `four`. What happens?', options: ['N becomes 4', 'N becomes "four"', 'The run stops'], answer: 2, why: 'The text must fit the type. A run that cannot continue correctly stops with a sentence.' }, + ], + flavors: [ + { id: 'money', hook: 'A tip calculator is three lines once INPUT exists: take the bill, work out the tip, show both.', task: 'Read a bill in cents and show the 15% tip (whole cents, rounded down) and the total.', solution: { pseudo: 'DECLARE Bill, Tip : INTEGER\nINPUT Bill\nTip <- Bill * 15 DIV 100 // ~ 15 percent, in whole cents\nOUTPUT "Tip: ", Tip\nOUTPUT "Total: ", Bill + Tip // ~ bill plus tip', expect: ['Tip: 600', 'Total: 4600'], input: ['4000'] } }, + { id: 'cells', hook: 'A read\'s length and its name arrive from a file. Printing them in a sentence is the start of every report.', task: 'Read a sequence name and a sequence, then report the name and its length in one sentence.', solution: { pseudo: 'DECLARE Name, Seq : STRING\nINPUT Name\nINPUT Seq\nOUTPUT Name, " has ", LENGTH(Seq), " bases." // ~ the report line', expect: ['read_7 has 7 bases.'], input: ['read_7', 'GATTACA'] } }, + { id: 'games', hook: 'A two-player score display reads two numbers and prints the match state. Every scoreboard is this program.', task: 'Read two scores and print them as "Home 3 - 1 Away".', solution: { pseudo: 'DECLARE Home, Away : INTEGER\nINPUT Home\nINPUT Away\nOUTPUT "Home ", Home, " - ", Away, " Away" // ~ one line with both scores', expect: ['Home 3 - 1 Away'], input: ['3', '1'] } }, + ], + }, + { + id: 'text-and-positions', title: 'Positions count from zero', stage: 'foundations', minutes: 20, + purpose: 'You will be able to point at any character of a text by its position, and know which positions exist.', + teaches: ['indexing'], requires: ['assignment', 'arithmetic'], + explanation: [ + 'A `STRING` is a row of characters, and each has a position. Positions count from 0: in `"GATTACA"` the `G` is at position 0 and the last `A` is at position 6. `T[I]` is the character at position `I`; `LENGTH(T)` is how many characters there are, 7 here. So the positions that exist are 0 up to `LENGTH(T) - 1`, and `T[LENGTH(T)]` does not exist. Asking for it stops the run: "position 7 is one past the end".', + 'Why zero? Because a position is a count of how many characters come before it. Before the `G` there are none. This makes the arithmetic clean: the piece of length 3 starting at position `P` occupies positions `P`, `P + 1`, `P + 2`, which is the range `[P, P + 3)`: from `P` up to but not including `P + 3`. `SLICE(T, A, B)` gives exactly the characters at positions `[A, B)`, and its length is `B - A` with no adjustment.', + 'The sequence ruler in the trace shows every position under its character. Use it until the zero-based habit is yours. Nearly every "off by one" mistake in programming is a length used where a position belongs, or the other way round.', + '- Positions are 0 to LENGTH - 1. The last character is at `LENGTH(T) - 1`.', + '- `SLICE(T, A, B)` takes positions `[A, B)`: A is included, B is not, length is B - A.', + '- A position that does not exist stops the run. No character is invented.', + ], + example: { + pseudo: 'DECLARE T : STRING\nT <- "GATTACA"\nOUTPUT LENGTH(T)\nOUTPUT T[0]\nOUTPUT T[6]\nOUTPUT T[LENGTH(T) - 1]\nOUTPUT SLICE(T, 0, 3)\nOUTPUT SLICE(T, 3, 7)', + expect: ['7', 'G', 'A', 'A', 'GAT', 'TACA'], + watch: 'The ruler under T. T[6] and T[LENGTH(T) - 1] are the same character. SLICE(T, 0, 3) and SLICE(T, 3, 7) meet at position 3 without overlapping: the two halves of a half-open split.', + }, + checks: [ + { question: 'In `"CAT"`, what is `T[3]`?', options: ['\'T\'', 'The run stops', '""'], answer: 1, why: 'Positions are 0, 1, 2. Position 3 is one past the end.' }, + { question: 'How long is `SLICE(T, 2, 5)`?', options: ['4', '3', 'It depends on T'], answer: 1, why: 'A half-open range [2, 5) holds positions 2, 3, 4: that is 5 - 2 = 3 characters.' }, + { question: 'Which is always the last character of T?', options: ['T[LENGTH(T)]', 'T[LENGTH(T) - 1]', 'T[-1]'], answer: 1, why: 'LENGTH counts characters; the last position is one less.' }, + ], + flavors: [ + { id: 'cells', hook: 'A codon is positions [P, P + 3) of a read. Getting the first codon right, and the last one, is the whole skill.', task: 'For the read GATTACAGG, show the first codon, the second, and the last complete one.', solution: { pseudo: 'DECLARE R : STRING\nR <- "GATTACAGG"\nOUTPUT SLICE(R, 0, 3)\nOUTPUT SLICE(R, 3, 6) // ~ the second codon starts at 3\nOUTPUT SLICE(R, 6, 9) // ~ the third codon ends at the length', expect: ['GAT', 'TAC', 'AGG'] } }, + { id: 'words', hook: 'Initials, file extensions, and area codes are all slices at fixed positions.', task: 'From the text "alignment.fasta", show the first letter, the three letters before the dot, and the extension after it (the dot is at position 9).', solution: { pseudo: 'DECLARE F : STRING\nF <- "alignment.fasta"\nOUTPUT F[0]\nOUTPUT SLICE(F, 6, 9) // ~ three characters ending just before the dot\nOUTPUT SLICE(F, 10, LENGTH(F)) // ~ from after the dot to the end', expect: ['a', 'ent', 'fasta'] } }, + { id: 'games', hook: 'A board row is a text, and a square is a position. Reading the wrong square is a bug a referee cannot have.', task: 'The row is "X.O.X". Show the piece on squares 0, 2, and the last square.', solution: { pseudo: 'DECLARE Row : STRING\nRow <- "X.O.X"\nOUTPUT Row[0]\nOUTPUT Row[2] // ~ the middle square\nOUTPUT Row[LENGTH(Row) - 1] // ~ the last square, by its position', expect: ['X', 'O', 'X'] } }, + ], + }, + { + id: 'decisions', title: 'Choosing a path', stage: 'foundations', minutes: 20, + purpose: 'You will be able to make a program do one thing or another depending on a condition, and say which path a given input takes.', + teaches: ['boolean', 'selection'], requires: ['assignment', 'arithmetic', 'io'], + explanation: [ + 'A comparison is a value. `Q >= 30` is `TRUE` or `FALSE`, of type `BOOL`, and you can print it, store it, or hand it to an `IF`. The six comparisons are `==`, `!=`, `<`, `<=`, `>`, `>=`. Note the double `==` for "is equal to"; a single `=` is reserved for naming constants, so writing it in a condition is caught as a mistake.', + '`IF` chooses a path. `IF Q >= 30 THEN` runs the lines under it when the condition is `TRUE`, and skips them otherwise. `ELSEIF` (one word) adds another condition that is tested only if the ones above were `FALSE`; `ELSE` catches everything left. One `ENDIF` closes the whole thing. Exactly one body runs: the first whose condition is `TRUE`, or the `ELSE`.', + 'Read the trace to see that "first TRUE wins" is literal. With `Q` at 25, `Q >= 30` is `FALSE`, so the `ELSEIF Q >= 20` is tested and found `TRUE`, and the `ELSE` is never looked at. Change `Q` to 35: now the first condition is `TRUE` and the `ELSEIF` is not even tested, though it would also be `TRUE`.', + '- A comparison is a BOOL value. `==` asks; `<-` stores.', + '- Of an IF, ELSEIF, ELSE chain, exactly one body runs: the first TRUE condition.', + '- Every IF ends with ENDIF, however many branches it has.', + ], + example: { + pseudo: 'DECLARE Q : INTEGER\nQ <- 25\nOUTPUT Q >= 30\nIF Q >= 30 THEN\n OUTPUT "keep"\nELSEIF Q >= 20 THEN\n OUTPUT "check"\nELSE\n OUTPUT "discard"\nENDIF\nOUTPUT "done"', + expect: ['FALSE', 'check', 'done'], + watch: 'The current line jumps from IF to ELSEIF (first condition FALSE), into "check", and then straight past ELSE to the line after ENDIF. Change Q to 35 and 12 and watch the path change.', + }, + checks: [ + { question: 'With `X <- 7`, what does `OUTPUT X > 5` print?', options: ['7', 'TRUE', 'X > 5'], answer: 1, why: 'A comparison is a BOOL value, and OUTPUT prints it.' }, + { question: '`IF A THEN … ELSEIF B THEN … ELSE … ENDIF`, with A and B both TRUE. Which bodies run?', options: ['The first only', 'The first and the second', 'All three'], answer: 0, why: 'The first TRUE condition wins and the rest are not tested.' }, + { question: 'What is wrong with `IF X = 3 THEN`?', options: ['Nothing', '= should be ==', 'THEN is missing'], answer: 1, why: 'Equality is asked with ==. A bare = is refused before the run.' }, + ], + flavors: [ + { id: 'cells', hook: 'A base-call below a quality threshold is discarded; a sequencing pipeline makes this decision billions of times.', task: 'Read a quality score and print "discard" below 20, "marginal" below 30, and "keep" otherwise.', solution: { pseudo: 'DECLARE Q : INTEGER\nINPUT Q\nIF Q < 20 THEN\n OUTPUT "discard"\nELSEIF Q < 30 THEN // ~ the middle band\n OUTPUT "marginal"\nELSE\n OUTPUT "keep"\nENDIF', expect: ['marginal'], input: ['27'] } }, + { id: 'money', hook: 'Free shipping over a threshold is an IF. So is every discount, tax band, and overdraft fee.', task: 'Read an order total in cents. Shipping is 0 at 5000 or more, otherwise 499. Print the shipping and the final total.', solution: { pseudo: 'DECLARE Order, Shipping : INTEGER\nINPUT Order\nIF Order >= 5000 THEN\n Shipping <- 0\nELSE\n Shipping <- 499 // ~ the charge below the threshold\nENDIF\nOUTPUT "Shipping: ", Shipping\nOUTPUT "Total: ", Order + Shipping', expect: ['Shipping: 499', 'Total: 4998'], input: ['4499'] } }, + { id: 'games', hook: 'Rock, paper, scissors is nothing but decisions. A referee that gets one branch wrong is a referee nobody plays with.', task: 'Read two moves as single letters (R, P, S). Print "draw" when equal, "left wins" when the first beats the second, otherwise "right wins".', solution: { pseudo: 'DECLARE L, R : CHAR\nINPUT L\nINPUT R\nIF L == R THEN\n OUTPUT "draw"\nELSEIF L == \'R\' THEN\n IF R == \'S\' THEN\n OUTPUT "left wins"\n ELSE\n OUTPUT "right wins"\n ENDIF\nELSEIF L == \'P\' THEN\n IF R == \'R\' THEN // ~ paper beats rock\n OUTPUT "left wins"\n ELSE\n OUTPUT "right wins"\n ENDIF\nELSE\n IF R == \'P\' THEN // ~ scissors beat paper\n OUTPUT "left wins"\n ELSE\n OUTPUT "right wins"\n ENDIF\nENDIF', expect: ['left wins'], input: ['P', 'R'] } }, + ], + }, + { + id: 'combining-conditions', title: 'AND, OR, NOT', stage: 'foundations', minutes: 15, + purpose: 'You will be able to build one condition from several and know why the guard goes first.', + teaches: ['logic'], requires: ['boolean', 'indexing'], + explanation: [ + '`A AND B` is `TRUE` only when both are. `A OR B` is `TRUE` when either is, or both. `NOT A` flips it. These make one condition out of several: "a strong base" is `B == \'G\' OR B == \'C\'`; "in range" is `0 <= I AND I < N`, which pseudo also lets you write as `0 <= I < N`.', + 'The rule that matters for correctness: `AND` stops as soon as its left side is `FALSE`, and `OR` stops as soon as its left side is `TRUE`. The right side is never worked out. So `I < LENGTH(T) AND T[I] == \'A\'` is safe even when `I` is past the end: the left side is `FALSE`, and `T[I]`, which would stop the run, is never read. Written the other way round, `T[I] == \'A\' AND I < LENGTH(T)` stops the run first. The guard goes first.', + 'Two readings trip people. `NOT B == \'G\'` means `NOT (B == \'G\')`: `NOT` binds looser than a comparison. And "B is not G or C" in English means `B != \'G\' AND B != \'C\'`, not `B != \'G\' OR B != \'C\'` (which is always `TRUE`). When in doubt, parenthesise and read it back aloud.', + '- AND needs both; OR needs either; NOT flips.', + '- The right side of AND is skipped once the left is FALSE: put the check that keeps the rest safe first.', + '- "Not G or C" is two ANDs of not-equal, or `NOT (B == \'G\' OR B == \'C\')`.', + ], + example: { + pseudo: 'DECLARE T : STRING\nDECLARE I : INTEGER\nT <- "GAT"\nI <- 3\nIF I < LENGTH(T) AND T[I] == \'A\' THEN\n OUTPUT "A at ", I\nELSE\n OUTPUT "no A at ", I\nENDIF\nDECLARE B : CHAR\nB <- \'C\'\nOUTPUT B == \'G\' OR B == \'C\'\nOUTPUT NOT B == \'G\'\nOUTPUT NOT (B == \'G\' OR B == \'C\')', + expect: ['no A at 3', 'TRUE', 'TRUE', 'FALSE'], + watch: 'Line 5 with I at 3: T has no position 3, yet the run does not stop. The left side of AND was FALSE and T[I] was never read. Swap the two sides and run again.', + }, + checks: [ + { question: 'Which is TRUE for every B?', options: ['B != \'G\' OR B != \'C\'', 'B != \'G\' AND B != \'C\'', 'NOT (B == \'G\')'], answer: 0, why: 'No character is both G and C, so at least one of the two != tests is TRUE for any B. That is the "not G or C" trap.' }, + { question: '`I < LENGTH(T) AND T[I] == \'A\'` with I past the end: what happens?', options: ['The run stops at T[I]', 'The condition is FALSE and the run goes on', 'The condition is TRUE'], answer: 1, why: 'AND stops at a FALSE left side and never reads T[I].' }, + ], + flavors: [ + { id: 'cells', hook: 'Purines are A or G; a GC-content filter wants G or C; a valid base is one of four. All of these are ORs.', task: 'For the base in B, print whether it is a purine (A or G), and whether it is a valid DNA base at all (one of A, C, G, T).', solution: { pseudo: 'DECLARE B : CHAR\nB <- \'N\'\nOUTPUT B == \'A\' OR B == \'G\'\nOUTPUT B == \'A\' OR B == \'C\' OR B == \'G\' OR B == \'T\' // ~ one of the four bases', expect: ['FALSE', 'FALSE'] } }, + { id: 'money', hook: 'A card payment goes through when the card is valid AND the balance covers it AND the shop is not blocked. One FALSE anywhere declines it.', task: 'Decide whether a 3500-cent payment is approved: the card must be active, the balance must cover it, and the amount must be under the 5000 limit.', solution: { pseudo: 'DECLARE Active : BOOL\nDECLARE Balance, Amount, Limit : INTEGER\nActive <- TRUE\nBalance <- 4000\nAmount <- 3500\nLimit <- 5000\nIF Active AND Balance >= Amount AND Amount < Limit THEN // ~ all three must hold\n OUTPUT "approved"\nELSE\n OUTPUT "declined"\nENDIF', expect: ['approved'] } }, + { id: 'games', hook: 'A move is legal when the square exists AND it is empty. Checking emptiness first would read a square that is not there.', task: 'The row is "X.O". A move to square S is legal when 0 <= S < 3 and the square is \'.\'. Print the legality for S = 1 and S = 5 with the guard first.', solution: { pseudo: 'DECLARE Row : STRING\nDECLARE S : INTEGER\nRow <- "X.O"\nS <- 1\nOUTPUT 0 <= S AND S < LENGTH(Row) AND Row[S] == \'.\'\nS <- 5\nOUTPUT 0 <= S AND S < LENGTH(Row) AND Row[S] == \'.\' // ~ the guard keeps Row[5] from being read', expect: ['TRUE', 'FALSE'] } }, + ], + }, + { + id: 'counted-loops', title: 'Counting with FOR', stage: 'foundations', minutes: 25, + purpose: 'You will be able to repeat a step a known number of times and say exactly which values the counter takes.', + teaches: ['for-loop', 'half-open'], requires: ['indexing', 'arithmetic', 'io'], + explanation: [ + 'A `FOR` loop repeats its body once for each value of a counter. `FOR I <- 0 TO 7` runs the body with `I` at 0, then 1, and so on, and stops when `I` reaches 7, without running the body for 7. The counter takes the values of the half-open range `[0, 7)`: seven passes, positions 0 to 6, exactly the positions of a seven-character text. `NEXT I` closes the loop and names the counter so the reader knows which loop ended.', + 'This is the single most useful rule in the notation, so it is worth saying plainly: `TO` is "up to but not including". `FOR I <- 0 TO LENGTH(T)` visits every position of `T` and never asks for the one past the end. The number of passes is the stop minus the start. Two loops `0 TO K` and `K TO N` together cover `0 TO N` with no gap and no overlap.', + 'Some rules about the counter. It needs no `DECLARE`; the loop makes it. You may read it but not assign to it inside the loop. After `NEXT I` it has no value: the loop is over and the counter went with it. `STEP` changes the increment: `FOR J <- 10 TO 0 STEP -2` counts 10, 8, 6, 4, 2, stopping before 0, since with a negative step the loop runs while the counter is strictly greater than the stop.', + '- `FOR I <- A TO B` runs with I = A, A+1, …, B-1. B is never reached. The number of passes is B - A.', + '- `FOR I <- 0 TO LENGTH(T)` is the loop over every position of T.', + '- The counter is made by the loop and gone after NEXT.', + ], + example: { + pseudo: 'DECLARE T : STRING\nT <- "GATTACA"\nFOR I <- 0 TO LENGTH(T)\n OUTPUT I, ":", T[I]\nNEXT I\nFOR J <- 10 TO 0 STEP -3\n OUTPUT J\nNEXT J', + expect: ['0:G', '1:A', '2:T', '3:T', '4:A', '5:C', '6:A', '10', '7', '4', '1'], + watch: 'I in the variables panel: 0, 1, …, 6, and then the loop ends without a pass at 7. Use Shift+→ to jump a whole pass at a time. After NEXT I, the panel shows I has no value.', + }, + checks: [ + { question: 'How many times does the body of `FOR I <- 3 TO 8` run?', options: ['6', '5', '8'], answer: 1, why: 'I takes 3, 4, 5, 6, 7: that is 8 - 3 = 5 passes.' }, + { question: 'Which visits every position of T exactly once?', options: ['FOR I <- 0 TO LENGTH(T)', 'FOR I <- 1 TO LENGTH(T)', 'FOR I <- 0 TO LENGTH(T) - 1'], answer: 0, why: 'Positions are [0, LENGTH). The second skips position 0; the third skips the last position.' }, + { question: 'What does `OUTPUT I` after `NEXT I` do?', options: ['Prints the last value', 'Prints the stop value', 'Stops the run: I has no value'], answer: 2, why: 'The counter belongs to the loop and is gone when the loop ends.' }, + ], + flavors: [ + { id: 'cells', hook: 'Every per-base operation in genomics, from quality trimming to GC content, is a FOR over the positions of a read.', task: 'Print each base of "ACGT" together with its position, then print the read backwards using a negative STEP.', solution: { pseudo: 'DECLARE R : STRING\nR <- "ACGT"\nFOR I <- 0 TO LENGTH(R)\n OUTPUT I, " ", R[I]\nNEXT I\nFOR J <- LENGTH(R) - 1 TO -1 STEP -1 // ~ from the last position down to 0\n OUTPUT R[J]\nNEXT J', expect: ['0 A', '1 C', '2 G', '3 T', 'T', 'G', 'C', 'A'] } }, + { id: 'money', hook: 'A repayment schedule is a FOR over months. Each pass is one payment; the number of passes is the term.', task: 'A 6-month plan pays 250 cents a month on a 1500-cent debt. Print the remaining debt after each payment.', solution: { pseudo: 'DECLARE Debt : INTEGER\nDebt <- 1500\nFOR Month <- 1 TO 7 // ~ months 1 to 6: the stop is one past the last month\n Debt <- Debt - 250\n OUTPUT "After month ", Month, ": ", Debt\nNEXT Month', expect: ['After month 1: 1250', 'After month 2: 1000', 'After month 3: 750', 'After month 4: 500', 'After month 5: 250', 'After month 6: 0'] } }, + { id: 'everyday', hook: 'A countdown is a FOR with a negative step, and the launch is what happens after NEXT.', task: 'Count down from 5 to 1, then print "Go".', solution: { pseudo: 'FOR N <- 5 TO 0 STEP -1 // ~ stops before 0\n OUTPUT N\nNEXT N\nOUTPUT "Go"', expect: ['5', '4', '3', '2', '1', 'Go'] } }, + ], + }, + { + id: 'accumulators', title: 'Keeping a running total', stage: 'foundations', minutes: 20, + purpose: 'You will be able to count or sum over a collection with one variable, set before the loop and updated inside it.', + teaches: ['accumulator'], requires: ['for-loop', 'selection'], + explanation: [ + 'Most loops exist to build up one answer. The pattern has three parts and the same shape every time: a variable set to its starting value before the loop (`Total <- 0`), one update per pass inside it (`Total <- Total + 1`), and the finished value read after it. The variable is called an accumulator because it accumulates.', + 'Counting and summing are the same pattern. To count the `A`s in a text, add 1 on each pass where the condition holds. To sum a list of prices, add the price on every pass. To find the largest, keep the largest so far and replace it when a bigger one appears. The accumulator always has a meaning you can say in one sentence at the top of every pass: "Total is the number of `A`s among the positions before `I`". That sentence is the loop invariant, and a later lesson shows how the prover checks it.', + 'The two mistakes are both about the start. Forgetting `Total <- 0` means the first `Total + 1` reads a name with no value, and the run stops there and tells you. Putting `Total <- 0` inside the loop resets the count on every pass, so the answer is 0 or 1. The trace makes both visible in one step.', + '- Start the accumulator before the loop. Update it once per pass. Read it after.', + '- The accumulator has a one-sentence meaning that is true at the top of every pass.', + '- A count is a sum of ones; a maximum is a "best so far".', + ], + example: { + pseudo: 'DECLARE T : STRING\nDECLARE Total : INTEGER\nT <- "GATTACA"\nTotal <- 0\nFOR I <- 0 TO LENGTH(T)\n IF T[I] == \'A\' THEN\n Total <- Total + 1\n ENDIF\nNEXT I\nOUTPUT Total', + expect: ['3'], + watch: 'Total in the variables panel: 0, then 1 at position 1, 2 at position 4, 3 at position 6. At the top of every pass, Total is the number of A among the positions before I.', + }, + checks: [ + { question: 'Where does `Total <- 0` go?', options: ['Before the loop', 'As the first line inside the loop', 'After NEXT'], answer: 0, why: 'Inside the loop it would reset the total every pass. After the loop it is too late.' }, + { question: 'A loop keeps the largest value seen in Best. What is the update?', options: ['Best <- X', 'IF X > Best THEN Best <- X ENDIF', 'Best <- Best + X'], answer: 1, why: 'Best only changes when a larger value appears.' }, + ], + flavors: [ + { id: 'cells', hook: 'GC content, the fraction of G and C in a read, is the first number any sequencing report shows. It is one accumulator and one division.', task: 'Count the G and C bases in "GGCATCGA" and print the count and the GC fraction.', solution: { pseudo: 'DECLARE R : STRING\nDECLARE GC : INTEGER\nR <- "GGCATCGA"\nGC <- 0 // ~ start the count\nFOR I <- 0 TO LENGTH(R)\n IF R[I] == \'G\' OR R[I] == \'C\' THEN\n GC <- GC + 1 // ~ one more strong base\n ENDIF\nNEXT I\nOUTPUT GC\nOUTPUT GC / LENGTH(R)', expect: ['5', '0.625'] } }, + { id: 'money', hook: 'A shopping total is a sum accumulator; the dearest item is a maximum accumulator. Receipts do both.', task: 'Prices in cents are 299, 1250, 499, 875. Print the total and the dearest price. (Each price is handled in its own step here; lists come later.)', solution: { pseudo: 'DECLARE Total, Dearest : INTEGER\nTotal <- 0\nDearest <- 0\nFOR I <- 0 TO 4\n DECLARE P : INTEGER\n IF I == 0 THEN\n P <- 299\n ELSEIF I == 1 THEN\n P <- 1250\n ELSEIF I == 2 THEN\n P <- 499\n ELSE\n P <- 875\n ENDIF\n Total <- Total + P // ~ add this price to the total\n IF P > Dearest THEN\n Dearest <- P // ~ a new dearest\n ENDIF\nNEXT I\nOUTPUT Total, " ", Dearest', expect: ['2923 1250'] } }, + { id: 'games', hook: 'A dice total over several rolls, and the number of sixes, are two accumulators running in the same loop.', task: 'Rolls are given as the text "361654". Print the total of the rolls and how many were sixes.', solution: { pseudo: 'DECLARE Rolls : STRING\nDECLARE Total, Sixes : INTEGER\nRolls <- "361654"\nTotal <- 0\nSixes <- 0\nFOR I <- 0 TO LENGTH(Rolls)\n Total <- Total + TO_INTEGER(Rolls[I]) // ~ the digit as a number\n IF Rolls[I] == \'6\' THEN\n Sixes <- Sixes + 1 // ~ count the six\n ENDIF\nNEXT I\nOUTPUT Total, " ", Sixes', expect: ['25 2'] } }, + ], + }, + { + id: 'reading-a-run', title: 'Reading a run, not the code', stage: 'foundations', minutes: 20, + purpose: 'You will be able to predict the next state of a short program before stepping, and use the trace to find where a program does something other than what you meant.', + teaches: ['tracing'], requires: ['accumulator'], + explanation: [ + 'Code says what you meant. A run shows what the machine did. When they disagree, the run is right, and the only way to find the disagreement is to look at the run one step at a time. This lesson is about using the trace deliberately rather than watching it go by.', + 'Three habits. First, predict before you step: say which line runs next and what changes, then press Step and check. Predict mode (press P) hides the next state until you commit. Second, watch one variable: its history ("was 15; changed at step 4") tells you when it last changed, which is usually where a mistake is. Third, go to the last step first: if the final answer is wrong, step backwards from there to the last place the answer-carrying variable was right.', + 'The example has two surprises for most readers, and both are the kind of thing an agent-written program gets wrong quietly. Predict the two outputs before you run it.', + '- Predict, then step, then compare. The gap between prediction and run is where understanding happens.', + '- A wrong final answer means a specific step went wrong. Walk back from the end to find it.', + '- The trace is the ground truth. The code is the claim.', + ], + example: { + pseudo: 'DECLARE A, B : INTEGER\nA <- 5\nB <- A\nA <- A + 1\nOUTPUT A, " ", B\nDECLARE T : STRING\nDECLARE Count : INTEGER\nT <- "ABC"\nCount <- 0\nFOR I <- 1 TO LENGTH(T)\n Count <- Count + 1\nNEXT I\nOUTPUT Count', + expect: ['6 5', '2'], + watch: 'Predict the first OUTPUT. B was a copy of A at the moment of B <- A, so it stays 5. Then predict Count: the loop starts at 1, not 0, so it visits positions 1 and 2 only. The trace shows I taking two values.', + }, + checks: [ + { question: 'The final answer of a loop is wrong. Where do you look first?', options: ['The first line', 'The last step where the answer variable changed', 'The DECLARE lines'], answer: 1, why: 'Walking back from the wrong answer to its last change finds the step that went wrong.' }, + { question: 'In predict mode you guessed the next line wrong. What does that tell you?', options: ['The program has a bug', 'Your model of the program differs from the program', 'Nothing'], answer: 1, why: 'The run is what the program does. A wrong prediction is the gap between what you meant and what it says.' }, + ], + flavors: [ + { id: 'cells', hook: 'A GC count that comes out one short is the most common bug in a first bioinformatics script. The trace finds it in under a minute.', task: 'This program is meant to count the G and C in "GCGC" and print 4. It prints 3. Step through it, find the off-by-one, and fix it.', starter: 'DECLARE R : STRING\nDECLARE GC : INTEGER\nR <- "GCGC"\nGC <- 0\nFOR I <- 0 TO LENGTH(R) - 1\n IF R[I] == \'G\' OR R[I] == \'C\' THEN\n GC <- GC + 1\n ENDIF\nNEXT I\nOUTPUT GC', solution: { pseudo: 'DECLARE R : STRING\nDECLARE GC : INTEGER\nR <- "GCGC"\nGC <- 0\nFOR I <- 0 TO LENGTH(R)\n IF R[I] == \'G\' OR R[I] == \'C\' THEN\n GC <- GC + 1\n ENDIF\nNEXT I\nOUTPUT GC', expect: ['4'] } }, + { id: 'money', hook: 'A total that resets itself is a bug that prints a plausible number. Only the trace shows the reset.', task: 'This program should total 100 + 200 + 300 and print 600. It prints 300. Step through, watch Total, and fix it.', starter: 'DECLARE Total : INTEGER\nFOR I <- 1 TO 4\n Total <- 0\n Total <- Total + I * 100\nNEXT I\nOUTPUT Total', solution: { pseudo: 'DECLARE Total : INTEGER\nTotal <- 0\nFOR I <- 1 TO 4\n Total <- Total + I * 100\nNEXT I\nOUTPUT Total', expect: ['600'] } }, + { id: 'games', hook: 'A swap that loses a value is the classic trace exercise: after it, both players hold the same score.', task: 'The two scores should swap, printing "2 7". The program prints "2 2". Step through, watch both names, and fix it with a third variable.', starter: 'DECLARE Home, Away : INTEGER\nHome <- 7\nAway <- 2\nHome <- Away\nAway <- Home\nOUTPUT Home, " ", Away', solution: { pseudo: 'DECLARE Home, Away, Temp : INTEGER\nHome <- 7\nAway <- 2\nTemp <- Home\nHome <- Away\nAway <- Temp\nOUTPUT Home, " ", Away', expect: ['2 7'] } }, + ], + }, +]; diff --git a/src/curriculum/index.ts b/src/curriculum/index.ts new file mode 100644 index 0000000..d23f6b1 --- /dev/null +++ b/src/curriculum/index.ts @@ -0,0 +1,75 @@ +/** + * The lesson corpus verifies the interpreter; there is no Learn UI. This + * module is kept as a fixture: `LESSONS` is a curriculum-shaped set of + * programs whose dependency graph (derived from `teaches` and `requires`, + * never written by hand) `tests/curriculum.test.mjs` checks for strictness, + * and whose examples and solutions it runs through the real interpreter. + */ +import type { Lesson, Stage } from './types.js'; +import { CONCEPTS, CONCEPT_BY_ID } from './concepts.js'; +import { FOUNDATIONS } from './foundations.js'; +import { CORE } from './core.js'; +import { ADVANCED } from './advanced.js'; +import { SCIENCE } from './science.js'; + +export * from './types.js'; +export { CONCEPTS, CONCEPT_BY_ID } from './concepts.js'; + +export const LESSONS: Lesson[] = [...FOUNDATIONS, ...CORE, ...ADVANCED, ...SCIENCE]; +export const LESSON_BY_ID: Record = Object.fromEntries(LESSONS.map((l) => [l.id, l])); +export const STAGES: Array<{ id: Stage; title: string; blurb: string }> = [ + { id: 'foundations', title: 'Foundations', blurb: 'From "what is a procedure" to reading a run. Nothing assumed.' }, + { id: 'core', title: 'Core', blurb: 'The rest of the notation: loops of every kind, subroutines, collections, and the flavors of code.' }, + { id: 'advanced', title: 'Advanced', blurb: 'Recursion, specification and proof, cost, the classic algorithms, the machine, and reading code you did not write.' }, +]; + +/** The lesson that teaches a concept. Exactly one, which the tests enforce. */ +export function teacherOf(conceptId: string): Lesson | undefined { + return LESSONS.find((l) => l.teaches.includes(conceptId)); +} + +/** The lessons a lesson depends on directly: those teaching any concept it requires. */ +export function prerequisitesOf(lesson: Lesson): Lesson[] { + const seen = new Set(); + const out: Lesson[] = []; + for (const c of lesson.requires) { + const t = teacherOf(c); + if (t && !seen.has(t.id)) { seen.add(t.id); out.push(t); } + } + return out; +} + +/** The lessons that depend on this one directly: those requiring a concept it teaches. */ +export function dependentsOf(lesson: Lesson): Lesson[] { + return LESSONS.filter((l) => l.requires.some((c) => lesson.teaches.includes(c))); +} + +/** Every lesson that must be understood before this one, transitively, in curriculum order. */ +export function allPrerequisitesOf(lesson: Lesson): Lesson[] { + const seen = new Set(); + const visit = (l: Lesson) => { for (const p of prerequisitesOf(l)) if (!seen.has(p.id)) { seen.add(p.id); visit(p); } }; + visit(lesson); + return LESSONS.filter((l) => seen.has(l.id)); +} + +/** Depth in the dependency graph: 0 for lessons with no requirements. Used to lay out the map. */ +export function depthOf(lesson: Lesson, memo = new Map()): number { + if (memo.has(lesson.id)) return memo.get(lesson.id)!; + memo.set(lesson.id, -1); // cycle guard; the tests make sure this never matters + const d = prerequisitesOf(lesson).reduce((m, p) => Math.max(m, depthOf(p, memo) + 1), 0); + memo.set(lesson.id, d); + return d; +} + +/* ------------------------------------------------------------------ */ +/* Status, for the dependency-graph tests */ +/* ------------------------------------------------------------------ */ + +export interface Progress { done: string[]; flavor?: string; checks: Record } + +export type LessonStatus = 'done' | 'available' | 'locked'; + +export function statusOf(lesson: Lesson, progress: Progress): LessonStatus { + if (progress.done.includes(lesson.id)) return 'done'; + return prerequisitesOf(lesson).every((p) => progress.done.includes(p.id)) ? 'available' : 'locked'; +} diff --git a/src/curriculum/science.ts b/src/curriculum/science.ts new file mode 100644 index 0000000..0f967ea --- /dev/null +++ b/src/curriculum/science.ts @@ -0,0 +1,87 @@ +import type { Lesson } from './types.js'; + +/** Science: the numerical vocabulary, equations as a source language, and loops whose passes may run at once. */ +export const SCIENCE: Lesson[] = [ + { + id: 'numbers-for-science', title: 'Numbers for science', stage: 'advanced', minutes: 25, + purpose: 'You will be able to use the statistical, trigonometric, logarithmic and random built-ins, know which are loops in disguise, and keep counts and measurements in their own types.', + teaches: ['numerics'], requires: ['float-precision', 'list', 'for-each', 'accumulator'], + explanation: [ + 'A built-in is a word for a loop you could write. `SUM_OF(L)`, `MEAN_OF(L)`, `VARIANCE_OF(L)`, `STDEV_OF(L)` and `MEDIAN_OF(L)` each stand for a pass over the list with an accumulator; `VARIANCE_OF` divides by N - 1, the sample variance, which is what a statistics course and a spreadsheet mean. `DOT(A, B)` and `NORM_OF(A)` are the same idea for vectors, and `FILL(N, V)` makes a list of N copies of V, which is how a buffer is made before a loop fills it.', + 'The rest are functions in the school sense: `SQRT`, `EXP`, `LOG` (natural), `LOG2`, `LOG10` (pH and decibels), `SIN`, `COS`, `TAN`, `ATAN2(Y, X)`, `PI()`, `FLOOR`, `CEIL`, `ROUND`, `ABS`, `MIN`, `MAX`. `CHOOSE(N, K)` is the binomial coefficient, the count behind every "how many ways" and every binomial probability. `RAND(X)` draws a FLOAT in [0, X), `RANDINT(A, B)` an INTEGER in [A, B), `RANDN()` a draw from the standard normal distribution.', + 'The discipline from the floats lesson still rules. `CHOOSE` and `FLOOR` give INTEGERs because they count; `MEAN_OF` gives a FLOAT because it measures. A probability is a FLOAT between 0 and 1; a number of trials is an INTEGER; `/` turns a count into a measurement, and `DIV` keeps it a count. When a formula from a paper mixes the two, decide which each symbol is before you type it.', + '- SUM_OF, MEAN_OF, VARIANCE_OF (N - 1), STDEV_OF, MEDIAN_OF, DOT, NORM_OF, FILL work on lists of numbers.', + '- CHOOSE(N, K), RAND, RANDINT, RANDN, and the trigonometric and logarithmic functions are the rest.', + '- Counts are INTEGER, measurements are FLOAT, and the built-ins respect that: CHOOSE counts, MEAN_OF measures.', + ], + example: { + pseudo: 'FUNCTION BINOMIAL(K : INTEGER, N : INTEGER, P : FLOAT) RETURNS FLOAT\n RETURN CHOOSE(N, K) * P ^ K * (1 - P) ^ (N - K)\nENDFUNCTION\nDECLARE Lengths : LIST OF FLOAT\nLengths <- [4.1, 3.9, 4.6, 4.2, 3.8, 4.4]\nOUTPUT ROUND(MEAN_OF(Lengths) * 100) / 100, " ", ROUND(STDEV_OF(Lengths) * 100) / 100, " ", MEDIAN_OF(Lengths)\nOUTPUT ROUND(BINOMIAL(3, 10, 0.25) * 10000) / 10000\nOUTPUT FLOOR(7 / 2), " ", CEIL(7 / 2), " ", ROUND(LOG10(1000.0))', + expect: ['4.17 0.3 4.15', '0.2503', '3 4 3'], + watch: 'The first line is three built-ins over one list. The binomial line is one formula with its three kinds of name: K and N count, P measures. The last line shows the three ways a FLOAT becomes a whole number.', + }, + checks: [ + { question: '`VARIANCE_OF([2.0, 4.0, 4.0, 6.0])` is 2.67, not 2. Why?', options: ['A float rounding error', 'It divides by N - 1, the sample variance', 'It ignores the first value'], answer: 1, why: 'The sample variance corrects for estimating the mean from the same values; dividing by N would give 2.' }, + { question: 'Which of these is an INTEGER?', options: ['MEAN_OF(L)', 'CHOOSE(10, 3)', 'SQRT(16)'], answer: 1, why: 'CHOOSE counts ways to pick, so it is whole. The other two measure and give FLOATs, even when the answer happens to be whole.' }, + ], + flavors: [ + { id: 'cells', hook: 'A sequencing run reports the mean and spread of its read lengths, and a z-score says how unusual one read is.', task: 'Write Z_SCORE(X, Xs) returning how many standard deviations X is from the mean of Xs, rounded to two decimals, and print it for 4.6 in [4.1, 3.9, 4.6, 4.2, 3.8, 4.4].', solution: { pseudo: 'FUNCTION Z_SCORE(X : FLOAT, Xs : LIST OF FLOAT) RETURNS FLOAT\n RETURN ROUND((X - MEAN_OF(Xs)) / STDEV_OF(Xs) * 100) / 100 // ~ the distance from the mean, in standard deviations\nENDFUNCTION\nOUTPUT Z_SCORE(4.6, [4.1, 3.9, 4.6, 4.2, 3.8, 4.4])', expect: ['1.44'] } }, + { id: 'games', hook: 'How likely is at least one six in four throws of a die? Count the ways the complement happens, or sum the binomial terms.', task: 'Write AT_LEAST_ONE(N, P) returning 1 - (1 - P) ^ N, and print it for 4 throws with P = 1 / 6, rounded to three decimals.', solution: { pseudo: 'FUNCTION AT_LEAST_ONE(N : INTEGER, P : FLOAT) RETURNS FLOAT\n RETURN 1 - (1 - P) ^ N // ~ one minus the chance of no success at all\nENDFUNCTION\nOUTPUT ROUND(AT_LEAST_ONE(4, 1 / 6) * 1000) / 1000', expect: ['0.518'] } }, + { id: 'money', hook: 'Continuous compounding is EXP in disguise: a balance grows by e to the rate times the years.', task: 'Write GROWTH(Rate, Years) returning EXP(Rate * Years), and print 1000 cents grown at 5% for 10 years, rounded to a whole cent.', solution: { pseudo: 'FUNCTION GROWTH(Rate : FLOAT, Years : FLOAT) RETURNS FLOAT\n RETURN EXP(Rate * Years) // ~ e to the power of rate times time\nENDFUNCTION\nOUTPUT ROUND(1000 * GROWTH(0.05, 10.0))', expect: ['1649'] } }, + { id: 'everyday', hook: 'A ladder leaning at an angle: how high does it reach? Trigonometry is three built-ins and a conversion from degrees.', task: 'Write REACH(Length, Degrees) returning Length * SIN(Degrees * PI() / 180), and print it for a 5 m ladder at 75 degrees, rounded to two decimals.', solution: { pseudo: 'FUNCTION REACH(Length : FLOAT, Degrees : FLOAT) RETURNS FLOAT\n RETURN Length * SIN(Degrees * PI() / 180) // ~ SIN wants radians\nENDFUNCTION\nOUTPUT ROUND(REACH(5.0, 75.0) * 100) / 100', expect: ['4.83'] } }, + ], + }, + { + id: 'from-equations', title: 'From an equation to a procedure', stage: 'advanced', minutes: 25, + purpose: 'You will be able to turn a formula written in LaTeX into pseudo by hand, read the translation the matho pane produces, and say why each of its decisions was made.', + teaches: ['equations'], requires: ['numerics', 'nesting', 'function'], + explanation: [ + 'A formula says what a quantity is; a procedure says how to get it. The gap is small and precise: a sum is a loop with an accumulator, a subscript is a position in a list, a case bracket is an IF, and every letter has to be given a type. The matho pane does this translation in front of you. Paste `\\bar{x} = \\frac{1}{n} \\sum_{i=0}^{n-1} x_i` and the pseudo pane shows `FUNCTION X_BAR(N : INTEGER, X : LIST OF FLOAT)` with a `FOR I <- 0 TO N` loop adding `X[I]` into `Total`.', + 'Read the notes it leaves in the Problems view; they are the decisions you would otherwise make without noticing. `n` bounds a sum, so it is an INTEGER. `x` carries a subscript, so it is a list. A sum from `i = 1` meets a list that starts at 0, so `x_i` becomes `X[I - 1]`. A reference to an earlier equation inside a loop does not change from pass to pass, so it is computed once before the loop. Two symbols that would be one name in pseudo (`p` and `P`) are told apart with a suffix.', + 'Some mathematics is refused, and the refusal says what to write instead: an integral is the sum it approximates, over small steps; a sum to infinity needs a bound that becomes a parameter; a limit or a derivative is a difference quotient. These are not limitations of the pane. They are the difference between a statement about numbers and a procedure a machine can run, and learning to write the procedure yourself is the point.', + '- A sum with bounds is FOR plus an accumulator; the upper bound is inclusive in mathematics and TO is exclusive.', + '- A subscript that is an index makes a LIST; any other subscript is part of the name (v_0 is V0).', + '- Integrals, infinity, limits and derivatives are refused with the finite procedure to write instead.', + ], + example: { + pseudo: '// \\bar{x} = \\frac{1}{n} \\sum_{i=0}^{n-1} x_i\nFUNCTION X_BAR(N : INTEGER, X : LIST OF FLOAT) RETURNS FLOAT\n DECLARE Total : FLOAT\n Total <- 0.0\n FOR I <- 0 TO N\n Total <- Total + X[I]\n NEXT I\n RETURN 1 / N * Total\nENDFUNCTION\n\n// \\sigma^2 = \\frac{1}{n - 1} \\sum_{i=0}^{n-1} (x_i - \\bar{x})^2\nFUNCTION SIGMA_SQUARED(N : INTEGER, X : LIST OF FLOAT) RETURNS FLOAT\n DECLARE Total : FLOAT\n DECLARE XBar : FLOAT\n XBar <- X_BAR(N, X)\n Total <- 0.0\n FOR I <- 0 TO N\n Total <- Total + (X[I] - XBar) ^ 2\n NEXT I\n RETURN 1 / (N - 1) * Total\nENDFUNCTION\nOUTPUT SIGMA_SQUARED(4, [2.0, 4.0, 4.0, 6.0]), " ", VARIANCE_OF([2.0, 4.0, 4.0, 6.0])', + expect: ['2.6666666666666665 2.6666666666666665'], + watch: 'This is exactly what the matho pane produces for the two equations in the comments; paste them there and compare. XBar is computed once, before the second loop. The last line checks the translation against the built-in.', + }, + checks: [ + { question: 'In `\\sum_{i=1}^{n} x_i`, what does `x_i` become?', options: ['X[I]', 'X[I - 1], because lists start at 0', 'XI, a name'], answer: 1, why: 'The sum counts from 1 and the list from 0, so the position is one less than the counter; the pane says so in a note.' }, + { question: 'Why is `\\int_0^1 f(x) dx` refused?', options: ['Integrals are too slow', 'An integral is not a finite procedure; write the sum over steps that approximates it', 'The pane does not know f'], answer: 1, why: 'A program runs a finite number of steps. The sum over n steps of width h is the procedure; n becomes a parameter.' }, + ], + flavors: [ + { id: 'cells', hook: 'The root-mean-square deviation between two structures is a formula every structural biologist has typed wrong once: sum the squared distances, divide, take the root.', task: 'Translate rmsd = \\sqrt{\\frac{1}{n} \\sum_{i=0}^{n-1} d_i^2} by hand into RMSD(N, D), then print it for [3.0, 4.0].', solution: { pseudo: 'FUNCTION RMSD(N : INTEGER, D : LIST OF FLOAT) RETURNS FLOAT\n DECLARE Total : FLOAT\n Total <- 0.0\n FOR I <- 0 TO N\n Total <- Total + D[I] ^ 2 // ~ the sum of the squared distances\n NEXT I\n RETURN SQRT(1 / N * Total) // ~ divide, then the root\nENDFUNCTION\nOUTPUT RMSD(2, [3.0, 4.0])', expect: ['3.5355339059327378'] } }, + { id: 'money', hook: 'The present value of a series of payments is a sum with the discount factor raised to the year: a loop with a power inside it.', task: 'Translate PV = \\sum_{t=1}^{n} \\frac{c}{(1 + r)^t} into PRESENT_VALUE(N, C, R) and print it for 3 payments of 100.0 at 10%, rounded to two decimals.', solution: { pseudo: 'FUNCTION PRESENT_VALUE(N : INTEGER, C : FLOAT, R : FLOAT) RETURNS FLOAT\n DECLARE Total : FLOAT\n Total <- 0.0\n FOR T <- 1 TO N + 1 // ~ the sum runs from 1 to n inclusive\n Total <- Total + C / (1 + R) ^ T\n NEXT T\n RETURN Total\nENDFUNCTION\nOUTPUT ROUND(PRESENT_VALUE(3, 100.0, 0.1) * 100) / 100', expect: ['248.69'] } }, + { id: 'machines', hook: 'An integral in a paper is a sum in a program. Writing the Riemann sum yourself, with its step, is the whole trick behind numerical integration.', task: 'Write AREA(N, A, B) returning \\sum_{i=0}^{n-1} f(a + i h) h with h = (b - a) / n and f(x) = x^2, and print it for 1000 steps from 0 to 1, rounded to three decimals.', solution: { pseudo: 'FUNCTION F(X : FLOAT) RETURNS FLOAT\n RETURN X ^ 2\nENDFUNCTION\nFUNCTION AREA(N : INTEGER, A : FLOAT, B : FLOAT) RETURNS FLOAT\n DECLARE H, Total : FLOAT\n H <- (B - A) / N // ~ the width of one step, computed once\n Total <- 0.0\n FOR I <- 0 TO N\n Total <- Total + F(A + I * H) * H // ~ one rectangle per step\n NEXT I\n RETURN Total\nENDFUNCTION\nOUTPUT ROUND(AREA(1000, 0.0, 1.0) * 1000) / 1000', expect: ['0.333'] } }, + ], + }, + { + id: 'many-cores', title: 'Many cores at once', stage: 'advanced', minutes: 25, + purpose: 'You will be able to say when a loop\'s passes are independent, write it as PARALLEL FOR, and read the data-race report when they are not.', + teaches: ['parallel'], requires: ['numerics', 'nesting', 'runtime-stops', 'cost'], + explanation: [ + 'A `FOR` loop runs its passes one after another. `PARALLEL FOR I <- A TO B` promises that the passes do not depend on each other, so a machine with many cores may run them all at once; this is what a GPU does with thousands of cores, and what every multi-core processor does with a few. The notation does not run them in parallel itself. It runs them in order and checks the promise: if a pass writes a value that another pass reads or writes, the run stops with a data race, and the message names the value and the two passes.', + 'The rule for keeping the promise: every pass owns its own slot. `Squares[I] <- Xs[I] * Xs[I]` is parallel because pass I touches only position I of each list. `Total <- Total + Xs[I]` is not, because every pass writes `Total`; there is no first pass in a parallel loop, so the result would depend on which core got there first. The fix is always the same: give each pass a slot, with `FILL`, and combine the slots after the loop, with `SUM_OF` or a plain loop. That combining step is called a reduction, and it is the half of parallel programming that is actually hard.', + 'Reading from one list and writing to another is what makes a stencil (each new value from its old neighbors) parallel. Write the result back into the list being read and the detector stops the run on the first neighbor clobbered. The code pane shows what each language would use for the loop: `parallelize` in Mojo, `#pragma omp parallel for` in C, `par_iter` in Rust. None of them check the promise. Pseudo does, which is why the habit is worth building here.', + '- PARALLEL FOR promises independent passes; the run stops with a race when two passes touch one location.', + '- Each pass owns its slot: FILL a list before the loop, write Slot[I] inside it, combine after.', + '- A reduction (a total, a maximum) is not one parallel loop; it is a parallel map and then a combine.', + ], + example: { + pseudo: 'FUNCTION SUM_OF_SQUARES(Xs : LIST OF FLOAT) RETURNS FLOAT\n DECLARE Squares : LIST OF FLOAT\n Squares <- FILL(LENGTH(Xs), 0.0)\n PARALLEL FOR I <- 0 TO LENGTH(Xs)\n Squares[I] <- Xs[I] * Xs[I]\n NEXT I\n RETURN SUM_OF(Squares)\nENDFUNCTION\nOUTPUT SUM_OF_SQUARES([1.0, 2.0, 3.0, 4.0])', + expect: ['30.0'], + watch: 'Each pass writes its own position of Squares and reads its own position of Xs. Now change the body to Total <- Total + Xs[I] * Xs[I] (declaring Total) and run: the detector stops on pass 2 and tells you why.', + }, + checks: [ + { question: 'Which body can be a PARALLEL FOR?', options: ['Out[I] <- Out[I - 1] + Xs[I]', 'Out[I] <- Xs[I] * 2', 'Total <- Total + Xs[I]'], answer: 1, why: 'Only the middle one touches nothing another pass touches. The first reads what pass I - 1 writes; the last has every pass write Total.' }, + { question: 'What does the run do when two passes write the same location?', options: ['Runs them in order and continues', 'Stops with a data race that names the location and the passes', 'Uses the last value'], answer: 1, why: 'The promise was broken; the notation refuses to pretend an order exists.' }, + ], + flavors: [ + { id: 'cells', hook: 'A sequencing run has millions of reads and the GC fraction of each is independent of every other: the textbook parallel map.', task: 'Write GC_OF_EACH(Reads) returning a list with the GC fraction of each read, computed in a PARALLEL FOR, and print it for ["GGCC", "GATTACA", "AT"].', solution: { pseudo: 'FUNCTION GC(T : STRING) RETURNS FLOAT\n DECLARE N : INTEGER\n N <- 0\n FOR EACH B IN T\n IF B == \'G\' OR B == \'C\' THEN\n N <- N + 1\n ENDIF\n NEXT B\n RETURN N / LENGTH(T)\nENDFUNCTION\nFUNCTION GC_OF_EACH(Reads : LIST OF STRING) RETURNS LIST OF FLOAT\n DECLARE Out : LIST OF FLOAT\n Out <- FILL(LENGTH(Reads), 0.0) // ~ one slot per read, before the loop\n PARALLEL FOR I <- 0 TO LENGTH(Reads)\n Out[I] <- GC(Reads[I]) // ~ pass I owns slot I\n NEXT I\n RETURN Out\nENDFUNCTION\nOUTPUT GC_OF_EACH(["GGCC", "GATTACA", "AT"])', expect: ['[1.0, 0.2857142857142857, 0.0]'] } }, + { id: 'machines', hook: 'The tempting way to total a list in parallel is wrong, and the report says exactly how. Reading it once is worth more than a lecture on memory models.', task: 'Write the wrong version: ADD_UP(Xs) with a PARALLEL FOR whose every pass does Total <- Total + Xs[I]. Run it and read the race report.', solution: { pseudo: 'FUNCTION ADD_UP(Xs : LIST OF FLOAT) RETURNS FLOAT\n DECLARE Total : FLOAT\n Total <- 0.0\n PARALLEL FOR I <- 0 TO LENGTH(Xs)\n Total <- Total + Xs[I] // ~ every pass writes Total: the race\n NEXT I\n RETURN Total\nENDFUNCTION\nOUTPUT ADD_UP([1.0, 2.0, 3.0])', expect: [], stops: 'race' } }, + { id: 'games', hook: 'Scoring every square of a board from the board as it was is a stencil: read the old board, write a new one.', task: 'Write NEIGHBOR_SUMS(Row) returning, for each position of a list, the sum of its neighbors (0 beyond the ends), in a PARALLEL FOR, and print it for [1.0, 2.0, 3.0, 4.0].', solution: { pseudo: 'FUNCTION NEIGHBOR_SUMS(Row : LIST OF FLOAT) RETURNS LIST OF FLOAT\n DECLARE N : INTEGER\n DECLARE Out : LIST OF FLOAT\n N <- LENGTH(Row)\n Out <- FILL(N, 0.0)\n PARALLEL FOR I <- 0 TO N\n IF I > 0 THEN\n Out[I] <- Out[I] + Row[I - 1] // ~ read Row, write Out: independent passes\n ENDIF\n IF I < N - 1 THEN\n Out[I] <- Out[I] + Row[I + 1]\n ENDIF\n NEXT I\n RETURN Out\nENDFUNCTION\nOUTPUT NEIGHBOR_SUMS([1.0, 2.0, 3.0, 4.0])', expect: ['[2.0, 4.0, 6.0, 3.0]'] } }, + ], + }, +]; diff --git a/src/curriculum/types.ts b/src/curriculum/types.ts new file mode 100644 index 0000000..2d9e26a --- /dev/null +++ b/src/curriculum/types.ts @@ -0,0 +1,103 @@ +/** + * The educational wing, as data. + * + * A concept is one thing a person can understand or not understand. A lesson + * teaches one or two concepts and names the concepts a reader must already + * hold before it will make sense. That `requires` list is the dependency map: + * it is checked by `tests/curriculum.test.mjs` (every required concept is + * taught by an earlier lesson; the graph has no cycles) and drawn on the + * Learn page. A lesson is available once every lesson that teaches one of its + * requirements is marked understood. + * + * Every lesson has one worked example and two or more flavors. A flavor is the + * same concept wearing a different motivation (cells, money, games, words, + * everyday life), with a hook and a small exercise. The exercise is written as + * a complete solution; lines ending in `// ~ ` are the ones the starter + * blanks out (see `starterOf`). Every example and every solution runs under + * the real interpreter in the tests and must print exactly `expect`. + */ + +export type Stage = 'foundations' | 'core' | 'advanced'; + +export interface Concept { + id: string; + name: string; + /** One sentence a reader could use to check whether they hold the concept. */ + summary: string; +} + +export type FlavorId = 'cells' | 'money' | 'games' | 'words' | 'everyday' | 'machines'; + +export interface FlavorInfo { id: FlavorId; name: string; blurb: string } + +export interface Program { + pseudo: string; + /** The exact lines OUTPUT produces. */ + expect: string[]; + /** Lines INPUT reads, in order. */ + input?: string[]; + /** When the program is meant to stop: the kind of stop (index, unset, key, divide, overflow, depth). */ + stops?: string; +} + +export interface Flavor { + id: FlavorId; + /** Why someone who cares about this topic needs the concept. One or two sentences. */ + hook: string; + /** The task, as a request to the reader. */ + task: string; + /** A complete solution; `// ~ hint` marks the lines the starter blanks out. */ + solution: Program; + /** A starter that is not derived from the solution: a program to fix, say. */ + starter?: string; +} + +export interface Check { + question: string; + options: string[]; + /** Index into options. */ + answer: number; + why: string; +} + +export interface Lesson { + id: string; + title: string; + stage: Stage; + /** The one sentence that says what the reader will understand afterwards. */ + purpose: string; + teaches: string[]; + requires: string[]; + minutes: number; + /** Paragraphs. A line starting with `- ` is a bullet; backticks mark code. */ + explanation: string[]; + example: Program & { watch: string }; + checks: Check[]; + flavors: Flavor[]; + /** Catalog problems that practice this lesson, by id. */ + problems?: string[]; + /** A catalog problem whose assembly is shown beside the example (the machine lesson). */ + showAsmOf?: string; +} + +export const FLAVORS: FlavorInfo[] = [ + { id: 'cells', name: 'Cells and genomes', blurb: 'DNA, reads, proteins, and the counting that biology runs on.' }, + { id: 'money', name: 'Money', blurb: 'Budgets, interest, prices, and the arithmetic that must not be off by a cent.' }, + { id: 'games', name: 'Games', blurb: 'Dice, scores, boards, and rules a machine can referee.' }, + { id: 'words', name: 'Words and text', blurb: 'Letters, words, ciphers, and search.' }, + { id: 'everyday', name: 'Everyday life', blurb: 'Recipes, timetables, shopping, and the procedures you already follow.' }, + { id: 'machines', name: 'Machines and agents', blurb: 'What a processor, a compiler, or a code-writing agent is doing underneath.' }, +]; + +/** The starter a reader opens: every `// ~ hint` line becomes a TODO comment, everything else stays. */ +export function starterOf(p: Program): string { + return p.pseudo.split('\n').map((line) => { + const m = /^(\s*)(.*?)\s*\/\/ ~ (.*)$/.exec(line); + return m ? `${m[1]}// TODO: ${m[3]}` : line; + }).join('\n'); +} + +/** The solution with the markers removed, which is what the tests run. */ +export function solutionOf(p: Program): string { + return p.pseudo.split('\n').map((line) => line.replace(/\s*\/\/ ~ .*$/, '')).join('\n'); +} diff --git a/src/engines/asm86/cache.js b/src/engines/asm86/cache.js new file mode 100644 index 0000000..b04aada --- /dev/null +++ b/src/engines/asm86/cache.js @@ -0,0 +1,250 @@ +/** + * asm86/cache.js + * + * Caches, and the hierarchy they form. + * + * Memory is far away. A load that has to go all the way to DRAM waits a + * couple of hundred cycles, long enough to run hundreds of instructions. So + * every core keeps copies of recently used memory in small, fast stores near + * it: an L1 cache of a few tens of kilobytes that answers in about four + * cycles, a larger L2 behind it, a larger and slower L3 shared between + * cores, and only then memory itself. + * + * Three ideas carry the whole subject, and each is visible in these models: + * + * Lines. A cache never holds one byte. It holds lines, typically + * 64 bytes, so reading T[0] also brings T[1] to T[63]. + * That is why a sequential scan is fast. + * Sets. An address can only live in one set, chosen by a few bits + * of the address. Two addresses that choose the same set + * compete for its few ways, even when the rest of the cache + * is empty. That is a conflict miss. + * Replacement. When a set is full, one line must go. Least recently used + * is the usual answer. + * + * Every address splits into three fields: + * + * | tag | set index | offset in line | + * + * and the views draw exactly that split. + */ + +export const LEVEL_ORDER = ['L1i', 'L1d', 'L2', 'L3', 'memory']; + +function log2(n) { + const k = Math.log2(n); + if (!Number.isInteger(k)) throw new Error(`cache sizes must be powers of two; ${n} is not`); + return k; +} + +/** A deterministic stand-in for random replacement, so replays agree. */ +function hash32(a, b) { + let h = (a * 0x9e3779b1) ^ (b * 0x85ebca6b); + h ^= h >>> 15; h = Math.imul(h, 0x2c1b3c6d); h ^= h >>> 12; + return h >>> 0; +} + +/** + * One set-associative cache. + * + * config = { + * size: bytes, line: bytes per line, ways: lines per set, + * latency: cycles to answer a hit, + * policy: 'lru' | 'fifo' | 'random' + * } + */ +export class Cache { + constructor(name, config) { + this.name = name; + this.size = config.size; + this.line = config.line; + this.ways = config.ways; + this.latency = config.latency; + this.policy = config.policy || 'lru'; + this.numSets = this.size / (this.line * this.ways); + if (!Number.isInteger(this.numSets) || this.numSets < 1) throw new Error(`${name}: ${this.size} bytes cannot be split into ${this.ways}-way sets of ${this.line}-byte lines`); + this.offsetBits = log2(this.line); + this.indexBits = log2(this.numSets); + this.sets = new Map(); // set index -> array of `ways` slots, made on first use + this.stats = { accesses: 0, hits: 0, misses: 0, evictions: 0, writebacks: 0, prefetches: 0, prefetchHits: 0 }; + } + + /** Where an address lands: its tag, its set, and its byte within the line. */ + split(addr) { + const offset = addr & (this.line - 1); + const index = Math.floor(addr / this.line) % this.numSets; + const tag = Math.floor(addr / (this.line * this.numSets)); + return { tag, index, offset, lineAddr: addr - offset }; + } + + lineAddrOf(tag, index) { return (tag * this.numSets + index) * this.line; } + + set(index) { + let s = this.sets.get(index); + if (!s) { + s = Array.from({ length: this.ways }, () => ({ valid: false, tag: 0, dirty: false, lastUse: 0, filledAt: 0, prefetched: false })); + this.sets.set(index, s); + } + return s; + } + + find(index, tag) { + const s = this.sets.get(index); + if (!s) return -1; + return s.findIndex((w) => w.valid && w.tag === tag); + } + + victim(index, now) { + const s = this.set(index); + const empty = s.findIndex((w) => !w.valid); + if (empty >= 0) return empty; + if (this.policy === 'fifo') return s.reduce((best, w, k) => (w.filledAt < s[best].filledAt ? k : best), 0); + if (this.policy === 'random') return hash32(index, now) % this.ways; + return s.reduce((best, w, k) => (w.lastUse < s[best].lastUse ? k : best), 0); + } + + clone() { + const c = Object.create(Cache.prototype); + Object.assign(c, this); + c.sets = new Map([...this.sets].map(([k, v]) => [k, v.map((w) => ({ ...w }))])); + c.stats = { ...this.stats }; + return c; + } +} + +/** + * The levels between a core and memory. + * + * config = { + * l1i, l1d, l2, l3: cache configs, or null to leave a level out + * memory: { latency } + * prefetch: 'none' | 'next-line' + * } + * + * Every line size must match: real hierarchies almost always agree on it, + * and the model stays simple. + * + * Writes are write-back and write-allocate at every level: a store brings + * the line in, marks it dirty, and memory hears about it only when the line + * is finally evicted. + */ +export class Hierarchy { + constructor(config) { + this.config = config; + this.l1i = config.l1i ? new Cache('L1i', config.l1i) : null; + this.l1d = new Cache('L1d', config.l1d); + this.shared = [config.l2 ? new Cache('L2', config.l2) : null, config.l3 ? new Cache('L3', config.l3) : null].filter(Boolean); + this.memLatency = config.memory?.latency ?? 200; + this.prefetch = config.prefetch || 'none'; + this.now = 0; + this.memStats = { reads: 0, writes: 0 }; + const lines = [this.l1i, this.l1d, ...this.shared].filter(Boolean).map((c) => c.line); + if (new Set(lines).size > 1) throw new Error('every cache level must use the same line size'); + this.line = lines[0]; + } + + levels(side) { return [side === 'inst' && this.l1i ? this.l1i : this.l1d, ...this.shared]; } + + get all() { return [this.l1i, this.l1d, ...this.shared].filter(Boolean); } + + /** Bring one line to level i, filling every level above on the way back. */ + fetchLine(levels, i, lineAddr, isWrite, trail, isPrefetch = false) { + if (i >= levels.length) { + this.memStats.reads++; + trail.push({ level: 'memory', hit: true, latency: this.memLatency }); + return { latency: this.memLatency, level: 'memory' }; + } + const c = levels[i]; + const { index, tag } = c.split(lineAddr); + this.now++; + // A prefetch is the hardware's own guess, not a demand from the program, + // so it is counted apart: hit rates describe what the program asked for. + if (isPrefetch) c.stats.prefetches++; else c.stats.accesses++; + const way = c.find(index, tag); + if (way >= 0) { + if (!isPrefetch) c.stats.hits++; + const slot = c.set(index)[way]; + slot.lastUse = this.now; + const wasPrefetched = slot.prefetched && !isPrefetch; + if (wasPrefetched) { c.stats.prefetchHits++; slot.prefetched = false; } + if (isWrite && i === 0) slot.dirty = true; + trail.push({ level: c.name, hit: true, set: index, way, tag, latency: c.latency, wasPrefetched }); + return { latency: c.latency, level: c.name }; + } + if (!isPrefetch) c.stats.misses++; + const entry = { level: c.name, hit: false, set: index, tag, latency: c.latency }; + trail.push(entry); + const below = this.fetchLine(levels, i + 1, lineAddr, false, trail, isPrefetch); + const v = c.victim(index, this.now); + const old = c.set(index)[v]; + if (old.valid) { + c.stats.evictions++; + entry.evicted = { tag: old.tag, lineAddr: c.lineAddrOf(old.tag, index), dirty: old.dirty }; + if (old.dirty) { c.stats.writebacks++; this.writeBack(levels, i + 1, entry.evicted.lineAddr); } + } + Object.assign(old, { valid: true, tag, dirty: isWrite && i === 0, lastUse: this.now, filledAt: this.now, prefetched: isPrefetch }); + entry.way = v; + return below; + } + + /** A dirty line leaving level i-1 lands at level i, or in memory. */ + writeBack(levels, i, lineAddr) { + if (i >= levels.length) { this.memStats.writes++; return; } + const c = levels[i]; + const { index, tag } = c.split(lineAddr); + const way = c.find(index, tag); + this.now++; + if (way >= 0) { const s = c.set(index)[way]; s.dirty = true; s.lastUse = this.now; return; } + const v = c.victim(index, this.now); + const old = c.set(index)[v]; + if (old.valid) { + c.stats.evictions++; + if (old.dirty) { c.stats.writebacks++; this.writeBack(levels, i + 1, c.lineAddrOf(old.tag, index)); } + } + Object.assign(old, { valid: true, tag, dirty: true, lastUse: this.now, filledAt: this.now, prefetched: false }); + } + + /** + * One access of `size` bytes. An access that straddles a line boundary + * touches two lines, and costs as much as the slower of them. + */ + access(addr, size, kind, side = 'data') { + const levels = this.levels(side); + const first = Math.floor(addr / this.line); + const last = Math.floor((addr + Math.max(1, size) - 1) / this.line); + const lines = []; + for (let n = first; n <= last; n++) { + const lineAddr = n * this.line; + const trail = []; + const r = this.fetchLine(levels, 0, lineAddr, kind === 'write', trail); + let prefetch = null; + // Tagged next-line prefetch: a miss, or the first use of a line the + // prefetcher brought in, asks for the line after it. A steady stream + // therefore stays one line ahead of the program. + if (this.prefetch === 'next-line' && side === 'data' && (!trail[0].hit || trail[0].wasPrefetched)) { + const next = lineAddr + this.line; + const top = levels[0].split(next); + if (levels[0].find(top.index, top.tag) < 0) { + const ptrail = []; + this.fetchLine(levels, 0, next, false, ptrail, true); + prefetch = { lineAddr: next, trail: ptrail }; + } + } + lines.push({ lineAddr, trail, latency: r.latency, level: r.level, prefetch }); + } + const worst = lines.reduce((a, b) => (b.latency > a.latency ? b : a)); + return { addr, size, kind, side, lines, latency: worst.latency, level: worst.level, split: lines.length > 1 }; + } + + clone() { + const h = Object.create(Hierarchy.prototype); + Object.assign(h, this); + h.l1i = this.l1i ? this.l1i.clone() : null; + h.l1d = this.l1d.clone(); + h.shared = this.shared.map((c) => c.clone()); + h.memStats = { ...this.memStats }; + return h; + } + + cache(name) { return this.all.find((c) => c.name === name) || null; } +} diff --git a/src/engines/asm86/dom.js b/src/engines/asm86/dom.js new file mode 100644 index 0000000..d7225dc --- /dev/null +++ b/src/engines/asm86/dom.js @@ -0,0 +1,35 @@ +/** + * asm86/dom.js + * + * DOM helpers for the assembly widgets: the shared core (el, sv, add, clear, + * commas, plural, ...) plus the helpers that carry this family's CSS class + * prefix (.a86-*, see css/widgets/asm86.css). + */ + +import { el } from '../shared/dom.js'; +import { REGS, regValue } from './interp.js'; + +export * from '../shared/dom.js'; + +export function panel(title, body, extraClass = '') { + return el('section', { class: `a86-panel ${extraClass}` }, [ + title ? el('h4', { class: 'a86-panel-title', text: title }) : null, + body, + ]); +} + +/** Resolve a spec field that may be a register name or a plain number. */ +export function valueOf(state, spec) { + if (spec == null) return null; + if (typeof spec === 'number') return BigInt(spec); + if (typeof spec === 'bigint') return spec; + if (typeof spec === 'string' && REGS[spec.toLowerCase()]) return regValue(state, spec); + return null; +} + +/** A small legend: [[className, label], ...] */ +export function legend(items, extraClass = '') { + return el('div', { class: `a86-legend ${extraClass}` }, items.map(([cls, label]) => el('span', { class: 'a86-legend-item' }, [ + el('span', { class: `a86-swatch ${cls}` }), label, + ]))); +} diff --git a/src/engines/asm86/encode.js b/src/engines/asm86/encode.js new file mode 100644 index 0000000..5e3022a --- /dev/null +++ b/src/engines/asm86/encode.js @@ -0,0 +1,590 @@ +/** + * asm86/encode.js + * + * The rung below assembly: the bytes the processor actually fetches. This + * turns each instruction the assembler produced into x86-64 machine code, + * and keeps a record of which byte is which, so a view can show a student + * that `movzx r8d, byte ptr [rdi + rcx*1]` is five bytes, and what every + * bit of those five bytes says. + * + * The encodings match GNU as byte for byte for every form this subset + * accepts (test/encode.test.mjs checks that against the real assembler). + * Where x86 offers two encodings of one instruction, this picks the one GNU + * picks, which is usually the shorter. + * + * An x86-64 instruction is assembled from up to seven parts, always in this + * order, and every one of them except the opcode is optional: + * + * prefixes REX opcode ModRM SIB displacement immediate + * + * Public API + * encodeInstruction(instr, addr, labelAddr, near) -> { bytes, parts } + * layoutProgram(program, base) -> layout + * + * No dependencies, so interp.js can import it without a cycle. + */ + +/* ------------------------------------------------------------------ */ +/* Registers as the encoding sees them */ +/* ------------------------------------------------------------------ */ + +const GPR = ['rax', 'rcx', 'rdx', 'rbx', 'rsp', 'rbp', 'rsi', 'rdi']; +const REGINFO = (() => { + const t = {}; + const n32 = ['eax', 'ecx', 'edx', 'ebx', 'esp', 'ebp', 'esi', 'edi']; + const n16 = ['ax', 'cx', 'dx', 'bx', 'sp', 'bp', 'si', 'di']; + const n8 = ['al', 'cl', 'dl', 'bl', 'spl', 'bpl', 'sil', 'dil']; + for (let i = 0; i < 8; i++) { + t[GPR[i]] = { num: i, size: 8 }; + t[n32[i]] = { num: i, size: 4 }; + t[n16[i]] = { num: i, size: 2 }; + t[n8[i]] = { num: i, size: 1, needsRex: i >= 4 }; + } + ['ah', 'ch', 'dh', 'bh'].forEach((r, i) => { t[r] = { num: i + 4, size: 1, high: true }; }); + for (let i = 8; i < 16; i++) { + t[`r${i}`] = { num: i, size: 8 }; + t[`r${i}d`] = { num: i, size: 4 }; + t[`r${i}w`] = { num: i, size: 2 }; + t[`r${i}b`] = { num: i, size: 1 }; + } + for (let i = 0; i < 16; i++) t[`xmm${i}`] = { num: i, size: 16, vector: true }; + return t; +})(); + +export class EncodeError extends Error { } + +const reg = (name) => { + const r = REGINFO[name]; + if (!r) throw new EncodeError(`the encoder does not know register ${name}`); + return r; +}; + +/* ------------------------------------------------------------------ */ +/* Condition codes */ +/* ------------------------------------------------------------------ */ + +/** The four-bit condition number each jcc, setcc, and cmovcc carries. */ +export const CC = { + o: 0, no: 1, b: 2, c: 2, nae: 2, ae: 3, nb: 3, nc: 3, e: 4, z: 4, ne: 5, nz: 5, + be: 6, na: 6, a: 7, nbe: 7, s: 8, ns: 9, p: 10, pe: 10, np: 11, po: 11, + l: 12, nge: 12, ge: 13, nl: 13, le: 14, ng: 14, g: 15, nle: 15, +}; + +/* ------------------------------------------------------------------ */ +/* Small helpers */ +/* ------------------------------------------------------------------ */ + +const fits8 = (v) => v >= -128n && v <= 127n; +const fits32 = (v) => v >= -(1n << 31n) && v < (1n << 31n); + +function signedIn(v, width) { + const bits = BigInt(width * 8); + let x = BigInt(v) & ((1n << bits) - 1n); + if (x & (1n << (bits - 1n))) x -= 1n << bits; + return x; +} + +function le(v, n) { + const out = []; + let x = BigInt.asUintN(n * 8, BigInt(v)); + for (let k = 0; k < n; k++) { out.push(Number(x & 0xffn)); x >>= 8n; } + return out; +} + +const hex2 = (b) => b.toString(16).padStart(2, '0'); + +const GROUP1 = { add: 0, or: 1, adc: 2, sbb: 3, and: 4, sub: 5, xor: 6, cmp: 7 }; +const GROUP2 = { rol: 0, ror: 1, shl: 4, sal: 4, shr: 5, sar: 7 }; +const GROUP3 = { test: 0, not: 2, neg: 3, mul: 4, imul: 5, div: 6, idiv: 7 }; + +const MOD_TEXT = ['memory, no displacement', 'memory + 8-bit displacement', 'memory + 32-bit displacement', 'a register, not memory']; + +/* ------------------------------------------------------------------ */ +/* The builder */ +/* ------------------------------------------------------------------ */ + +/** + * Collects the parts of one instruction. Nothing is emitted until finish(), + * because the REX byte depends on registers named in ModRM and SIB, and it + * has to come before the opcode. + */ +class Builder { + constructor(instr) { + this.instr = instr; + this.prefixes = []; + this.rex = { W: 0, R: 0, X: 0, B: 0, force: false }; + this.opcode = []; + this.opcodeNote = ''; + this.modrm = null; + this.sib = null; + this.disp = null; + this.imm = null; + this.rel = null; + this.highUsed = false; + } + + prefix(byte, why) { this.prefixes.push({ byte, why }); return this; } + + operandSize(size, { defaultIs64 = false } = {}) { + if (size === 2) this.prefix(0x66, 'operand-size prefix: this instruction works on 16 bits'); + if (size === 8 && !defaultIs64) this.rex.W = 1; + return this; + } + + op(bytes, note) { this.opcode = bytes; this.opcodeNote = note; return this; } + + noteReg(name) { + const r = reg(name); + if (r.needsRex) this.rex.force = true; + if (r.high) this.highUsed = true; + return r; + } + + /** Put a register number in the low three bits of the opcode. */ + opReg(name) { + const r = this.noteReg(name); + this.opcode = [...this.opcode.slice(0, -1), this.opcode[this.opcode.length - 1] + (r.num & 7)]; + if (r.num & 8) this.rex.B = 1; + this.opRegName = name; + return this; + } + + /** + * ModRM, and SIB and displacement when memory is involved. + * regField is a register name or an opcode extension number (/0 to /7). + */ + modRM(regField, rm) { + let regNum; let regText; + if (typeof regField === 'number') { + regNum = regField; regText = `/${regField}: not a register; three more bits of opcode`; + } else { + const r = this.noteReg(regField); + regNum = r.num; regText = regField; + if (r.num & 8) this.rex.R = 1; + } + const m = { mod: 0, reg: regNum & 7, rm: 0, regText, rmText: '' }; + + if (rm.kind === 'reg') { + const r = this.noteReg(rm.name); + m.mod = 3; m.rm = r.num & 7; m.rmText = rm.name; + if (r.num & 8) this.rex.B = 1; + this.modrm = m; + return this; + } + + const base = rm.base; const index = rm.index; + const disp = BigInt(rm.disp || 0n); + if (base === 'rip') { + m.mod = 0; m.rm = 5; m.rmText = 'rip + 32-bit displacement'; + this.modrm = m; + this.disp = { bytes: le(disp, 4), value: disp, text: `displacement ${disp} from the next instruction` }; + return this; + } + const b = base ? this.noteReg(base) : null; + const x = index ? this.noteReg(index) : null; + if (x && (x.num & 8)) this.rex.X = 1; + if (b && (b.num & 8)) this.rex.B = 1; + + const needSib = !b || x || (b.num & 7) === 4; + let dispSize; + if (!b) dispSize = 4; + else if (disp === 0n && (b.num & 7) !== 5) dispSize = 0; + else if (fits8(disp)) dispSize = 1; + else dispSize = 4; + m.mod = !b ? 0 : dispSize === 0 ? 0 : dispSize === 1 ? 1 : 2; + + if (needSib) { + m.rm = 4; m.rmText = 'a SIB byte follows'; + const scaleBits = { 1: 0, 2: 1, 4: 2, 8: 3 }[rm.scale || 1]; + this.sib = { + scale: x ? scaleBits : 0, + index: x ? (x.num & 7) : 4, + base: b ? (b.num & 7) : 5, + scaleText: x ? `×${rm.scale || 1}` : 'unused', + indexText: x ? index : 'none (100 means no index)', + baseText: b ? base : 'none: a 32-bit displacement instead', + }; + } else { + m.rm = b.num & 7; m.rmText = `[${base}]`; + } + if (dispSize) { + const extra = b && (b.num & 7) === 5 && disp === 0n ? ' (rbp and r13 cannot go without one)' : ''; + this.disp = { bytes: le(disp, dispSize), value: disp, text: `displacement ${disp}, ${dispSize * 8} bits${extra}` }; + } + this.modrm = m; + return this; + } + + immediate(value, size, text) { + this.imm = { bytes: le(value, size), value: BigInt(value), text: text ?? `immediate ${signedIn(value, size)}, ${size * 8} bits` }; + return this; + } + + relative(size, target, labelName) { + this.rel = { size, target, label: labelName }; + return this; + } + + /** Assemble the parts, now that every register has been seen. */ + finish(addr) { + const parts = []; + for (const p of this.prefixes) parts.push({ kind: 'prefix', bytes: [p.byte], text: p.why }); + const { W, R, X, B } = this.rex; + const needRex = W || R || X || B || this.rex.force; + if (needRex && this.highUsed) throw new EncodeError('ah, bh, ch, and dh cannot be encoded in an instruction that needs a REX prefix'); + if (needRex) { + const byte = 0x40 | (W << 3) | (R << 2) | (X << 1) | B; + const why = []; + if (W) why.push('W: 64-bit operands'); + if (R) why.push('R: ModRM.reg reaches r8–r15'); + if (X) why.push('X: SIB.index reaches r8–r15'); + if (B) why.push('B: the base or r/m register reaches r8–r15'); + if (!why.length) why.push('no bits set: its presence alone selects spl, bpl, sil, or dil'); + parts.push({ kind: 'rex', bytes: [byte], text: `REX 0100WRXB, ${why.join('; ')}`, fields: { W, R, X, B } }); + } + parts.push({ kind: 'opcode', bytes: this.opcode, text: this.opcodeNote }); + if (this.modrm) { + const m = this.modrm; + const byte = (m.mod << 6) | (m.reg << 3) | m.rm; + parts.push({ + kind: 'modrm', bytes: [byte], + text: `ModRM mod=${m.mod.toString(2).padStart(2, '0')} reg=${m.reg.toString(2).padStart(3, '0')} r/m=${m.rm.toString(2).padStart(3, '0')}`, + fields: { mod: m.mod, reg: m.reg, rm: m.rm, modText: MOD_TEXT[m.mod], regText: m.regText, rmText: m.rmText }, + }); + } + if (this.sib) { + const s = this.sib; + const byte = (s.scale << 6) | (s.index << 3) | s.base; + parts.push({ + kind: 'sib', bytes: [byte], + text: `SIB scale=${s.scale.toString(2).padStart(2, '0')} index=${s.index.toString(2).padStart(3, '0')} base=${s.base.toString(2).padStart(3, '0')}`, + fields: { ...s }, + }); + } + if (this.disp) parts.push({ kind: 'disp', bytes: this.disp.bytes, text: this.disp.text }); + if (this.rel) { + const len = parts.reduce((n, p) => n + p.bytes.length, 0) + this.rel.size + (this.imm ? this.imm.bytes.length : 0); + const offset = BigInt(this.rel.target - (addr + len)); + if (this.rel.size === 1 && !fits8(offset)) throw new EncodeError('short jump out of range'); + parts.push({ + kind: 'rel', bytes: le(offset, this.rel.size), value: offset, + text: `${this.rel.size * 8}-bit offset ${offset >= 0n ? '+' : ''}${offset}, counted from the end of this instruction, lands on ${this.rel.label} at 0x${this.rel.target.toString(16)}`, + }); + } + if (this.imm) parts.push({ kind: 'imm', bytes: this.imm.bytes, text: this.imm.text }); + const bytes = parts.flatMap((p) => p.bytes); + if (bytes.length > 15) throw new EncodeError('longer than the 15-byte limit on an x86 instruction'); + return { bytes, parts, hex: bytes.map(hex2).join(' ') }; + } +} + +/* ------------------------------------------------------------------ */ +/* Instruction forms */ +/* ------------------------------------------------------------------ */ + +const isAcc = (o) => o.kind === 'reg' && reg(o.name).num === 0 && !reg(o.name).high && !reg(o.name).vector; +const widthOf = (ops) => { + for (const o of ops) { if (o.kind === 'reg') return reg(o.name).size; if (o.kind === 'mem' && o.size) return o.size; } + return 8; +}; + +function encodeGroup1(b, op, [d, s]) { + const ext = GROUP1[op]; + const w = widthOf([d, s]); + b.operandSize(w); + if (s.kind === 'imm') { + const v = signedIn(s.value, w); + if (w === 1) { + if (isAcc(d)) return b.op([ext * 8 + 4], `${op} al, imm8: the short form for the accumulator`).immediate(v, 1); + return b.op([0x80], `${op} r/m8, imm8 (group 1; ModRM.reg picks ${op})`).modRM(ext, d).immediate(v, 1); + } + if (fits8(v)) return b.op([0x83], `${op} r/m, imm8, sign-extended (group 1; ModRM.reg picks ${op})`).modRM(ext, d).immediate(v, 1); + const isz = w === 2 ? 2 : 4; + if (isAcc(d)) return b.op([ext * 8 + 5], `${op} ${d.name}, imm: the short form for the accumulator`).immediate(v, isz); + return b.op([0x81], `${op} r/m, imm${isz * 8} (group 1; ModRM.reg picks ${op})`).modRM(ext, d).immediate(v, isz); + } + if (s.kind === 'mem') return b.op([ext * 8 + (w === 1 ? 2 : 3)], `${op} reg, r/m: destination in ModRM.reg`).modRM(d.name, s); + return b.op([ext * 8 + (w === 1 ? 0 : 1)], `${op} r/m, reg: source in ModRM.reg`).modRM(s.name, d); +} + +function encodeMov(b, op, [d, s]) { + const w = widthOf([d, s]); + if (s.kind === 'imm') { + const v = signedIn(s.value, w); + if (d.kind === 'reg') { + if (w === 8 && (op === 'movabs' || !fits32(signedIn(s.value, 8)))) { + return b.operandSize(8).op([0xb8], 'mov r64, imm64: register number in the low bits of the opcode').opReg(d.name).immediate(s.value, 8); + } + if (w === 8) return b.operandSize(8).op([0xc7], 'mov r/m64, imm32, sign-extended to 64 bits').modRM(0, d).immediate(v, 4); + b.operandSize(w); + return b.op([w === 1 ? 0xb0 : 0xb8], `mov r${w * 8}, imm: register number in the low bits of the opcode`).opReg(d.name).immediate(v, w === 8 ? 4 : w); + } + b.operandSize(w); + return b.op([w === 1 ? 0xc6 : 0xc7], `mov r/m, imm${Math.min(w, 4) * 8}${w === 8 ? ', sign-extended' : ''}`).modRM(0, d).immediate(v, Math.min(w, 4)); + } + b.operandSize(w); + if (s.kind === 'mem') return b.op([w === 1 ? 0x8a : 0x8b], 'mov reg, r/m: a load').modRM(d.name, s); + return b.op([w === 1 ? 0x88 : 0x89], d.kind === 'mem' ? 'mov r/m, reg: a store' : 'mov r/m, reg').modRM(s.name, d); +} + +function encodeUnary(b, op, [d], ext, note) { + const w = widthOf([d]); + b.operandSize(w); + return b.op([w === 1 ? 0xf6 : 0xf7], note ?? `${op} r/m (group 3; ModRM.reg picks ${op})`).modRM(ext, d); +} + +function encodeShift(b, op, ops) { + const [d, n] = ops; + const ext = GROUP2[op]; + const w = widthOf([d]); + b.operandSize(w); + const small = w === 1; + if (!n || (n.kind === 'imm' && n.value === 1n)) return b.op([small ? 0xd0 : 0xd1], `${op} r/m, 1 (group 2; ModRM.reg picks ${op})`).modRM(ext, d); + if (n.kind === 'reg') return b.op([small ? 0xd2 : 0xd3], `${op} r/m, cl (group 2; ModRM.reg picks ${op})`).modRM(ext, d); + return b.op([small ? 0xc0 : 0xc1], `${op} r/m, imm8 (group 2; ModRM.reg picks ${op})`).modRM(ext, d).immediate(n.value, 1); +} + +function encodeSSE(b, prefix, opcode, regName, rm, note, rexW = false) { + if (prefix) b.prefix(prefix, prefix === 0x66 ? 'mandatory prefix 66: part of the opcode, not a size change' : `mandatory prefix ${hex2(prefix)}: part of the opcode`); + if (rexW) b.rex.W = 1; + return b.op([0x0f, opcode], note).modRM(regName, rm); +} + +/** + * Encode one instruction at a given address. + * labelAddr(name) -> the address a label stands for + * near -> true to use a 32-bit offset for a jump + */ +export function encodeInstruction(instr, addr = 0, labelAddr = () => 0, near = false) { + const b = new Builder(instr); + const op = instr.op; + const ops = instr.operands; + const [d, s] = ops; + + if (op in GROUP1) encodeGroup1(b, op, ops); + else if (op === 'mov' || op === 'movabs') encodeMov(b, op, ops); + else if (op === 'test') { + const w = widthOf(ops); + b.operandSize(w); + if (s.kind === 'imm') { + const v = signedIn(s.value, w); + if (isAcc(d)) b.op([w === 1 ? 0xa8 : 0xa9], 'test accumulator, imm: the short form').immediate(v, Math.min(w, 4)); + else b.op([w === 1 ? 0xf6 : 0xf7], 'test r/m, imm (group 3 /0)').modRM(0, d).immediate(v, Math.min(w, 4)); + } else { + const [rmOp, regOp] = d.kind === 'mem' ? [d, s] : s.kind === 'mem' ? [s, d] : [d, s]; + b.op([w === 1 ? 0x84 : 0x85], 'test r/m, reg').modRM(regOp.name, rmOp); + } + } else if (op === 'lea') { + b.operandSize(reg(d.name).size); + b.op([0x8d], 'lea reg, [address]: the address itself goes to ModRM.reg').modRM(d.name, s); + } else if (op === 'movzx' || op === 'movsx') { + const dw = reg(d.name).size; + const sw = s.kind === 'reg' ? reg(s.name).size : s.size; + b.operandSize(dw); + const code = (op === 'movzx' ? 0xb6 : 0xbe) + (sw === 2 ? 1 : 0); + b.op([0x0f, code], `${op} reg, r/m${sw * 8}: a two-byte opcode, 0f then ${hex2(code)}`).modRM(d.name, s); + } else if (op === 'movsxd') { + b.operandSize(8).op([0x63], 'movsxd r64, r/m32').modRM(d.name, s); + } else if (op === 'inc' || op === 'dec') { + const w = widthOf(ops); + b.operandSize(w); + b.op([w === 1 ? 0xfe : 0xff], `${op} r/m (ModRM.reg /${op === 'inc' ? 0 : 1} picks ${op})`).modRM(op === 'inc' ? 0 : 1, d); + } else if (op === 'neg' || op === 'not' || op === 'mul' || op === 'div' || op === 'idiv' || (op === 'imul' && ops.length === 1)) { + encodeUnary(b, op, ops, GROUP3[op]); + } else if (op === 'imul') { + const w = reg(d.name).size; + b.operandSize(w); + if (ops.length === 2) b.op([0x0f, 0xaf], 'imul reg, r/m: a two-byte opcode').modRM(d.name, s); + else { + const v = signedIn(ops[2].value, w); + if (fits8(v)) b.op([0x6b], 'imul reg, r/m, imm8').modRM(d.name, s).immediate(v, 1); + else b.op([0x69], `imul reg, r/m, imm${w === 2 ? 16 : 32}`).modRM(d.name, s).immediate(v, w === 2 ? 2 : 4); + } + } else if (op in GROUP2) encodeShift(b, op, ops); + else if (op === 'xchg') { + const w = widthOf(ops); + b.operandSize(w); + const acc = [d, s].find(isAcc); + const other = acc === d ? s : d; + if (w === 8 && isAcc(d) && isAcc(s)) { + // GNU as drops REX.W here: 48 90 and 90 do the same nothing. + b.rex.W = 0; + b.op([0x90], 'xchg rax, rax changes nothing and is written as a plain nop'); + } else if (w > 1 && acc && other.kind === 'reg' && !(isAcc(other) && w === 4)) { + b.op([0x90], 'xchg with the accumulator: register number in the low bits').opReg(other.name); + } else { + const [rmOp, regOp] = s.kind === 'mem' ? [s, d] : [d, s]; + b.op([w === 1 ? 0x86 : 0x87], 'xchg r/m, reg').modRM(regOp.name, rmOp); + } + } else if (op === 'bt') { + const w = reg(d.name).size; + b.operandSize(w); + if (s.kind === 'imm') b.op([0x0f, 0xba], 'bt r/m, imm8 (ModRM.reg /4 picks bt)').modRM(4, d).immediate(s.value, 1); + else b.op([0x0f, 0xa3], 'bt r/m, reg').modRM(s.name, d); + } else if (op === 'push' || op === 'pop') { + if (d.kind === 'reg') b.op([op === 'push' ? 0x50 : 0x58], `${op} r64: register number in the low bits; 64 bits is the default and needs no REX.W`).opReg(d.name); + else if (d.kind === 'imm') { + const v = signedIn(d.value, 8); + if (fits8(v)) b.op([0x6a], 'push imm8, sign-extended to 64 bits').immediate(v, 1); + else b.op([0x68], 'push imm32, sign-extended to 64 bits').immediate(v, 4); + } else if (op === 'push') b.op([0xff], 'push r/m64 (ModRM.reg /6 picks push)').modRM(6, d); + else b.op([0x8f], 'pop r/m64 (ModRM.reg /0)').modRM(0, d); + } else if (op === 'jmp' || op === 'call' || /^j/.test(op)) { + const target = labelAddr(d.name); + if (op === 'call') b.op([0xe8], 'call rel32').relative(4, target, d.name); + else if (op === 'jmp') { + if (near) b.op([0xe9], 'jmp rel32: the long form').relative(4, target, d.name); + else b.op([0xeb], 'jmp rel8: the short form, for targets within 128 bytes').relative(1, target, d.name); + } else { + const cc = CC[op.slice(1)]; + if (near) b.op([0x0f, 0x80 + cc], `j${op.slice(1)} rel32: 0f 8x, where x = ${cc} is the condition`).relative(4, target, d.name); + else b.op([0x70 + cc], `j${op.slice(1)} rel8: 7x, where x = ${cc} is the condition`).relative(1, target, d.name); + } + } else if (op.startsWith('set')) { + const cc = CC[op.slice(3)]; + b.op([0x0f, 0x90 + cc], `set${op.slice(3)} r/m8: 0f 9x, where x = ${cc} is the condition`).modRM(0, d); + } else if (op.startsWith('cmov')) { + const cc = CC[op.slice(4)]; + b.operandSize(reg(d.name).size); + b.op([0x0f, 0x40 + cc], `cmov${op.slice(4)} reg, r/m: 0f 4x, where x = ${cc} is the condition`).modRM(d.name, s); + } else if (op === 'popcnt' || op === 'tzcnt' || op === 'lzcnt') { + const w = reg(d.name).size; + if (w === 2) b.prefix(0x66, 'operand-size prefix: 16 bits'); + b.prefix(0xf3, `mandatory prefix f3: turns ${op === 'popcnt' ? 'this opcode' : op === 'tzcnt' ? 'bsf' : 'bsr'} into ${op}`); + if (w === 8) b.rex.W = 1; + b.op([0x0f, { popcnt: 0xb8, tzcnt: 0xbc, lzcnt: 0xbd }[op]], `${op} reg, r/m`).modRM(d.name, s); + } else if (op === 'bsf' || op === 'bsr') { + b.operandSize(reg(d.name).size); + b.op([0x0f, op === 'bsf' ? 0xbc : 0xbd], `${op} reg, r/m`).modRM(d.name, s); + } else if (op === 'movdqu' || op === 'movdqa') { + const pre = op === 'movdqu' ? 0xf3 : 0x66; + if (d.kind === 'mem') encodeSSE(b, pre, 0x7f, s.name, d, `${op} m128, xmm: a store`); + else encodeSSE(b, pre, 0x6f, d.name, s, `${op} xmm, xmm/m128`); + } else if (op === 'movd') { + if (reg(d.name || 'rax').vector && d.kind === 'reg') encodeSSE(b, 0x66, 0x6e, d.name, s, 'movd xmm, r/m32'); + else encodeSSE(b, 0x66, 0x7e, s.name, d, 'movd r/m32, xmm'); + } else if (op === 'movq') { + const dv = d.kind === 'reg' && reg(d.name).vector; + const sv = s.kind === 'reg' && reg(s.name).vector; + if (dv && sv) encodeSSE(b, 0xf3, 0x7e, d.name, s, 'movq xmm, xmm/m64'); + else if (dv && s.kind === 'mem') encodeSSE(b, 0xf3, 0x7e, d.name, s, 'movq xmm, m64'); + else if (sv && d.kind === 'mem') encodeSSE(b, 0x66, 0xd6, s.name, d, 'movq m64, xmm'); + else if (dv) encodeSSE(b, 0x66, 0x6e, d.name, s, 'movq xmm, r64: movd with REX.W', true); + else encodeSSE(b, 0x66, 0x7e, s.name, d, 'movq r64, xmm: movd with REX.W', true); + } else if (op === 'pmovmskb') { + encodeSSE(b, 0x66, 0xd7, d.name, s, 'pmovmskb reg, xmm'); + } else if (['pcmpeqb', 'pand', 'por', 'pxor', 'psubb', 'paddb'].includes(op)) { + const code = { pcmpeqb: 0x74, pand: 0xdb, por: 0xeb, pxor: 0xef, psubb: 0xf8, paddb: 0xfc }[op]; + encodeSSE(b, 0x66, code, d.name, s, `${op} xmm, xmm/m128`); + } else if (op === 'nop') { + if (!ops.length) b.op([0x90], 'nop: officially xchg eax, eax with no effect'); + else { + const w = widthOf(ops); + b.operandSize(w); + b.op([0x0f, 0x1f], 'nop r/m: a multi-byte no-op compilers use as padding').modRM(0, d); + } + } else { + const fixed = { + ret: [[0xc3], 'ret: pop the return address and go there'], + leave: [[0xc9], 'leave'], + syscall: [[0x0f, 0x05], 'syscall: enter the kernel'], + cqo: [[0x99], 'cqo: cdq with REX.W'], + cdq: [[0x99], 'cdq'], + cdqe: [[0x98], 'cdqe: cwde with REX.W'], + stc: [[0xf9], 'stc'], + clc: [[0xf8], 'clc'], + cmc: [[0xf5], 'cmc'], + }[op]; + if (!fixed) throw new EncodeError(`the encoder has no form for ${op}`); + if (op === 'cqo' || op === 'cdqe') b.rex.W = 1; + b.op(fixed[0], fixed[1]); + } + return b.finish(addr); +} + +/* ------------------------------------------------------------------ */ +/* Layout */ +/* ------------------------------------------------------------------ */ + +/** + * Give every instruction an address and its bytes. + * + * Jumps start short, with an 8-bit offset. Any whose target turns out to be + * more than 128 bytes away is widened to a 32-bit offset, which moves every + * instruction after it, which can push another jump out of range, so the + * process repeats until nothing changes. Real assemblers call this branch + * relaxation. It always finishes, because jumps only ever grow. + * + * layout = { + * ok, base, end, + * addr[i], len[i], bytes[i], parts[i], hex[i], + * byAddr: Map(address -> instruction index), + * labelAddr: { name: address }, + * errors: [ { index, message } ], + * relaxed: number of jumps that needed the long form + * } + */ +export function layoutProgram(program, base = 0x401000) { + const n = program.instrs.length; + const near = new Array(n).fill(false); + let addr = []; + let encoded = []; + const errors = []; + + for (let pass = 0; pass < 64; pass++) { + // lengths from the current guesses + addr = []; encoded = []; + let pc = base; + const guesses = program.instrs.map((ins, i) => { + try { + return encodeInstruction(ins, pc, () => pc, near[i]).bytes.length; + } catch (e) { + return 0; + } + }); + for (let i = 0; i < n; i++) { addr.push(pc); pc += guesses[i] || 1; } + const end = pc; + const labelAddr = (name) => { + const idx = program.labels[name]; + return idx >= n ? end : addr[idx]; + }; + let changed = false; + errors.length = 0; + for (let i = 0; i < n; i++) { + try { + encoded[i] = encodeInstruction(program.instrs[i], addr[i], labelAddr, near[i]); + } catch (e) { + if (/short jump out of range/.test(e.message) && !near[i]) { near[i] = true; changed = true; continue; } + errors.push({ index: i, message: e.message }); + encoded[i] = null; + } + } + if (!changed) { + const len = encoded.map((e) => (e ? e.bytes.length : 1)); + const byAddr = new Map(addr.map((a, i) => [a, i])); + const labels = {}; + for (const name of Object.keys(program.labels)) labels[name] = labelAddr(name); + return { + ok: errors.length === 0, + base, + end, + addr, + len, + bytes: encoded.map((e) => (e ? e.bytes : [])), + parts: encoded.map((e) => (e ? e.parts : [])), + hex: encoded.map((e) => (e ? e.hex : '??')), + byAddr, + labelAddr: labels, + errors, + relaxed: near.filter(Boolean).length, + totalBytes: end - base, + }; + } + } + throw new EncodeError('branch relaxation did not settle'); +} + +/** Every encoded byte of a laid-out program, in order, for a hex dump. */ +export function programBytes(layout) { + return layout.bytes.flat(); +} diff --git a/src/engines/asm86/hwviews.js b/src/engines/asm86/hwviews.js new file mode 100644 index 0000000..b975798 --- /dev/null +++ b/src/engines/asm86/hwviews.js @@ -0,0 +1,1347 @@ +/** + * asm86/hwviews.js + * + * Views of the machine underneath the instruction set: machine code, the + * arithmetic unit at bit level, the stack, caches, virtual memory, the + * five-stage pipeline, the out-of-order core, the operating system boundary, + * and a whole-chip diagram. + * + * Same contract as views.js: each factory takes (trace, options) and returns + * { node, update(frame) }. Views that need timing ask hardwareFor(trace), + * which runs the models in machine.js once per trace and machine. + * + * Color roles are the ones css/widgets/kit.css defines, extended in one direction: + * in the cache and memory views, green is a hit and red is a miss, which + * reads the same way as a match and a mismatch. + */ + +import { + OPS, REGS, formatHex, formatSigned, formatBinary, byteChar, readByte, regValue, + MASK, SIGNBIT, toSigned, namesOf, REG64, +} from './interp.js'; +import { el, sv, add, clear, panel, commas, legend, plural } from './dom.js'; +import { hardwareFor } from './machine.js'; +import { STAGES, stageOccupancy } from './pipeline.js'; +import { coreAt } from './ooo.js'; +import { PORTS, PORT_ORDER, UOP_KINDS, depName, isFlagGroup } from './uops.js'; +import { LEVEL_NAMES } from './paging.js'; + +const lastIndex = (frame) => frame.t - 1; + +/** The step a single-instruction view should describe. */ +function focus(frame, which = 'auto') { + if (which === 'next') return frame.next ? { instr: frame.next, index: frame.t, ran: false } : null; + if (frame.last) return { instr: frame.last.instr, index: frame.t - 1, ran: true, step: frame.last }; + if (which === 'auto' && frame.next) return { instr: frame.next, index: frame.t, ran: false }; + return null; +} + +const machineNote = (hw) => el('p', { class: 'a86-hw-machine', text: `Machine: ${hw.config.name}${hw.config.description ? ` (${hw.config.description})` : ''}.` }); + +const kib = (bytes) => (bytes >= 1048576 ? `${bytes / 1048576} MiB` : bytes >= 1024 ? `${bytes / 1024} KiB` : `${bytes} B`); + +/* ------------------------------------------------------------------ */ +/* 1. Machine code */ +/* ------------------------------------------------------------------ */ + +const PART_NAME = { prefix: 'prefix', rex: 'REX', opcode: 'opcode', modrm: 'ModRM', sib: 'SIB', disp: 'displacement', imm: 'immediate', rel: 'offset' }; + +function bitCells(byte, groups) { + // groups: [[label, width, meaning], ...] from the top bit down + const row = el('div', { class: 'a86-enc-bits' }); + let bit = 7; + for (const [label, width, meaning] of groups) { + const cells = []; + for (let k = 0; k < width; k++, bit--) cells.push(el('span', { class: 'a86-enc-bit', text: String((byte >> bit) & 1) })); + add(row, el('div', { class: 'a86-enc-field' }, [ + el('div', { class: 'a86-enc-field-bits' }, cells), + el('div', { class: 'a86-enc-field-name', text: label }), + meaning ? el('div', { class: 'a86-enc-field-meaning', text: meaning }) : null, + ])); + } + return row; +} + +/** + * The bytes of one instruction, colored by what each byte is for, with the + * REX, ModRM, and SIB bytes broken into their bit fields. + * + * options: { which: 'auto' | 'last' | 'next', fields: true } + */ +export function encodingView(trace, options = {}) { + const opts = { which: 'auto', fields: true, ...options }; + const body = el('div', { class: 'a86-enc' }); + const layout = trace.layout; + + function update(frame) { + clear(body); + if (!layout?.ok) { + add(body, el('p', { class: 'a86-error', text: `This program could not be encoded: ${layout?.error || layout?.errors?.map((e) => `${trace.program.instrs[e.index].text}: ${e.message}`).join('; ')}` })); + return; + } + const f = focus(frame, opts.which); + if (!f) { add(body, el('p', { class: 'a86-muted', text: 'No instruction to show.' })); return; } + const i = f.instr.index; + const parts = layout.parts[i]; + add(body, el('p', { class: 'a86-enc-head' }, [ + el('code', { class: 'a86-enc-addr', text: `0x${layout.addr[i].toString(16)}` }), + el('code', { class: 'a86-enc-text', text: f.instr.text }), + el('span', { class: 'a86-muted', text: `${plural(layout.len[i], 'byte')}${f.ran ? '' : ', runs next'}` }), + ])); + + const strip = el('div', { class: 'a86-enc-strip' }); + for (const p of parts) { + add(strip, el('div', { class: `a86-enc-part is-${p.kind}`, title: p.text }, [ + el('div', { class: 'a86-enc-bytes' }, p.bytes.map((b) => el('code', { class: 'a86-enc-byte', text: b.toString(16).padStart(2, '0') }))), + el('div', { class: 'a86-enc-kind', text: PART_NAME[p.kind] }), + ])); + } + add(body, strip); + + if (!opts.fields) return; + const list = el('div', { class: 'a86-enc-details' }); + for (const p of parts) { + const item = el('div', { class: `a86-enc-detail is-${p.kind}` }, [ + el('div', { class: 'a86-enc-detail-head' }, [ + el('span', { class: 'a86-enc-detail-name', text: PART_NAME[p.kind] }), + el('code', { class: 'a86-enc-detail-bytes', text: p.bytes.map((b) => b.toString(16).padStart(2, '0')).join(' ') }), + ]), + el('div', { class: 'a86-enc-detail-text', text: p.text }), + ]); + if (p.kind === 'rex') { + add(item, bitCells(p.bytes[0], [['0100', 4, 'fixed'], ['W', 1, p.fields.W ? '64-bit' : ''], ['R', 1], ['X', 1], ['B', 1]])); + } else if (p.kind === 'modrm') { + const fl = p.fields; + add(item, bitCells(p.bytes[0], [['mod', 2, fl.modText], ['reg', 3, fl.regText], ['r/m', 3, fl.rmText]])); + } else if (p.kind === 'sib') { + const fl = p.fields; + add(item, bitCells(p.bytes[0], [['scale', 2, fl.scaleText], ['index', 3, fl.indexText], ['base', 3, fl.baseText]])); + } else if (p.kind === 'disp' || p.kind === 'imm' || p.kind === 'rel') { + if (p.bytes.length > 1) add(item, el('div', { class: 'a86-enc-detail-text a86-muted', text: 'Little-endian: the lowest byte comes first, and the number reads backwards in the listing.' })); + } + add(list, item); + } + add(body, list); + } + + return { node: panel(opts.title ?? 'Machine code', body, 'a86-enc-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 2. The arithmetic unit, bit by bit */ +/* ------------------------------------------------------------------ */ + +function bitsOf(v, n) { + const out = []; + for (let k = 0; k < n; k++) out.push(Number((BigInt(v) >> BigInt(k)) & 1n)); + return out; // index 0 is the lowest bit +} + +/** One row of bits, drawn most significant first, with per-bit classes. */ +function bitRow(label, bits, { classes = {}, value = null, note = '', extraLeft = null, rowClass = '' } = {}) { + const n = bits.length; + const cells = []; + cells.push(el('span', { class: 'a86-bits-cell a86-bits-extra', text: extraLeft ?? '' })); + for (let k = n - 1; k >= 0; k--) { + const c = el('span', { class: `a86-bits-cell${k % 4 === 3 && k !== n - 1 ? ' is-nibble' : ''}${classes[k] ? ` ${classes[k]}` : ''}`, text: bits[k] == null ? '' : String(bits[k]) }); + cells.push(c); + } + return el('div', { class: `a86-bits-row ${rowClass}` }, [ + el('span', { class: 'a86-bits-label', text: label }), + el('span', { class: 'a86-bits-cells' }, cells), + el('span', { class: 'a86-bits-value', text: value ?? '' }), + note ? el('span', { class: 'a86-bits-note', text: note }) : null, + ]); +} + +const WIDE = { 1: 'ax', 2: 'dx:ax', 4: 'edx:eax', 8: 'rdx:rax' }; +const LOW = { 1: 'al', 2: 'ax', 4: 'eax', 8: 'rax' }; +const HIGH = { 1: 'ah', 2: 'dx', 4: 'edx', 8: 'rdx' }; + +const readings = (v, w) => `${(BigInt(v) & MASK[w]).toString()}${toSigned(v, w) < 0n ? ` / ${toSigned(v, w)}` : ''}`; + +function flagLine(name, value, why) { + return el('li', { class: 'a86-bits-flag' }, [ + el('code', { class: `a86-bits-flagname${value ? ' is-on' : ''}`, text: `${name}=${value ? 1 : 0}` }), + ` ${why}`, + ]); +} + +/** + * The last arithmetic instruction at the level of wires: each operand's + * bits, the carry rippling between columns, the result, and where every + * flag came from. Subtraction is drawn the way the hardware does it, as + * addition of the inverted operand plus one. + * + * options: { bits: 'auto' | 8 | 16 | 32 | 64 } how many low bits to draw + */ +export function bitsView(trace, options = {}) { + const opts = { bits: 'auto', ...options }; + const body = el('div', { class: 'a86-bits' }); + + function update(frame) { + clear(body); + const s = frame.last; + const alu = s?.alu; + if (!alu) { + add(body, el('p', { class: 'a86-muted', text: s ? `${s.instr.op} did no arithmetic, and there is nothing to draw at bit level.` : 'Step forward to watch the arithmetic unit.' })); + return; + } + const w = alu.width; + const n = opts.bits === 'auto' ? w * 8 : Math.min(w * 8, opts.bits); + const cut = n < w * 8 ? ` (low ${n} of ${w * 8} bits)` : ''; + const f = frame.state.flags; + const op = s.instr.op; + const rows = el('div', { class: 'a86-bits-grid' }); + const flags = el('ul', { class: 'a86-bits-flags' }); + const title = el('p', { class: 'a86-bits-title' }, [el('code', { text: s.instr.text }), `: ${w * 8}-bit ${alu.kind === 'sub' ? 'subtraction' : alu.kind === 'add' ? 'addition' : alu.kind}${cut}`]); + + if (alu.kind === 'add' || alu.kind === 'sub') { + const sub = alu.kind === 'sub'; + const a = bitsOf(alu.a, w * 8); + const b = bitsOf(alu.b, w * 8); + const bb = sub ? b.map((x) => 1 - x) : b; + const cin = sub ? 1 - Number(alu.cin || 0n) : Number(alu.cin || 0n); + const carry = [cin]; + const sum = []; + for (let k = 0; k < w * 8; k++) { + const x = a[k] + bb[k] + carry[k]; + sum.push(x & 1); carry.push(x >> 1); + } + const top = w * 8 - 1; + const cout = carry[w * 8]; const cinTop = carry[top]; + const carryCls = {}; for (let k = 0; k < n; k++) if (carry[k]) carryCls[k] = 'is-carry'; + add(rows, bitRow('carries in', carry.slice(0, n), { classes: carryCls, rowClass: 'is-carries', extraLeft: String(cout), note: 'the left cell is the carry out of the top bit' })); + add(rows, bitRow(s.instr.operands[0]?.source ?? 'a', a.slice(0, n), { value: readings(alu.a, w) })); + if (sub) { + add(rows, bitRow(`${s.instr.operands[1]?.source ?? 'b'}`, b.slice(0, n), { value: readings(alu.b, w), rowClass: 'is-muted' })); + add(rows, bitRow('inverted', bb.slice(0, n), { value: `+ ${cin} carried in`, note: 'the adder subtracts by adding the inverted bits and one' })); + } else { + add(rows, bitRow(s.instr.operands[1]?.source ?? (op === 'inc' ? '1' : 'b'), b.slice(0, n), { value: `${readings(alu.b, w)}${cin ? ' + carry' : ''}` })); + } + const resCls = {}; for (let k = 0; k < n; k++) if (sum[k]) resCls[k] = 'is-one'; + add(rows, bitRow(alu.discarded ? 'result (discarded)' : 'result', sum.slice(0, n), { classes: { ...resCls, [top]: `${resCls[top] || ''} is-top` }, value: readings(alu.result, w), rowClass: 'is-result' })); + add(flags, flagLine('ZF', f.ZF, f.ZF ? 'every result bit is 0' : 'at least one result bit is 1')); + add(flags, flagLine('SF', f.SF, `copies bit ${top} of the result, the sign bit`)); + if (op === 'inc' || op === 'dec') add(flags, flagLine('CF', f.CF, `${op} leaves CF exactly as it was`)); + else add(flags, flagLine('CF', f.CF, sub ? `the adder ${cout ? 'carried' : 'did not carry'} out of bit ${top}; no carry out means the subtraction borrowed, and CF is the inverse` : `the carry out of bit ${top}: unsigned overflow`)); + add(flags, flagLine('OF', f.OF, `the carry into bit ${top} (${cinTop}) ${cinTop === cout ? 'matches' : 'differs from'} the carry out of it (${cout}); a mismatch means the signed answer does not fit`)); + } else if (['and', 'or', 'xor', 'not'].includes(alu.kind)) { + const a = bitsOf(alu.a, w * 8); + add(rows, bitRow(s.instr.operands[0]?.source ?? 'a', a.slice(0, n), { value: readings(alu.a, w) })); + if (alu.b !== null) add(rows, bitRow(s.instr.operands[1]?.source ?? 'b', bitsOf(alu.b, w * 8).slice(0, n), { value: readings(alu.b, w) })); + const r = bitsOf(alu.result, w * 8); + const cls = {}; for (let k = 0; k < n; k++) if (r[k]) cls[k] = 'is-one'; + add(rows, bitRow(alu.discarded ? 'result (discarded)' : 'result', r.slice(0, n), { classes: cls, value: readings(alu.result, w), rowClass: 'is-result' })); + add(body, el('p', { class: 'a86-muted', text: `${alu.kind.toUpperCase()} works on each column separately. No carries pass between columns, and every bit is independent of its neighbors.` })); + if (alu.kind !== 'not') { + add(flags, flagLine('ZF', f.ZF, f.ZF ? 'every result bit is 0' : 'at least one result bit is 1')); + add(flags, flagLine('SF', f.SF, `copies the top bit of the result`)); + add(flags, flagLine('CF', f.CF, 'always cleared by a logical operation')); + add(flags, flagLine('OF', f.OF, 'always cleared by a logical operation')); + } else add(flags, el('li', { text: 'not leaves every flag alone.' })); + } else if (['shl', 'shr', 'sar', 'rol', 'ror'].includes(alu.kind)) { + const a = bitsOf(alu.a, w * 8); const r = bitsOf(alu.result, w * 8); + const count = alu.count; + const lost = {}; + if (count) { + const outBit = alu.kind === 'shl' || alu.kind === 'rol' ? w * 8 - count : count - 1; + if (outBit >= 0 && outBit < w * 8 && ['shl', 'shr', 'sar'].includes(alu.kind)) lost[outBit] = 'is-out'; + } + add(rows, bitRow('before', a.slice(0, n), { classes: lost, value: readings(alu.a, w) })); + const cls = {}; for (let k = 0; k < n; k++) if (r[k]) cls[k] = 'is-one'; + add(rows, bitRow(`${alu.kind} ${count}`, r.slice(0, n), { classes: cls, value: readings(alu.result, w), rowClass: 'is-result' })); + const fill = { shl: 'zeros come in at the bottom', shr: 'zeros come in at the top', sar: 'copies of the sign bit come in at the top, and a negative number stays negative', rol: 'bits leaving the top come back in at the bottom', ror: 'bits leaving the bottom come back in at the top' }[alu.kind]; + add(body, el('p', { class: 'a86-muted', text: count ? `Every bit moves ${count} place${count > 1 ? 's' : ''}; ${fill}.` : 'A count of zero moves nothing and leaves every flag alone.' })); + if (count) add(flags, flagLine('CF', f.CF, alu.kind.startsWith('ro') ? 'the last bit to wrap around' : 'the last bit pushed out of the register (outlined above)')); + if (count && !alu.kind.startsWith('ro')) { + add(flags, flagLine('ZF', f.ZF, f.ZF ? 'every result bit is 0' : 'at least one result bit is 1')); + add(flags, flagLine('SF', f.SF, 'copies the top bit of the result')); + } + if (count === 1) add(flags, flagLine('OF', f.OF, 'defined only for one-place shifts: whether the sign bit changed')); + } else if (alu.kind === 'bt') { + const a = bitsOf(alu.a, w * 8); + add(rows, bitRow(s.instr.operands[0].source, a.slice(0, n), { classes: { [alu.count]: 'is-out' }, value: readings(alu.a, w) })); + add(flags, flagLine('CF', f.CF, `a copy of bit ${alu.count}`)); + } else if (['popcnt', 'tzcnt', 'lzcnt', 'bsf', 'bsr'].includes(alu.kind)) { + const a = bitsOf(alu.a, w * 8); + const cls = {}; + if (alu.kind === 'popcnt') a.forEach((x, k) => { if (x) cls[k] = 'is-one'; }); + else if (alu.lowest >= 0 && (alu.kind === 'tzcnt' || alu.kind === 'bsf')) cls[alu.lowest] = 'is-out'; + else if (alu.highest >= 0) cls[alu.highest] = 'is-out'; + add(rows, bitRow(s.instr.operands[1].source, a.slice(0, n), { classes: cls, value: readings(alu.a, w) })); + const what = { + popcnt: `${alu.count} bits are set`, + tzcnt: alu.lowest < 0 ? `no bit is set, which makes the answer the width, ${w * 8}` : `${alu.lowest} zeros sit below the lowest set bit`, + lzcnt: alu.highest < 0 ? `no bit is set, which makes the answer the width, ${w * 8}` : `${w * 8 - 1 - alu.highest} zeros sit above the highest set bit`, + bsf: alu.lowest < 0 ? 'no bit is set: the destination is left alone and ZF is set' : `the lowest set bit is bit ${alu.lowest}`, + bsr: alu.highest < 0 ? 'no bit is set: the destination is left alone and ZF is set' : `the highest set bit is bit ${alu.highest}`, + }[alu.kind]; + add(body, el('p', { class: 'a86-muted', text: `${what}.` })); + } else { + // multiply and divide: too wide to draw bit by bit usefully + const box = el('div', { class: 'a86-bits-wide' }); + if (alu.kind === 'div' || alu.kind === 'idiv') { + add(box, el('p', {}, [`${alu.dividend} ÷ ${alu.divisor} = `, el('strong', { text: formatSigned(alu.result, w) }), ` remainder ${formatSigned(alu.remainder, w)}`])); + add(box, el('p', { class: 'a86-muted', text: `The dividend is ${w * 16} bits wide, ${WIDE[w]}. Division is the slowest integer instruction: tens of cycles, and the divider cannot start another until it finishes.` })); + } else { + const full = alu.kind === 'mul' ? BigInt(alu.a) * BigInt(alu.b) : toSigned(alu.a, w) * toSigned(alu.b, w); + add(box, el('p', {}, [`${alu.kind === 'mul' ? alu.a : toSigned(alu.a, w)} × ${alu.kind === 'mul' ? alu.b : toSigned(alu.b, w)} = `, el('strong', { text: full.toString() })])); + add(box, el('p', { + class: 'a86-muted', text: alu.wide + ? `The full product needs ${w * 16} bits: the low half lands in ${LOW[w]} and the high half in ${HIGH[w]}. CF and OF are set when the high half is needed.` + : `Only the low ${w * 8} bits are kept. CF and OF are ${f.CF ? 'set: the true product did not fit' : 'clear: the product fitted'}.` + })); + } + add(rows, box); + } + add(body, [title, rows, flags]); + } + + return { node: panel(opts.title ?? 'Inside the arithmetic unit', body, 'a86-bits-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 3. The stack */ +/* ------------------------------------------------------------------ */ + +/** + * The stack as a column of eight-byte slots, highest address at the top, + * with rsp and rbp marked and every slot labeled with what put it there. + * + * options: { slots: 12 } the most slots to draw at once + */ +export function stackView(trace, options = {}) { + const opts = { slots: 12, ...options }; + const body = el('div', { class: 'a86-stack' }); + const top = Number(trace.states[0].regs.rsp); + let lowest = top; + for (const st of trace.states) lowest = Math.min(lowest, Number(st.regs.rsp)); + const layout = trace.layout?.ok ? trace.layout : null; + + const where = (addr) => { + if (!layout) return `step ${addr}`; + const i = layout.byAddr.get(Number(addr)); + return i != null ? `line ${trace.program.instrs[i].lineNo + 1}: ${trace.program.instrs[i].text}` : 'an address outside the program'; + }; + + function update(frame) { + clear(body); + const state = frame.state; + const region = state.mem.find((r) => r.name === 'stack'); + if (!region) { add(body, el('p', { class: 'a86-muted', text: 'This machine has no stack.' })); return; } + const rsp = Number(state.regs.rsp); const rbp = Number(state.regs.rbp); + // labels for every slot written so far + const notes = new Map(); + for (let k = 0; k < frame.t; k++) { + for (const a of trace.steps[k].accesses) { + if (a.kind !== 'write' || a.addr < region.addr || a.addr >= region.addr + region.bytes.length) continue; + const slot = a.addr - ((a.addr - top) % 8 + 8) % 8; + let text; + if (a.stack?.kind === 'call') text = `return address → ${where(a.value)}`; + else if (a.stack?.kind === 'push') text = `pushed ${a.stack.what}`; + else text = `written by ${trace.steps[k].instr.text}`; + notes.set(slot, { text, step: k }); + } + } + const touched = new Set(); const readNow = new Set(); + for (const a of frame.last?.accesses || []) { + for (let b = 0; b < a.size; b++) (a.kind === 'write' ? touched : readNow).add(a.addr + b); + } + // choose a window: from the entry slot down past the lowest rsp ever reached + const want = Math.max(2, Math.ceil((top - lowest) / 8) + 2); + let hi = top; + if (want > opts.slots) hi = Math.min(top, rsp + 8 * Math.floor(opts.slots / 2)); + const slots = []; + for (let a = hi, n = 0; n < Math.min(want, opts.slots) && a >= region.addr; a -= 8, n++) slots.push(a); + + const list = el('div', { class: 'a86-stack-list' }); + if (hi < top) add(list, el('div', { class: 'a86-stack-more', text: `↑ ${(top - hi) / 8} older slot${(top - hi) / 8 > 1 ? 's' : ''} above` })); + for (const a of slots) { + let v = 0n; + for (let b = 7; b >= 0; b--) { try { v = (v << 8n) | BigInt(readByte(state, a + b)); } catch { v <<= 8n; } } + const live = a >= rsp; + const note = notes.get(a); + const wrote = [...Array(8).keys()].some((b) => touched.has(a + b)); + const read = [...Array(8).keys()].some((b) => readNow.has(a + b)); + const tags = []; + if (a === rsp) tags.push(el('span', { class: 'a86-stack-ptr is-rsp', text: 'rsp' })); + if (a === rbp) tags.push(el('span', { class: 'a86-stack-ptr is-rbp', text: 'rbp' })); + add(list, el('div', { class: `a86-stack-slot${live ? '' : ' is-free'}${wrote ? ' is-write' : ''}${read ? ' is-read' : ''}` }, [ + el('code', { class: 'a86-stack-addr', text: formatHex(BigInt(a), 4) }), + el('code', { class: 'a86-stack-val', text: formatHex(v) }), + el('span', { class: 'a86-stack-note', text: a === top && !note ? 'the caller’s slot: where rsp pointed at the start' : note ? note.text : live ? '' : 'free: anything below rsp is garbage' }), + el('span', { class: 'a86-stack-tags' }, tags), + ])); + } + add(body, list); + add(body, el('p', { class: 'a86-stack-foot' }, [ + `rsp = ${formatHex(BigInt(rsp), 4)}, ${top - rsp ? `${plural((top - rsp) / 8, 'slot')} below where it started` : 'where it started'}. `, + el('span', { class: 'a86-muted', text: 'The stack grows downward: push moves rsp to a lower address.' }), + ])); + } + + return { node: panel(opts.title ?? 'The stack', body, 'a86-stack-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 4. A cache */ +/* ------------------------------------------------------------------ */ + +function addressSplit(c, addr) { + const { tag, index, offset } = c.split(addr); + const bits = Math.max(16, Math.ceil(Math.log2(addr + 1))); + const bin = addr.toString(2).padStart(bits, '0'); + const ob = c.offsetBits; const ib = c.indexBits; + const tagBits = bin.slice(0, bits - ob - ib); + const idxBits = bin.slice(bits - ob - ib, bits - ob); + const offBits = bin.slice(bits - ob); + return el('div', { class: 'a86-split' }, [ + el('div', { class: 'a86-split-part is-tag' }, [el('code', { text: tagBits }), el('span', { text: `tag ${formatHex(BigInt(tag), 2)}` })]), + ib ? el('div', { class: 'a86-split-part is-index' }, [el('code', { text: idxBits }), el('span', { text: `set ${index}` })]) : null, + el('div', { class: 'a86-split-part is-offset' }, [el('code', { text: offBits }), el('span', { text: `byte ${offset}` })]), + ]); +} + +/** + * One cache level as its sets and ways, with the current access's set + * outlined, a hit in green, a fill in amber, and the evicted line named. + * + * options: { level: 'L1d' | 'L1i' | 'L2' | 'L3', maxSets: 16, data: true } + */ +export function cacheView(trace, options = {}) { + const opts = { level: 'L1d', maxSets: 16, data: true, ...options }; + const hw = hardwareFor(trace); + const body = el('div', { class: 'a86-cache' }); + + function update(frame) { + clear(body); + const mem = hw.memory(); + const { h } = mem.stateAt(frame.t); + const c = h.cache(opts.level); + if (!c) { add(body, el('p', { class: 'a86-muted', text: `This machine has no ${opts.level}.` })); return; } + const rec = frame.t ? mem.steps[frame.t - 1] : null; + const accesses = rec ? (opts.level === 'L1i' ? (rec.fetch ? [rec.fetch] : []) : [...rec.walks, ...rec.data]) : []; + const events = []; + for (const a of accesses) for (const line of a.lines) for (const e of line.trail) if (e.level === c.name) events.push({ a, line, e }); + for (const a of accesses) for (const line of a.lines) if (line.prefetch) for (const e of line.prefetch.trail) if (e.level === c.name) events.push({ a, line, e, prefetch: true }); + + add(body, el('p', { class: 'a86-cache-shape' }, [ + el('strong', { text: `${c.name}: ${kib(c.size)}` }), + ` = ${c.numSets} set${c.numSets > 1 ? 's' : ''} × ${c.ways} way${c.ways > 1 ? 's' : ''} × ${c.line}-byte lines, ${c.latency}-cycle hit, ${c.policy === 'lru' ? 'least recently used' : c.policy} replacement.`, + ])); + + if (events.length) { + const first = events[0]; + const addr = first.line.lineAddr + ((first.a.addr ?? 0) - first.line.lineAddr >= 0 && (first.a.addr ?? 0) - first.line.lineAddr < c.line ? first.a.addr - first.line.lineAddr : 0); + add(body, el('div', { class: 'a86-cache-access' }, [ + el('span', { class: 'a86-cache-access-what', text: `${first.a.side === 'inst' ? 'Fetch' : first.a.walkLevel ? `Page walk (${first.a.walkLevel})` : first.a.kind === 'write' ? 'Store' : 'Load'} at ${formatHex(BigInt(addr), 4)}${first.a.translation ? ` (virtual ${formatHex(BigInt(first.a.vaddr), 4)})` : ''}` }), + addressSplit(c, addr), + ])); + } else if (rec) add(body, el('p', { class: 'a86-muted', text: `The last instruction did not touch ${c.name}.` })); + + // which sets to draw + let setIdx = [...Array(c.numSets).keys()]; + if (c.numSets > opts.maxSets) { + const used = [...c.sets.keys()].filter((k) => c.sets.get(k).some((w) => w.valid)).sort((a, b) => a - b); + const active = new Set(events.map((e) => e.e.set)); + setIdx = [...new Set([...active, ...used])].sort((a, b) => a - b).slice(0, opts.maxSets); + } + const hitSet = new Map(); const fillSet = new Map(); + for (const { e, prefetch } of events) { + if (e.hit) hitSet.set(`${e.set}:${e.way}`, true); + else fillSet.set(`${e.set}:${e.way}`, prefetch ? 'prefetch' : 'fill'); + } + const activeSets = new Set(events.map((e) => e.e.set)); + + const grid = el('table', { class: 'a86-cache-grid' }); + const head = el('tr', {}, [el('th', { text: 'set' }), ...[...Array(c.ways).keys()].map((w) => el('th', { text: `way ${w}` }))]); + add(grid, el('thead', {}, head)); + const tbody = el('tbody'); + const paging = hw.config.paging?.enabled; + for (const si of setIdx) { + const set = c.sets.get(si); + const ranks = set ? [...set].map((w, k) => ({ k, t: w.lastUse, valid: w.valid })).filter((x) => x.valid).sort((a, b) => b.t - a.t).map((x) => x.k) : []; + const cells = [...Array(c.ways).keys()].map((w) => { + const slot = set?.[w]; + const key = `${si}:${w}`; + const cls = ['a86-cache-cell']; + if (!slot?.valid) cls.push('is-empty'); + if (hitSet.has(key)) cls.push('is-hit'); + if (fillSet.has(key)) cls.push(fillSet.get(key) === 'prefetch' ? 'is-prefetch' : 'is-fill'); + const kids = []; + if (slot?.valid) { + const lineAddr = c.lineAddrOf(slot.tag, si); + kids.push(el('code', { class: 'a86-cache-tag', text: `tag ${formatHex(BigInt(slot.tag), 2)}` })); + if (opts.data && !paging && c.name !== 'L1i') { + const bytes = []; + for (let b = 0; b < Math.min(c.line, 16); b++) { try { bytes.push(readByte(frame.state, lineAddr + b)); } catch { bytes.push(null); } } + const printable = bytes.every((b) => b !== null && b >= 32 && b < 127); + const text = printable ? bytes.map(byteChar).join('') : bytes.slice(0, 6).map((b) => (b === null ? '..' : b.toString(16).padStart(2, '0'))).join(' '); + kids.push(el('code', { class: `a86-cache-data${printable ? '' : ' is-hex'}`, text: text + (c.line > 16 || !printable ? '\u2026' : '') })); + } + kids.push(el('span', { class: 'a86-cache-meta' }, [ + el('span', { text: formatHex(BigInt(lineAddr), 4) }), + slot.dirty ? el('span', { class: 'a86-cache-dirty', text: 'dirty', title: 'changed since it was loaded; must be written back when evicted' }) : null, + slot.prefetched ? el('span', { class: 'a86-cache-pf', text: 'prefetched' }) : null, + c.ways > 1 ? el('span', { class: 'a86-cache-age', text: ranks.indexOf(w) === 0 ? 'newest' : ranks.indexOf(w) === ranks.length - 1 && ranks.length === c.ways ? 'next to go' : '' }) : null, + ])); + } else kids.push(el('span', { class: 'a86-muted', text: 'empty' })); + return el('td', { class: cls.join(' ') }, kids); + }); + add(tbody, el('tr', { class: activeSets.has(si) ? 'is-active' : '' }, [el('th', { text: String(si) }), ...cells])); + } + add(grid, tbody); + add(body, el('div', { class: 'a86-cache-wrap' }, grid)); + if (c.numSets > setIdx.length) add(body, el('p', { class: 'a86-muted', text: `Showing ${setIdx.length} of ${c.numSets} sets: the ones in use.` })); + + const notes = el('ul', { class: 'a86-cache-events' }); + for (const { e, prefetch } of events) { + if (prefetch) { add(notes, el('li', { class: 'is-prefetch', text: `The prefetcher guessed the next line would be wanted and brought it into set ${e.set}.` })); continue; } + if (e.hit) add(notes, el('li', { class: 'is-hit', text: `Hit in set ${e.set}, way ${e.way}: ${e.latency} cycles.` })); + else { + const ev = e.evicted ? ` To make room, the line with tag ${formatHex(BigInt(e.evicted.tag), 2)} was evicted${e.evicted.dirty ? ', and because it was dirty it was written back first' : ''}.` : ` Way ${e.way} was empty.`; + add(notes, el('li', { class: 'is-miss', text: `Miss in set ${e.set}: nothing there has tag ${formatHex(BigInt(e.tag), 2)}. The line came from the next level down.${ev}` })); + } + } + add(body, notes); + const st = c.stats; + add(body, el('p', { class: 'a86-cache-stats', text: `So far: ${commas(st.hits)} hits, ${commas(st.misses)} misses${st.accesses ? ` (${((st.hits / st.accesses) * 100).toFixed(1)}% hit rate)` : ''}, ${commas(st.evictions)} evictions, ${commas(st.writebacks)} written back.` })); + } + + return { node: panel(opts.title ?? `The ${opts.level} cache`, el('div', {}, [body, machineNote(hw)]), 'a86-cache-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 5. The memory hierarchy */ +/* ------------------------------------------------------------------ */ + +/** + * Every level between the core and DRAM, with its size and latency, the + * path the last instruction's accesses took down it, and running totals. + */ +export function hierarchyView(trace, options = {}) { + const hw = hardwareFor(trace); + const body = el('div', { class: 'a86-hier' }); + + function update(frame) { + clear(body); + const mem = hw.memory(); + const { h, space } = mem.stateAt(frame.t); + const rec = frame.t ? mem.steps[frame.t - 1] : null; + const all = rec ? [...(rec.fetch ? [rec.fetch] : []), ...rec.walks, ...rec.data] : []; + const touched = new Map(); // level -> 'hit' | 'miss' + for (const a of all) for (const line of a.lines) for (const e of line.trail) { + const was = touched.get(e.level); + touched.set(e.level, was === 'hit' || e.hit ? (e.hit ? 'hit' : was || 'miss') : 'miss'); + } + const maxLat = h.memLatency; + const rows = []; + const levels = [...(h.l1i ? [h.l1i] : []), h.l1d, ...h.shared]; + for (const c of levels) { + rows.push({ name: c.name, size: kib(c.size), shape: `${c.ways}-way, ${c.line} B lines`, latency: c.latency, stats: c.stats }); + } + rows.push({ name: 'memory', size: 'DRAM', shape: 'gigabytes, off the chip', latency: maxLat, stats: { accesses: h.memStats.reads, hits: h.memStats.reads, misses: 0 } }); + + const ladder = el('div', { class: 'a86-hier-ladder' }); + for (const r of rows) { + const state = touched.get(r.name); + const width = Math.max(4, (Math.log(r.latency + 1) / Math.log(maxLat + 1)) * 100); + add(ladder, el('div', { class: `a86-hier-row${state ? ` is-${state}` : ''}` }, [ + el('div', { class: 'a86-hier-name' }, [el('strong', { text: r.name }), el('span', { text: `${r.size}` })]), + el('div', { class: 'a86-hier-shape', text: r.shape }), + el('div', { class: 'a86-hier-lat' }, [ + el('div', { class: 'a86-hier-bar', style: { width: `${width}%` } }), + el('span', { text: `${r.latency} cycle${r.latency > 1 ? 's' : ''}` }), + ]), + el('div', { class: 'a86-hier-count', text: r.name === 'memory' ? `${commas(r.stats.accesses)} lines fetched` : `${commas(r.stats.hits)} hit / ${commas(r.stats.misses)} miss` }), + ])); + } + add(body, ladder); + if (space) add(body, el('p', { class: 'a86-hier-tlb', text: `TLB: ${space.stats.tlbHits} hits, ${space.stats.tlbMisses} misses, each miss a four-level page walk.` })); + + const paths = el('ul', { class: 'a86-hier-paths' }); + for (const a of all) { + for (const line of a.lines) { + const steps = line.trail.map((e) => `${e.level} ${e.level === 'memory' ? '' : e.hit ? 'hit' : 'miss'}`.trim()).join(' → '); + const what = a.side === 'inst' ? `fetch ${a.size} bytes of code` : a.walkLevel ? `page walk, ${a.walkLevel} entry` : `${a.kind === 'write' ? 'store' : 'load'} ${a.size} byte${a.size > 1 ? 's' : ''}${a.kernel ? ' (by the kernel)' : ''}`; + add(paths, el('li', { class: line.level === 'memory' ? 'is-far' : line.trail[0].hit ? 'is-near' : 'is-mid' }, [ + el('span', { text: `${what}, line ${formatHex(BigInt(line.lineAddr), 4)}: ` }), + el('code', { text: steps }), + el('span', { class: 'a86-hier-cost', text: ` ${line.latency} cycles` }), + a.split ? el('span', { class: 'a86-muted', text: ' (this access straddles two lines)' }) : null, + ])); + } + } + if (all.length) add(body, el('div', {}, [el('p', { class: 'a86-hier-label', text: 'The last instruction’s trips:' }), paths])); + else if (rec) add(body, el('p', { class: 'a86-muted', text: 'The last instruction touched no memory.' })); + const run = mem.running[frame.t]; + add(body, el('p', { class: 'a86-hier-total', text: `So far the loads and stores have waited ${commas(run.memCycles)} cycles in total, ${run.memory} trip${run.memory === 1 ? '' : 's'} of which went all the way to memory.` })); + } + + return { node: panel(options.title ?? 'The memory hierarchy', el('div', {}, [body, machineNote(hw)]), 'a86-hier-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 6. Virtual memory */ +/* ------------------------------------------------------------------ */ + +/** A virtual address split into its four table indexes and page offset, and the walk that translates it. */ +export function pagingView(trace, options = {}) { + const hw = hardwareFor(trace); + const body = el('div', { class: 'a86-pg' }); + + function update(frame) { + clear(body); + if (!hw.config.paging?.enabled) { + add(body, el('p', { class: 'a86-muted', text: 'Paging is off for this machine, and addresses reach the caches untranslated. Give the spec machine: { paging: { enabled: true } } to switch it on.' })); + return; + } + const mem = hw.memory(); + // The most recent data access, which is what a paging lesson is about; + // an instruction fetch only when no data access has happened yet. + let tr = null; let from = null; + for (let k = frame.t - 1; k >= 0 && !tr; k--) { tr = mem.steps[k].translations.find((x) => x.side === 'd') || null; if (tr) from = k; } + if (!tr && frame.t) { tr = mem.steps[frame.t - 1].translations[0] || null; from = frame.t - 1; } + const { space } = mem.stateAt(frame.t); + if (!tr) add(body, el('p', { class: 'a86-muted', text: 'Step forward to translate an address.' })); + else { + const v = BigInt(tr.vaddr); + const field = (name, bitsFrom, width, value, cls) => el('div', { class: `a86-pg-field ${cls}` }, [ + el('code', { text: ((v >> BigInt(bitsFrom)) & ((1n << BigInt(width)) - 1n)).toString(2).padStart(width, '0') }), + el('span', { text: `${name} ${value}` }), + ]); + add(body, el('p', { class: 'a86-pg-head' }, [ + `${tr.side === 'i' ? 'Instruction fetch' : 'Data access'} at virtual `, el('code', { text: formatHex(v, 8) }), + from !== frame.t - 1 ? el('span', { class: 'a86-muted', text: ` (step ${from}: ${trace.steps[from].instr.text}; the last instruction touched no data)` }) : null, + ])); + add(body, el('div', { class: 'a86-pg-split' }, [ + field('PML4', 39, 9, tr.pml4, 'is-l4'), field('PDPT', 30, 9, tr.pdpt, 'is-l3'), + field('PD', 21, 9, tr.pd, 'is-l2'), field('PT', 12, 9, tr.pt, 'is-l1'), + field('offset', 0, 12, `0x${tr.offset.toString(16)}`, 'is-offset'), + ])); + if (tr.tlbHit) { + add(body, el('p', { class: 'a86-pg-tlb is-hit', text: `TLB hit: virtual page 0x${tr.vpn.toString(16)} was translated recently. Its frame (0x${tr.frame.toString(16)}) comes straight from the TLB, and no table is read.` })); + } else { + add(body, el('p', { class: 'a86-pg-tlb is-miss', text: `TLB miss: virtual page 0x${tr.vpn.toString(16)} has no recent translation. The page walker reads one entry from each of four tables, each read depending on the one before.` })); + const walk = el('div', { class: 'a86-pg-walk' }); + tr.walk.forEach((w, k) => { + add(walk, el('div', { class: 'a86-pg-step' }, [ + el('div', { class: 'a86-pg-table', text: `${w.level} table` }), + el('code', { class: 'a86-pg-frame', text: `frame 0x${w.table.toString(16)}` }), + el('span', { text: `entry ${[tr.pml4, tr.pdpt, tr.pd, tr.pt][k]} at ${formatHex(BigInt(w.entryAddr), 4)}` }), + el('span', { class: 'a86-muted', text: `${w.created ? 'filled in by the OS on first touch; ' : ''}${w.access ? `read from ${w.access.level} in ${w.access.latency} cycles` : ''}` }), + el('strong', { text: `→ ${k < 3 ? `${LEVEL_NAMES[k + 1]} table in` : 'data in'} frame 0x${w.next.toString(16)}` }), + ])); + }); + add(body, walk); + } + add(body, el('p', { class: 'a86-pg-result' }, [ + 'Physical address: frame ', el('code', { text: `0x${tr.frame.toString(16)}` }), ' × 4096 + offset ', + el('code', { text: `0x${tr.offset.toString(16)}` }), ' = ', el('strong', { text: formatHex(BigInt(tr.paddr), 4) }), + ])); + } + if (space) { + const t = el('table', { class: 'a86-table a86-pg-tlbtable' }); + add(t, el('thead', {}, el('tr', {}, [el('th', { text: 'TLB entry' }), el('th', { text: 'virtual page' }), el('th', { text: 'physical frame' })]))); + const tb = el('tbody'); + const entries = space.tlbSnapshot('d').sort((a, b) => b.lastUse - a.lastUse); + entries.forEach((e, k) => add(tb, el('tr', { class: tr && e.vpn === tr.vpn && tr.side === 'd' ? 'is-current' : '' }, [el('td', { text: k === 0 ? 'most recent' : String(k) }), el('td', {}, el('code', { text: `0x${e.vpn.toString(16)}` })), el('td', {}, el('code', { text: `0x${e.frame.toString(16)}` }))]))); + for (let k = entries.length; k < space.tlbEntries && k < 8; k++) add(tb, el('tr', {}, [el('td', { text: '' }), el('td', { class: 'a86-muted', text: 'empty' }), el('td', {})])); + add(t, tb); + add(body, el('div', {}, [el('p', { class: 'a86-hier-label', text: `Data TLB, ${space.tlbEntries} entries: ${space.stats.tlbHits} hits, ${space.stats.tlbMisses} misses so far` }), t])); + } + } + + return { node: panel(options.title ?? 'Virtual memory', el('div', {}, [body, machineNote(hw)]), 'a86-pg-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 7. The five-stage pipeline */ +/* ------------------------------------------------------------------ */ + +/** + * The classic pipeline diagram: one row per instruction, one column per + * cycle, each cell the stage that instruction occupied. Stalls repeat their + * stage in amber; instructions fetched down a mispredicted path appear in + * red and are thrown away. + * + * options: { before: 7, after: 4, cycles: 20 } + */ +export function pipelineView(trace, options = {}) { + const opts = { before: 7, after: 4, cycles: 20, ...options }; + const hw = hardwareFor(trace); + const body = el('div', { class: 'a86-pipe' }); + + function update(frame) { + clear(body); + const p = hw.pipeline(); + if (!p.rows.length) { add(body, el('p', { class: 'a86-muted', text: 'Nothing ran.' })); return; } + const k = Math.max(0, lastIndex(frame)); + const now = frame.t ? p.rows[k].E.WB : 0; + const first = Math.max(0, k - opts.before); + const last = Math.min(p.rows.length - 1, k + opts.after); + const visible = []; + for (let i = first; i <= last; i++) { + visible.push({ row: p.rows[i] }); + const r = p.rows[i]; + r.wrongPath.forEach((ins, j) => visible.push({ ghost: ins, fetch: r.wrongFrom + j, until: r.wrongFrom + r.squashed, of: r })); + } + let start = Math.min(...visible.map((v) => (v.row ? v.row.E.IF : v.fetch))); + if (now - start > opts.cycles - 4) start = now - (opts.cycles - 4); + const end = start + opts.cycles; + + const table = el('table', { class: 'a86-pipe-grid' }); + const head = el('tr', {}, [el('th', { class: 'a86-pipe-instr', text: 'instruction' }), ...Array.from({ length: end - start }, (_, x) => el('th', { class: start + x === now && frame.t ? 'is-now' : '', text: String(start + x) }))]); + add(table, el('thead', {}, head)); + const tbody = el('tbody'); + for (const v of visible) { + const cells = []; + if (v.row) { + const r = v.row; + for (let x = start; x < end; x++) { + let content = ''; let cls = 'a86-pipe-cell'; + for (const s of STAGES) { + const e = r.E[s]; const d = r.D[s]; + if (x >= e && x < e + d) { + content = s; + const work = s === 'IF' ? r.work.IF : s === 'EX' ? r.work.EX : s === 'MEM' ? r.work.MEM : 1; + cls += ` is-${s.toLowerCase()}`; + if (x - e >= work) cls += ' is-stall'; + else if (x - e > 0) cls += ' is-busy'; + } + } + if (x === now && frame.t) cls += ' is-now'; + cells.push(el('td', { class: cls, text: content })); + } + const done = r.index < frame.t; + add(tbody, el('tr', { class: `${r.index === k && frame.t ? 'is-current' : ''}${done ? '' : ' is-future'}${r.squashed ? ' is-mispredicted' : ''}` }, [ + el('td', { class: 'a86-pipe-instr' }, [el('span', { class: 'a86-trace-step', text: `${r.index} ` }), el('code', { text: r.step.instr.text })]), + ...cells, + ])); + } else { + for (let x = start; x < end; x++) { + const stage = x >= v.fetch && x < v.until ? STAGES[x - v.fetch] : ''; + cells.push(el('td', { class: `a86-pipe-cell${stage ? ' is-squashed' : ''}`, text: stage ? `${stage}` : '' })); + } + add(tbody, el('tr', { class: 'is-ghost' }, [el('td', { class: 'a86-pipe-instr' }, [el('span', { class: 'a86-trace-step', text: '✖ ' }), el('code', { text: v.ghost.text })]), ...cells])); + } + } + add(table, tbody); + add(body, el('div', { class: 'a86-pipe-wrap' }, table)); + add(body, legend([['is-stage', 'a stage doing work'], ['is-stall', 'stalled: holding the stage, doing nothing'], ['is-squashed', 'fetched down the wrong path, then thrown away']], 'a86-pipe-legend')); + + const notes = el('ul', { class: 'a86-pipe-notes' }); + if (frame.t) { + const r = p.rows[k]; + for (const s of r.stalls) add(notes, el('li', { class: `is-${s.reason}`, text: `${r.step.instr.text}: ${s.text}.` })); + if (r.squashed) add(notes, el('li', { class: 'is-control', text: `${r.step.instr.text} was mispredicted. It settles in ${p.config.resolveIn}, by which time ${plural(r.squashed, 'wrong instruction')} had been fetched behind it; they are discarded and fetch restarts on the right path.` })); + if (!r.stalls.length && !r.squashed) add(notes, el('li', { class: 'a86-muted', text: `${r.step.instr.text} flowed through without a stall.` })); + const occ = stageOccupancy(p, now); + add(body, el('div', { class: 'a86-pipe-now' }, STAGES.map((s) => el('div', { class: `a86-pipe-stage is-${s.toLowerCase()}` }, [ + el('span', { class: 'a86-pipe-stage-name', text: s }), + el('code', { text: occ[s] ? occ[s].step.instr.text : '(empty)' }), + ])))); + } + add(body, notes); + const tt = p.totals; + add(body, el('p', { class: 'a86-pipe-total', text: `Whole run: ${commas(p.cycles)} cycles for ${commas(p.instructions)} instructions, ${p.cpi.toFixed(2)} cycles each (a perfect pipeline would take ${commas(p.ideal)}). Lost to data hazards ${tt.dataStalls} (${tt.loadUse} of them load-use), to cache misses ${tt.memoryStalls + tt.fetchStalls}, to flushed wrong-path fetches ${tt.flushed}, to multi-cycle operations ${tt.multiCycle}.` })); + } + + return { node: panel(options.title ?? 'The five-stage pipeline', el('div', {}, [body, machineNote(hw)]), 'a86-pipe-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 8. Micro-ops */ +/* ------------------------------------------------------------------ */ + +/** One instruction split into the uops the out-of-order core actually schedules. */ +export function uopsView(trace, options = {}) { + const opts = { which: 'auto', ...options }; + const hw = hardwareFor(trace); + const body = el('div', { class: 'a86-uops' }); + + function update(frame) { + clear(body); + const f = focus(frame, opts.which); + if (!f || !f.ran) { + add(body, el('p', { class: 'a86-muted', text: 'Step forward: this view shows the micro-ops of the instruction that just ran.' })); + return; + } + const core = hw.core(); + const inst = core.insts[f.index]; + if (inst.fusedInto !== null && inst.fusedInto !== undefined) { + const host = core.uops[inst.fusedInto]; + add(body, el('p', {}, [el('code', { text: f.instr.text }), ` has no micro-op of its own: it was fused with `, el('code', { text: core.insts[host.inst].instr.text }), ` into a single compare-and-branch micro-op, which ran on ${host.port ?? 'a branch port'} at cycle ${host.issue}.`])); + return; + } + const us = inst.uops.map((id) => core.uops[id]); + add(body, el('p', {}, [el('code', { text: f.instr.text }), ` → ${plural(us.length, 'micro-op')}.`])); + const t = el('table', { class: 'a86-table a86-uops-table' }); + add(t, el('thead', {}, el('tr', {}, ['micro-op', 'kind', 'can run on', 'waits for', 'latency', 'ran', 'note'].map((h) => el('th', { text: h }))))); + const tb = el('tbody'); + for (const u of us) { + const waits = [ + ...u.producers.map((p) => { const pu = core.uops[p]; return pu.inst === u.inst ? `uop ${p} (same instruction)` : `uop ${p}: ${core.insts[pu.inst].instr.text}`; }), + ...u.memProducers.map((p) => (core.uops[p].kind === 'locked' ? `the locked instruction in uop ${p}` : `the store in uop ${p}`)), + ]; + const ran = u.kind === 'eliminated' ? 'at rename' : u.kind === 'nop' ? 'not executed' : `cycle ${u.issue}${u.port ? ` on ${u.port}` : ''}, done ${u.complete}`; + add(tb, el('tr', { class: `is-${u.kind}` }, [ + el('td', {}, el('code', { text: `#${u.id}` })), + el('td', { text: u.fused ? 'branch (fused)' : u.kind }), + el('td', { text: u.ports.length ? u.ports.join(' ') : 'no port' }), + el('td', { text: waits.length ? waits.join('; ') : 'nothing' }), + el('td', { text: `${u.latency}${u.level && u.kind === 'load' ? ` (${u.level})` : ''}` }), + el('td', { text: ran }), + el('td', { class: 'a86-uops-note', text: u.note }), + ])); + } + add(t, tb); + add(body, el('div', { class: 'a86-uops-wrap' }, t)); + } + + return { node: panel(options.title ?? 'Micro-ops', el('div', {}, [body, machineNote(hw)]), 'a86-uops-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 9. The out-of-order timeline */ +/* ------------------------------------------------------------------ */ + +/** + * When each instruction entered the core, ran, finished, and retired, + * drawn against cycles. In a well-fed loop the bars overlap: that overlap + * is the whole point of an out-of-order core. + * + * options: { before: 12, after: 6 } + */ +export function timelineView(trace, options = {}) { + const opts = { before: 12, after: 6, ...options }; + const hw = hardwareFor(trace); + const host = el('div', { class: 'a86-tl' }); + + function update(frame) { + clear(host); + const core = hw.core(); + if (!core.insts.length) { add(host, el('p', { class: 'a86-muted', text: 'Nothing ran.' })); return; } + const k = Math.max(0, lastIndex(frame)); + const first = Math.max(0, k - opts.before); + const last = Math.min(core.insts.length - 1, k + opts.after); + const rows = core.insts.slice(first, last + 1); + const c0 = Math.min(...rows.map((r) => r.alloc)); + const c1 = Math.max(...rows.map((r) => r.retire)) + 1; + const now = frame.t ? core.insts[k].retire : c0; + const W = 700; const labelW = 250; const rowH = 18; const top = 22; + const plotW = W - labelW - 10; + const x = (c) => labelW + ((c - c0) / Math.max(1, c1 - c0)) * plotW; + const H = top + rows.length * rowH + 8; + const svgEl = sv('svg', { viewBox: `0 0 ${W} ${H}`, class: 'a86-tl-svg', role: 'img', 'aria-label': 'out-of-order timeline' }); + // axis + const span = c1 - c0; + const tick = span > 400 ? 100 : span > 200 ? 50 : span > 80 ? 20 : span > 30 ? 10 : 5; + for (let c = Math.ceil(c0 / tick) * tick; c <= c1; c += tick) { + add(svgEl, sv('line', { x1: x(c), x2: x(c), y1: top - 4, y2: H - 4, class: 'a86-tl-grid' })); + add(svgEl, sv('text', { x: x(c), y: 12, class: 'a86-tl-tick', 'text-anchor': 'middle', text: String(c) })); + } + // refill shading after mispredicts + for (const r of rows) { + for (const id of r.uops) { + const u = core.uops[id]; + if (u.mispredicted && u.complete !== null) { + const a = x(u.complete); const b = x(Math.min(c1, u.complete + core.config.mispredictPenalty)); + add(svgEl, sv('rect', { x: a, y: top - 4, width: Math.max(1, b - a), height: H - top, class: 'a86-tl-refill' })); + } + } + } + rows.forEach((r, i) => { + const y = top + i * rowH; + const cur = r.index === k && frame.t; + const done = r.index < frame.t; + const g = sv('g', { class: `a86-tl-row${cur ? ' is-current' : ''}${done ? '' : ' is-future'}` }); + if (cur) add(g, sv('rect', { x: 0, y: y - 1, width: W, height: rowH, class: 'a86-tl-highlight' })); + const label = r.instr.text.length > 30 ? `${r.instr.text.slice(0, 29)}…` : r.instr.text; + add(g, sv('text', { x: 4, y: y + 12, class: 'a86-tl-label', text: `${r.index} ${label}` })); + const us = r.uops.map((id) => core.uops[id]); + const eliminated = us.every((u) => !u.ports.length); + const issue = eliminated ? r.alloc : r.issue; + add(g, sv('rect', { x: x(r.alloc), y: y + 3, width: Math.max(1, x(issue) - x(r.alloc)), height: rowH - 6, class: 'a86-tl-wait' })); + add(g, sv('rect', { x: x(r.complete), y: y + 3, width: Math.max(0, x(r.retire) - x(r.complete)), height: rowH - 6, class: 'a86-tl-done' })); + // one bar per uop, so a load followed by the arithmetic that needed it + // reads as two segments: blue, then amber + if (eliminated) add(g, sv('rect', { x: x(r.alloc), y: y + 3, width: 3, height: rowH - 6, class: 'a86-tl-exec is-elim' })); + for (const u of us) { + if (u.issue === null || !u.ports.length) continue; + const kind = u.kind === 'load' || u.kind === 'locked' ? (u.level === 'L1d' || u.level === 'store buffer' ? 'is-load' : 'is-miss') : u.kind.startsWith('store') ? 'is-store' : 'is-exec'; + add(g, sv('rect', { x: x(u.issue), y: y + 3, width: Math.max(2, x(u.complete) - x(u.issue)), height: rowH - 6, class: `a86-tl-exec ${kind}` })); + } + add(g, sv('line', { x1: x(r.retire + 1), x2: x(r.retire + 1), y1: y + 1, y2: y + rowH - 1, class: 'a86-tl-retire' })); + if (us.some((u) => u.mispredicted)) add(g, sv('text', { x: x(r.complete) + 3, y: y + 12, class: 'a86-tl-flag', text: 'mispredicted' })); + const tip = sv('title', { text: `${r.instr.text}\nentered the core at cycle ${r.alloc}\nstarted ${issue}, finished ${r.complete}, retired ${r.retire}\n${us.map((u) => `${u.kind}${u.port ? ` on ${u.port}` : ''}${u.level ? ` from ${u.level}` : ''}`).join(', ')}` }); + add(g, tip); + add(svgEl, g); + }); + if (frame.t) add(svgEl, sv('line', { x1: x(now + 1), x2: x(now + 1), y1: top - 6, y2: H - 2, class: 'a86-tl-now' })); + add(host, svgEl); + add(host, legend([ + ['is-tl-wait', 'in the core, waiting for inputs or a port'], + ['is-tl-exec', 'executing'], ['is-tl-load', 'loading from L1d or the store buffer'], ['is-tl-miss', 'loading from further away'], ['is-tl-store', 'storing'], + ['is-tl-done', 'finished, waiting to retire in order'], ['is-tl-refill', 'front end refilling after a mispredict'], + ], 'a86-tl-legend')); + const r = core.insts[k]; + add(host, el('p', { + class: 'a86-tl-caption', text: frame.t + ? `${r.instr.text} entered the core at cycle ${r.alloc}, finished at ${r.complete}, and retired at ${r.retire}. By then ${core.insts.filter((q) => q.index > k && q.alloc <= r.retire).length} younger instructions had already entered.` + : 'Step forward to follow instructions through the core.' + })); + add(host, el('p', { class: 'a86-tl-total', text: `Whole run: ${commas(core.cycles)} cycles, ${commas(core.instructions)} instructions, ${core.ipc.toFixed(2)} per cycle.` })); + } + + return { node: panel(options.title ?? 'Out of order: when each instruction ran', el('div', {}, [host, machineNote(hw)]), 'a86-tl-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 10. What limits the speed */ +/* ------------------------------------------------------------------ */ + +/** + * Three lower bounds on the run time, set against the time the model took: + * the longest dependency chain, the busiest group of ports, and the front + * end's width. Plus where the lost cycles went. Whole-program. + */ +export function bottleneckView(trace, options = {}) { + const hw = hardwareFor(trace); + const body = el('div', { class: 'a86-bn' }); + const core = hw.core(); + const max = Math.max(core.cycles, ...core.bounds.map((b) => b.cycles), 1); + const bar = (label, value, cls, note) => el('div', { class: 'a86-bn-row' }, [ + el('span', { class: 'a86-bn-label', text: label }), + el('div', { class: 'a86-bn-track' }, [el('div', { class: `a86-bn-bar ${cls}`, style: { width: `${(value / max) * 100}%` } })]), + el('span', { class: 'a86-bn-value', text: commas(value) }), + note ? el('span', { class: 'a86-bn-note', text: note }) : null, + ]); + add(body, bar('actual', core.cycles, 'is-actual', `${core.ipc.toFixed(2)} instructions per cycle`)); + for (const b of core.bounds) add(body, bar(b.name, b.cycles, 'is-bound', b.text)); + const top = core.bounds[0]; + const lost = core.stallCycles; + const stallText = [ + lost.mispredict && `${commas(lost.mispredict)} waiting for mispredicted branches to be caught and the front end refilled`, + lost.fetch && `${commas(lost.fetch)} waiting for instruction fetch`, + lost.robFull && `${commas(lost.robFull)} with the reorder buffer full, usually behind a slow load`, + lost.rsFull && `${commas(lost.rsFull)} with the scheduler full`, + lost.sbFull && `${commas(lost.sbFull)} with the store buffer full, waiting for stores to reach the cache`, + lost.serialize && `${commas(lost.serialize)} in or waiting for system calls`, + ].filter(Boolean); + add(body, el('p', { class: 'a86-bn-verdict', text: `No schedule could beat ${commas(top.cycles)} cycles, the ${top.name} limit. The model took ${commas(core.cycles)}.${stallText.length ? ` Cycles in which nothing new entered the core: ${stallText.join('; ')}.` : ''}` })); + const ports = el('div', { class: 'a86-bn-ports' }); + for (const p of PORT_ORDER) { + const use = core.portUse[p] || 0; + const pct = core.cycles ? (use / core.cycles) * 100 : 0; + add(ports, el('div', { class: 'a86-bn-port', title: PORTS[p].units.join(', ') }, [ + el('div', { class: 'a86-bn-port-track' }, el('div', { class: 'a86-bn-port-fill', style: { height: `${Math.min(100, pct)}%` } })), + el('code', { text: p }), + el('span', { text: `${pct.toFixed(0)}%` }), + ])); + } + add(body, el('p', { class: 'a86-hier-label', text: 'How busy each execution port was:' })); + add(body, ports); + return { node: panel(options.title ?? 'What limits the speed', el('div', {}, [body, machineNote(hw)]), 'a86-bn-panel'), update() { } }; +} + +/* ------------------------------------------------------------------ */ +/* 11. The core, one cycle at a time */ +/* ------------------------------------------------------------------ */ + +/** + * The out-of-order core frozen at one cycle: what sits in the reorder + * buffer, what each port is doing, and which physical result every + * register name currently points at. Follows the widget's step by default; + * the cycle buttons move through time on their own. + */ +export function coreView(trace, options = {}) { + const opts = { rob: 14, ...options }; + const hw = hardwareFor(trace); + const body = el('div', { class: 'a86-core' }); + let offset = 0; let lastT = -1; + const controls = el('div', { class: 'a86-core-controls' }); + const readout = el('span', { class: 'a86-readout' }); + let current = null; + add(controls, [ + el('button', { class: 'a86-btn', type: 'button', text: '◂ cycle', onclick: () => { offset--; render(); } }), + el('button', { class: 'a86-btn', type: 'button', text: 'cycle ▸', onclick: () => { offset++; render(); } }), + el('button', { class: 'a86-btn', type: 'button', text: 'follow the step', onclick: () => { offset = 0; render(); } }), + readout, + ]); + + function render() { + if (!current) return; + const frame = current; + clear(body); + const core = hw.core(); + if (!core.uops.length) { add(body, el('p', { class: 'a86-muted', text: 'Nothing ran.' })); return; } + const k = lastIndex(frame); + const base = k >= 0 ? core.insts[k].retire : 0; + const cycle = Math.max(0, Math.min(core.cycles, base + offset)); + readout.textContent = `cycle ${cycle}${offset ? ` (${offset > 0 ? '+' : ''}${offset} from the step)` : ''}`; + const snap = coreAt(core, cycle); + const text = (u) => core.insts[u.inst].instr.text; + const stateOf = (u) => (u.retire === cycle ? 'retiring' : u.complete !== null && u.complete <= cycle ? 'done' : u.issue !== null && u.issue <= cycle ? `on ${u.port}` : !u.ports.length ? 'renamed' : 'waiting'); + + const robBox = el('div', { class: 'a86-core-rob' }); + add(robBox, el('p', { class: 'a86-hier-label', text: `Reorder buffer: ${snap.rob.length} of ${core.config.rob}, oldest first` })); + const list = el('ol', { class: 'a86-core-list' }); + snap.rob.slice(0, opts.rob).forEach((u) => { + const st = stateOf(u); + add(list, el('li', { class: `is-${st.startsWith('on') ? 'running' : st}` }, [ + el('code', { class: 'a86-core-id', text: `#${u.id}` }), + el('span', { class: 'a86-core-kind', text: u.fused ? 'cmp+jcc' : u.kind }), + el('code', { class: 'a86-core-text', text: text(u) }), + el('span', { class: 'a86-core-state', text: st }), + ])); + }); + add(robBox, list); + if (snap.rob.length > opts.rob) add(robBox, el('p', { class: 'a86-muted', text: `… and ${snap.rob.length - opts.rob} younger` })); + if (!snap.rob.length) add(robBox, el('p', { class: 'a86-muted', text: 'empty' })); + + const portsBox = el('div', { class: 'a86-core-ports' }); + add(portsBox, el('p', { class: 'a86-hier-label', text: 'Execution ports this cycle' })); + const grid = el('div', { class: 'a86-core-portgrid' }); + for (const p of PORT_ORDER) { + const run = snap.executing.find((u) => u.port === p); + const started = snap.issuedNow.find((u) => u.port === p); + add(grid, el('div', { class: `a86-core-port${run ? ' is-busy' : ''}${started ? ' is-start' : ''}` }, [ + el('code', { class: 'a86-core-portname', text: p }), + el('span', { class: 'a86-core-units', text: PORTS[p].units.filter((x) => !x.startsWith('vec-')).join(' · ') }), + el('span', { class: 'a86-core-portuop', text: run ? `#${run.id} ${run.kind}: ${text(run)}` : 'idle' }), + ])); + } + add(portsBox, grid); + + const ratBox = el('div', { class: 'a86-core-rat' }); + add(ratBox, el('p', { class: 'a86-hier-label', text: 'Rename table: where each name’s newest value comes from' })); + const used = new Set(); + for (const u of core.uops) for (const d of u.dsts) if (!/^t\d$/.test(d)) used.add(d); + const rt = el('table', { class: 'a86-table' }); + const tb = el('tbody'); + for (const name of [...used].sort((a, b) => isFlagGroup(a) - isFlagGroup(b) || a.localeCompare(b, 'en', { numeric: true }))) { + const u = snap.rat[name]; + const pending = u && (u.complete === null || u.complete > cycle); + add(tb, el('tr', { class: pending ? 'is-pending' : '' }, [ + el('td', {}, el('code', { text: depName(name) })), + el('td', { text: u ? `uop #${u.id}, ${text(u)}` : 'the starting value' }), + el('td', { class: 'a86-core-ready', text: u ? (pending ? 'not ready yet' : 'ready') : 'ready' }), + ])); + } + add(rt, tb); + add(ratBox, rt); + + add(body, el('div', { class: 'a86-core-cols' }, [robBox, portsBox, ratBox])); + const why = { mispredict: 'Nothing entered the core this cycle: the front end is still recovering from a mispredicted branch.', fetch: 'Nothing entered the core this cycle: instruction fetch is waiting on a cache miss.', robFull: 'Nothing entered the core this cycle: the reorder buffer is full until the oldest instruction retires.', rsFull: 'Nothing entered the core this cycle: the scheduler is full.', sbFull: 'Nothing entered the core this cycle: the store buffer is full, and the next store must wait for an older one to reach the cache.', serialize: 'The core is draining for a system call, which must run alone.' }[snap.stall]; + add(body, el('p', { class: 'a86-core-note', text: `${snap.allocNow.length} micro-op${snap.allocNow.length === 1 ? '' : 's'} entered, ${snap.issuedNow.length} started, ${snap.retiredNow.length} retired this cycle. ${why || ''}` })); + } + + function update(frame) { + current = frame; + if (frame.t !== lastT) { offset = 0; lastT = frame.t; } + render(); + } + + return { node: panel(options.title ?? 'Inside the out-of-order core', el('div', {}, [controls, body, machineNote(hw)]), 'a86-core-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 12. The terminal and the kernel boundary */ +/* ------------------------------------------------------------------ */ + +/** What the program printed, and every system call it made to get there. */ +export function consoleView(trace, options = {}) { + const body = el('div', { class: 'a86-console' }); + const screen = el('pre', { class: 'a86-console-screen' }); + const log = el('ol', { class: 'a86-console-log' }); + const note = el('p', { class: 'a86-console-note' }); + add(body, [screen, log, note]); + const calls = trace.steps.map((s, k) => (s.syscall ? { k, s } : null)).filter(Boolean); + + function update(frame) { + clear(screen); clear(log); + add(screen, [frame.state.out, el('span', { class: 'a86-console-cursor', text: ' ' })]); + for (const { k, s } of calls) { + if (k >= frame.t) break; + const sc = s.syscall; + const args = sc.name === 'write' ? `${sc.args[0]}, ${formatHex(sc.args[1], 4)}, ${sc.args[2]}` : sc.name === 'read' ? `${sc.args[0]}, ${formatHex(sc.args[1], 4)}, ${sc.args[2]}` : sc.name.startsWith('exit') ? `${sc.args[0]}` : ''; + add(log, el('li', { class: k === frame.t - 1 ? 'is-current' : '' }, [ + el('code', { text: `${sc.name}(${args})${sc.ret !== null ? ` = ${formatSigned(BigInt.asUintN(64, sc.ret))}` : ''}` }), + el('span', { class: 'a86-muted', text: ` step ${k}` }), + ])); + } + const s = frame.last; + if (s?.syscall) note.textContent = `syscall switched the processor from user mode to kernel mode, the kernel did the work, and sysret switched back. A round trip costs on the order of 100 cycles before the kernel does anything useful, which is why programs buffer their output.`; + else if (!calls.length) note.textContent = 'Nothing appears here, because this program never asks the kernel for anything. Printing needs the write system call: rax = 1, rdi = 1 for the terminal, rsi = the address of the text, rdx = its length.'; + else note.textContent = frame.state.exitCode != null ? `Exited with code ${frame.state.exitCode}.` : ''; + } + + return { node: panel(options.title ?? 'Terminal', body, 'a86-console-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 13. perf stat */ +/* ------------------------------------------------------------------ */ + +/** + * Whole-program counts from the models, laid out the way `perf stat` + * prints real hardware counters, so the real tool looks familiar the first + * time a student runs it. + */ +export function perfView(trace, options = {}) { + const hw = hardwareFor(trace); + const p = hw.perf(); + const line = (count, name, comment = '') => `${String(count).padStart(18)} ${name.padEnd(26)}${comment ? `# ${comment}` : ''}`; + const pct = (a, b) => (b ? `${((a / b) * 100).toFixed(2).padStart(6)}% of all ${''}` : ''); + const text = [ + ` Performance counter stats for '${options.name || trace.spec.title || 'this program'}' (${hw.config.name} machine, modeled):`, + '', + line(commas(p.cyclesOoO), 'cycles', `${p.ipc.toFixed(2).padStart(6)} insn per cycle`), + line(commas(p.instructions), 'instructions'), + line(commas(p.uops), 'uops_issued.any'), + line(commas(p.branches), 'branches'), + line(commas(p.mispredicts), 'branch-misses', p.branches ? `${pct(p.mispredicts, p.branches)}branches` : ''), + line(commas(p.l1dLoads), 'L1-dcache-loads'), + line(commas(p.l1dMisses), 'L1-dcache-load-misses', p.l1dLoads ? `${pct(p.l1dMisses, p.l1dLoads)}L1-dcache accesses` : ''), + line(commas(p.l1iMisses), 'L1-icache-load-misses'), + line(commas(p.l2Misses), 'l2_rqsts.miss'), + ...(hw.config.caches.l3 ? [line(commas(p.l3Misses), 'LLC-load-misses')] : []), + ...(p.tlbMisses !== null ? [line(commas(p.tlbMisses), 'dTLB-load-misses')] : []), + '', + `${(p.seconds).toFixed(9).padStart(18)} seconds time elapsed (at ${hw.config.clockGHz} GHz)`, + ].join('\n'); + const body = el('div', { class: 'a86-perf' }, [ + el('pre', { class: 'a86-perf-out', text }), + el('p', { class: 'a86-muted', text: 'Model counts, printed the way Linux perf prints real ones. On a real machine, perf stat -e cycles,instructions,branch-misses ./program measures the same events.' }), + ]); + return { node: panel(options.title ?? 'perf stat', body, 'a86-perf-panel'), update() { } }; +} + +/* ------------------------------------------------------------------ */ +/* 14. The whole chip */ +/* ------------------------------------------------------------------ */ + +const CHIP = { + bp: { x: 12, y: 30, w: 150, h: 40, label: 'branch predictor', group: 'front' }, + itlb: { x: 12, y: 84, w: 44, h: 40, label: 'iTLB', group: 'front' }, + l1i: { x: 62, y: 84, w: 100, h: 40, label: 'L1i cache', group: 'front' }, + decode: { x: 12, y: 138, w: 150, h: 40, label: 'fetch and decode', sub: 'x86 → micro-ops', group: 'front' }, + rename: { x: 190, y: 30, w: 260, h: 40, label: 'rename and allocate', sub: 'register names → physical registers', group: 'ooo' }, + rob: { x: 190, y: 84, w: 260, h: 40, label: 'reorder buffer', sub: 'retires in program order', group: 'ooo' }, + rs: { x: 190, y: 138, w: 260, h: 40, label: 'scheduler', sub: 'ready micro-ops go to free ports', group: 'ooo' }, + p0: { x: 190, y: 196, w: 29, h: 44, label: 'p0', group: 'port' }, + p1: { x: 223, y: 196, w: 29, h: 44, label: 'p1', group: 'port' }, + p5: { x: 256, y: 196, w: 29, h: 44, label: 'p5', group: 'port' }, + p6: { x: 289, y: 196, w: 29, h: 44, label: 'p6', group: 'port' }, + p2: { x: 322, y: 196, w: 29, h: 44, label: 'p2', group: 'port' }, + p3: { x: 355, y: 196, w: 29, h: 44, label: 'p3', group: 'port' }, + p4: { x: 388, y: 196, w: 29, h: 44, label: 'p4', group: 'port' }, + p7: { x: 421, y: 196, w: 29, h: 44, label: 'p7', group: 'port' }, + regs: { x: 190, y: 262, w: 126, h: 40, label: 'integer registers', group: 'ooo' }, + vregs: { x: 324, y: 262, w: 126, h: 40, label: 'vector registers', group: 'ooo' }, + sb: { x: 478, y: 30, w: 130, h: 40, label: 'store buffer', group: 'mem' }, + dtlb: { x: 478, y: 84, w: 44, h: 40, label: 'dTLB', group: 'mem' }, + l1d: { x: 528, y: 84, w: 80, h: 40, label: 'L1d cache', group: 'mem' }, + walker: { x: 478, y: 138, w: 70, h: 40, label: 'page walker', group: 'mem' }, + l2: { x: 632, y: 84, w: 116, h: 94, label: 'L2 cache', sub: 'code and data', group: 'mem' }, + l3: { x: 478, y: 196, w: 270, h: 44, label: 'L3 cache', sub: 'shared by every core', group: 'mem' }, + dram: { x: 478, y: 262, w: 270, h: 40, label: 'memory (DRAM)', sub: 'off the chip', group: 'mem' }, + os: { x: 12, y: 262, w: 150, h: 40, label: 'operating system', sub: 'kernel mode', group: 'front' }, +}; + +const PORT_SUB = { p0: 'alu br', p1: 'alu mul', p5: 'alu vec', p6: 'alu br', p2: 'load', p3: 'load', p4: 'store', p7: 'addr' }; + +const CHIP_EDGES = [ + ['bp', 'l1i'], ['l1i', 'decode'], ['decode', 'rename'], ['rename', 'rob'], ['rob', 'rs'], + ['rs', 'ports'], ['ports', 'regs'], ['p2', 'l1d'], ['p4', 'sb'], ['sb', 'l1d'], + ['l1d', 'l2'], ['l1i', 'l2'], ['l2', 'l3'], ['l3', 'dram'], ['dtlb', 'walker'], ['walker', 'l1d'], +]; + +/** + * A whole core, from branch predictor to DRAM, with the parts the last + * instruction used lit up: the path its bytes took to be fetched, the ports + * its uops ran on, and how far down the memory hierarchy its loads had to go. + */ +let chipIds = 0; + +export function machineView(trace, options = {}) { + const hw = hardwareFor(trace); + const W = 760; const H = 314; + const svgEl = sv('svg', { viewBox: `0 0 ${W} ${H}`, class: 'a86-chip-svg', role: 'img', 'aria-label': 'a processor core and its memory hierarchy' }); + // One arrowhead per color, sized in the drawing's own units so a thicker line does not grow it. Only a lit + // edge carries a head, and the head points the way the data moved (see `edge` in update). + const uid = `a86-chip-${++chipIds}`; + const head = (kind) => sv('marker', { id: `${uid}-${kind}`, class: `a86-chip-head is-${kind}`, viewBox: '0 0 10 10', refX: 8, refY: 5, markerWidth: 9, markerHeight: 9, markerUnits: 'userSpaceOnUse', orient: 'auto-start-reverse' }, [sv('path', { d: 'M 0 0 L 10 5 L 0 10 z' })]); + add(svgEl, sv('defs', {}, [head('active'), head('load'), head('store')])); + // group outlines + const groups = [ + { x: 4, y: 6, w: 166, h: 180, label: 'front end' }, + { x: 182, y: 6, w: 276, h: 302, label: 'out-of-order engine' }, + { x: 470, y: 6, w: 286, h: 302, label: 'memory' }, + ]; + for (const g of groups) { + add(svgEl, sv('rect', { x: g.x, y: g.y, width: g.w, height: g.h, rx: 6, class: 'a86-chip-group' })); + add(svgEl, sv('text', { x: g.x + 8, y: g.y + 16, class: 'a86-chip-grouplabel', text: g.label })); + } + const nodeEls = {}; + const edgeEls = {}; + const center = (n) => ({ x: n.x + n.w / 2, y: n.y + n.h / 2 }); + const edgePath = (a, b) => { + const A = a === 'ports' ? { x: 320, y: 240, w: 0, h: 0 } : CHIP[a]; + const B = b === 'ports' ? { x: 320, y: 196, w: 0, h: 0 } : CHIP[b]; + const ca = a === 'ports' ? { x: 320, y: 240 } : center(A); const cb = b === 'ports' ? { x: 320, y: 196 } : center(B); + if (a === 'bp') return 'M 112 70 L 112 84'; + if (a === 'l1i' && b === 'decode') return 'M 112 124 L 112 138'; + if (a === 'decode') return 'M 162 158 L 176 158 L 176 50 L 190 50'; + if (a === 'rs' && b === 'ports') return `M ${ca.x} ${A.y + A.h} L ${ca.x} 196`; + if (a === 'ports' && b === 'regs') return 'M 253 240 L 253 262'; + if (a === 'p2') return `M ${A.x + A.w / 2} ${A.y} L ${A.x + A.w / 2} 187 L 584 187 L 584 124`; + if (a === 'p4') return `M ${A.x + A.w / 2} ${A.y} L ${A.x + A.w / 2} 183 L 466 183 L 466 50 L ${B.x} 50`; + if (a === 'sb') return 'M 568 70 L 568 84'; + if (a === 'l1i' && b === 'l2') return 'M 140 84 L 140 78 L 176 78 L 176 3 L 690 3 L 690 84'; + if (a === 'l1d' && b === 'l2') return 'M 608 104 L 632 104'; + if (a === 'l2' && b === 'l3') return 'M 690 178 L 690 196'; + if (a === 'l3') return 'M 613 240 L 613 262'; + if (a === 'dtlb') return 'M 500 124 L 500 138'; + if (a === 'walker') return 'M 548 158 L 560 158 L 560 124'; + if (Math.abs(ca.y - cb.y) < 2) return `M ${A.x + A.w} ${ca.y} L ${B.x} ${cb.y}`; + if (ca.x === cb.x || (a === 'rename' || a === 'rob')) return `M ${ca.x} ${A.y + A.h} L ${ca.x} ${B.y}`; + return `M ${A.x + A.w} ${ca.y} L ${B.x} ${cb.y}`; + }; + for (const [a, b] of CHIP_EDGES) { + const p = sv('path', { d: edgePath(a, b), class: 'a86-chip-edge' }); + edgeEls[`${a}>${b}`] = p; + add(svgEl, p); + } + for (const [key, n] of Object.entries(CHIP)) { + const g = sv('g', { class: `a86-chip-node is-${n.group}` }); + add(g, sv('rect', { x: n.x, y: n.y, width: n.w, height: n.h, rx: 4 })); + if (n.group === 'port') { + add(g, sv('text', { x: n.x + n.w / 2, y: n.y + 18, 'text-anchor': 'middle', class: 'a86-chip-label', text: n.label })); + const units = PORTS[key].units.filter((u) => !u.startsWith('vec-') && !u.startsWith('lea-')); + add(g, sv('title', { text: `${PORTS[key].label}: ${PORTS[key].units.join(', ')}` })); + add(g, sv('text', { x: n.x + n.w / 2, y: n.y + 33, 'text-anchor': 'middle', class: 'a86-chip-sub', text: PORT_SUB[key] })); + } else { + add(g, sv('text', { x: n.x + n.w / 2, y: n.y + (n.sub ? 17 : 24), 'text-anchor': 'middle', class: 'a86-chip-label', text: n.label })); + if (n.sub) add(g, sv('text', { x: n.x + n.w / 2, y: n.y + 31, 'text-anchor': 'middle', class: 'a86-chip-sub', text: n.sub })); + } + nodeEls[key] = g; + add(svgEl, g); + } + const caption = el('p', { class: 'a86-chip-caption' }); + + function update(frame) { + for (const g of Object.values(nodeEls)) g.classList.remove('is-active', 'is-hit', 'is-miss', 'is-load', 'is-store', 'is-bad', 'is-far'); + for (const p of Object.values(edgeEls)) { p.classList.remove('is-active', 'is-load', 'is-store', 'is-back'); p.removeAttribute('marker-end'); p.removeAttribute('marker-start'); } + edgeEls['ports>regs'].setAttribute('d', edgePath('ports', 'regs')); + if (!frame.last) { caption.textContent = 'Step forward to light up the parts of the core the next instruction uses.'; return; } + const k = frame.t - 1; + const step = frame.last; + const mem = hw.memory(); + const core = hw.core(); + const rec = mem.steps[k]; + const inst = core.insts[k]; + const on = (key, cls = 'is-active') => nodeEls[key]?.classList.add('is-active', cls); + // An edge is drawn from a to b. `back` marks data that travels from b to a: what a load or a fetch brings back + // from further down the memory hierarchy. The arrowhead goes on the end the data reaches. + const edge = (a, b, cls = 'is-active', back = false) => { + const p = edgeEls[`${a}>${b}`]; + if (!p) return; + p.classList.add('is-active', cls); + if (back) p.classList.add('is-back'); + p.setAttribute(back ? 'marker-start' : 'marker-end', `url(#${uid}-${cls.replace('is-', '')})`); + }; + const say = []; + + // fetch + on('decode'); edge('decode', 'rename'); on('rename'); + if (rec.fetch) { + const f = rec.fetch; + const deep = f.level; + on('l1i', deep === 'L1i' ? 'is-hit' : 'is-miss'); edge('l1i', 'decode'); + if (deep !== 'L1i') { edge('l1i', 'l2', 'is-active', true); on('l2', deep === 'L2' ? 'is-hit' : 'is-miss'); if (deep === 'L3' || deep === 'memory') { edge('l2', 'l3', 'is-active', true); on('l3', deep === 'L3' ? 'is-hit' : 'is-miss'); } if (deep === 'memory') { edge('l3', 'dram', 'is-active', true); on('dram', 'is-far'); } } + say.push(deep === 'L1i' ? `its ${f.size} bytes came from the L1i cache` : `its ${f.size} bytes missed the L1i cache and came from ${deep === 'memory' ? 'memory' : deep}, ${f.latency} cycles away`); + if (f.translation && !f.translation.tlbHit) { on('itlb', 'is-miss'); say.push('the iTLB missed, and the page walker ran first'); } + else if (f.translation) on('itlb', 'is-hit'); + } + if (step.branch?.kind === 'conditional' || step.branch?.kind === 'return') { + const bad = step.prediction && !step.prediction.correct; + on('bp', bad ? 'is-bad' : 'is-hit'); edge('bp', 'l1i'); + say.push(bad ? 'the branch predictor guessed wrong, and everything fetched after the branch was thrown away' : 'the branch predictor guessed right'); + } + if (step.syscall) { on('os', 'is-active'); say.push(`the kernel ran ${step.syscall.name}`); } + + const us = inst.fusedInto !== null && inst.fusedInto !== undefined ? [core.uops[inst.fusedInto]] : inst.uops.map((id) => core.uops[id]); + const real = us.filter((u) => u.ports.length); + if (!real.length && us.length && us.every((u) => u.kind === 'eliminated')) say.push('rename handled it completely: no execution unit was needed'); + if (real.length) { on('rob'); edge('rename', 'rob'); on('rs'); edge('rob', 'rs'); edge('rs', 'ports'); } + const portsUsed = [...new Set(real.map((u) => u.port).filter(Boolean))]; + for (const p of portsUsed) on(p); + if (portsUsed.length) say.push(`${real.length === 1 ? 'its micro-op ran' : `its ${real.length} micro-ops ran`} on ${portsUsed.join(', ')}`); + if (inst.fusedInto !== null && inst.fusedInto !== undefined) say.push('it was fused with the compare before it into one micro-op'); + else if (us.some((u) => u.fused)) say.push('its micro-op also carries the conditional jump after it, because the two were fused'); + + for (const [j, d] of rec.data.entries()) { + const isWrite = d.kind === 'write'; + if (isWrite) { + on('sb', 'is-store'); edge('p4', 'sb', 'is-store'); edge('sb', 'l1d', 'is-store'); on('l1d', d.lines[0].trail[0].hit ? 'is-hit' : 'is-miss'); + if (j === rec.data.length - 1 || !say.some((x) => x.startsWith('the store'))) say.push(`the store went into the store buffer, to reach the L1d cache later`); + } else { + // the load's data comes back up: from wherever it was found, through each level above, to the load port + edge('p2', 'l1d', 'is-load', true); + const lvl = d.level; + on('l1d', lvl === 'L1d' ? 'is-hit' : 'is-miss'); + if (lvl !== 'L1d') { edge('l1d', 'l2', 'is-load', true); on('l2', lvl === 'L2' ? 'is-hit' : 'is-miss'); } + if (lvl === 'L3' || lvl === 'memory') { edge('l2', 'l3', 'is-load', true); on('l3', lvl === 'L3' ? 'is-hit' : 'is-miss'); } + if (lvl === 'memory') { edge('l3', 'dram', 'is-load', true); on('dram', 'is-far'); } + const fwd = real.find((u) => u.forwarded); + say.push(fwd ? 'the load was answered by the store buffer, from a store still on its way to the cache' : `the load ${lvl === 'L1d' ? 'hit in the L1d cache' : `missed L1d and was served by ${lvl === 'memory' ? 'memory' : lvl}`} after ${d.latency} cycles`); + } + if (d.translation) { + if (d.translation.tlbHit) on('dtlb', 'is-hit'); + else { on('dtlb', 'is-miss'); on('walker'); edge('dtlb', 'walker'); edge('walker', 'l1d', 'is-active', true); say.push('the dTLB missed and the page walker read four table entries'); } + } + } + const writesVec = step.diff.regs.some((d) => d.vector); + const writesInt = step.diff.regs.some((d) => !d.vector) || step.diff.flags.length; + if (writesInt) on('regs'); + if (writesVec) on('vregs'); + if ((writesInt || writesVec) && real.length) { + // the result goes into the register file from the port that produced it, not from a fixed place + const from = CHIP[portsUsed[portsUsed.length - 1]] ?? null; + const to = CHIP[writesInt ? 'regs' : 'vregs']; + if (from) { + const px = from.x + from.w / 2; + const inside = px >= to.x + 8 && px <= to.x + to.w - 8; + const tx = inside ? px : Math.min(Math.max(px, to.x + 12), to.x + to.w - 12); + edgeEls['ports>regs'].setAttribute('d', inside ? `M ${px} ${from.y + from.h} L ${px} ${to.y}` : `M ${px} ${from.y + from.h} L ${px} ${from.y + from.h + 11} L ${tx} ${from.y + from.h + 11} L ${tx} ${to.y}`); + } + edge('ports', 'regs'); + } + caption.textContent = `${step.instr.text}: ${say.join('; ')}.`; + } + + return { + node: panel(options.title ?? 'The whole core', el('div', {}, [svgEl, caption, legend([['is-chip-active', 'used by this instruction'], ['is-chip-hit', 'cache hit or correct guess'], ['is-chip-miss', 'miss, or a wrong guess'], ['is-chip-load', 'a load’s path'], ['is-chip-store', 'a store’s path']], 'a86-chip-legend'), machineNote(hw)]), 'a86-chip-panel'), + update, + }; +} + +export const HW_VIEWS = { + encoding: encodingView, + bits: bitsView, + stack: stackView, + cache: cacheView, + hierarchy: hierarchyView, + paging: pagingView, + pipeline: pipelineView, + uops: uopsView, + timeline: timelineView, + bottleneck: bottleneckView, + core: coreView, + console: consoleView, + perf: perfView, + machine: machineView, +}; diff --git a/src/engines/asm86/interp.js b/src/engines/asm86/interp.js new file mode 100644 index 0000000..0983331 --- /dev/null +++ b/src/engines/asm86/interp.js @@ -0,0 +1,2591 @@ +/** + * asm86/interp.js + * + * A small x86-64 interpreter for teaching. It covers the instruction subset + * used in the course's x86-64 reference page, so a student who has read that + * page can read this file too. + * + * This file defines what the machine *does*: the architectural state after + * every instruction, which is all a program can observe. It executes one + * instruction at a time, in order, and records a full snapshot after each. + * + * How long each instruction takes is a separate question, answered by the + * hardware models in machine.js. They read the trace this file records and + * replay it through caches, a pipeline, and an out-of-order core. Keeping the + * two apart mirrors the real split between an instruction set and the + * processors that implement it: every model must agree with this file about + * results, and may disagree with each other about time. + * + * Public API + * assemble(source) -> program (throws AsmError on a bad line) + * run(spec) -> trace + * regValue(state, name) -> BigInt (any register width) + * readByte(state, addr) -> Number 0..255 + * formatHex(v, bytes) -> String + * + * No dependencies beyond encode.js. Works in a browser, in Observable JS, + * and in Node. + */ + +import { layoutProgram } from './encode.js'; + +/* ------------------------------------------------------------------ */ +/* Registers */ +/* ------------------------------------------------------------------ */ + +/** In hardware encoding order: rax is register 0, rcx is 1, and so on. */ +export const REG64 = [ + 'rax', 'rcx', 'rdx', 'rbx', 'rsp', 'rbp', 'rsi', 'rdi', + 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', +]; + +const NAMES32 = ['eax', 'ecx', 'edx', 'ebx', 'esp', 'ebp', 'esi', 'edi']; +const NAMES16 = ['ax', 'cx', 'dx', 'bx', 'sp', 'bp', 'si', 'di']; +const NAMES8 = ['al', 'cl', 'dl', 'bl', 'spl', 'bpl', 'sil', 'dil']; +/** Bits 8 to 15 of the first four registers, a survivor from the 8086. */ +const NAMES8H = ['ah', 'ch', 'dh', 'bh']; + +/** The vector registers, each holding sixteen bytes side by side. */ +export const XMM = Array.from({ length: 16 }, (_, i) => `xmm${i}`); + +/** Every named slot the machine keeps, general-purpose and vector alike. */ +export const ALL_SLOTS = [...REG64, ...XMM]; + +/** name -> { base: 'rax', size: 1|2|4|8|16, high?: true, vector?: true } */ +export const REGS = (() => { + const t = {}; + REG64.forEach((r, i) => { + t[r] = { base: r, size: 8 }; + if (i < 8) { + t[NAMES32[i]] = { base: r, size: 4 }; + t[NAMES16[i]] = { base: r, size: 2 }; + t[NAMES8[i]] = { base: r, size: 1 }; + if (i < 4) t[NAMES8H[i]] = { base: r, size: 1, high: true }; + } else { + t[r + 'd'] = { base: r, size: 4 }; + t[r + 'w'] = { base: r, size: 2 }; + t[r + 'b'] = { base: r, size: 1 }; + } + }); + for (const x of XMM) t[x] = { base: x, size: 16, vector: true }; + return t; +})(); + +/** Every name for one 64-bit register, widest first: rax, eax, ax, al (and ah). */ +export function namesOf(base) { + const i = REG64.indexOf(base); + if (i < 0) return [base]; + if (i < 8) return [base, NAMES32[i], NAMES16[i], NAMES8[i], ...(i < 4 ? [NAMES8H[i]] : [])]; + return [base, `${base}d`, `${base}w`, `${base}b`]; +} + +export const MASK = { + 1: 0xffn, 2: 0xffffn, 4: 0xffffffffn, 8: 0xffffffffffffffffn, + 16: 0xffffffffffffffffffffffffffffffffn, +}; +export const SIGNBIT = { + 1: 0x80n, 2: 0x8000n, 4: 0x80000000n, 8: 0x8000000000000000n, + 16: 0x80000000000000000000000000000000n, +}; + +/** Split a 128-bit value into its sixteen bytes, lane 0 lowest. */ +export function lanes(value) { + const out = []; + let v = BigInt(value) & MASK[16]; + for (let k = 0; k < 16; k++) { out.push(Number(v & 0xffn)); v >>= 8n; } + return out; +} + +/** Rebuild a 128-bit value from sixteen bytes. */ +export function fromLanes(bytes) { + let v = 0n; + for (let k = 15; k >= 0; k--) v = (v << 8n) | BigInt(bytes[k] & 0xff); + return v; +} + +/** Apply a function to each of the sixteen byte lanes of two vectors. */ +function perLane(a, b, fn) { + const la = lanes(a); const lb = lanes(b); + return fromLanes(la.map((x, k) => fn(x, lb[k]) & 0xff)); +} + +/** Read a value as signed at a given width. */ +export function toSigned(v, width) { + let x = BigInt(v) & MASK[width]; + if (x & SIGNBIT[width]) x -= MASK[width] + 1n; + return x; +} + +/** Read a register at any width. */ +export function regValue(state, name) { + const r = REGS[name.toLowerCase()]; + if (!r) throw new AsmError(`unknown register: ${name}`); + if (r.high) return (state.regs[r.base] >> 8n) & 0xffn; + return state.regs[r.base] & MASK[r.size]; +} + +/** + * Write a register at any width, following the real x86-64 rules: + * a 4-byte write clears the upper 32 bits; a 1- or 2-byte write leaves + * the upper bits alone. That asymmetry surprises everyone once. + */ +function regWrite(state, name, value) { + const r = REGS[name.toLowerCase()]; + const v = BigInt(value) & MASK[r.size]; + if (r.high) { + state.regs[r.base] = (state.regs[r.base] & ~0xff00n) | (v << 8n); + } else if (r.size >= 8) state.regs[r.base] = v; // 8 bytes, or a whole vector + else if (r.size === 4) state.regs[r.base] = v; // zero-extends + else { + const keep = state.regs[r.base] & ~MASK[r.size]; + state.regs[r.base] = keep | v; + } +} + +/** + * An error the machine or the assembler raises. `kind` names the hardware + * event where there is one, so a view can say "page fault" rather than just + * "error": 'assemble', 'page-fault', 'divide-error', 'bad-return', + * 'alignment', 'general-protection'. + */ +export class AsmError extends Error { + constructor(message, kind = 'assemble', extra = {}) { + super(message); + this.kind = kind; + Object.assign(this, extra); + } +} + +/* ------------------------------------------------------------------ */ +/* Memory */ +/* ------------------------------------------------------------------ */ + +/** + * Memory is a list of named regions, each a contiguous run of bytes at a + * base address. Regions are shared between snapshots and cloned only when + * written, so stepping through a long scan stays cheap. + * + * An address outside every region is an address the operating system never + * mapped. Touching one is what a segmentation fault is, and the machine + * reports it as a page fault, which is the hardware event underneath. + */ +/** + * The most bytes one region may hold, and the most bytes a run may copy while writing. Every step that writes a + * region keeps a copy of it for the snapshots, so a region of a megabyte written at each of 20,000 steps would keep + * twenty gigabytes; a setup from a share link names the sizes (tests/reader-limits.test.mjs, docs/limits.md). + */ +const MAX_REGION = 1 << 20; +const MAX_COPIED = 128 << 20; + +function makeMemory(regionSpecs) { + for (const r of regionSpecs) { + const n = r.size ?? (r.text != null ? String(r.text).length : r.bytes != null ? r.bytes.length : 0); + if (!Number.isSafeInteger(n) || n < 0 || n > MAX_REGION) { + throw new AsmError(`the memory region ${r.label || r.name || ''} asks for ${n} bytes; this machine maps regions of 0 to ${MAX_REGION.toLocaleString('en-US')} bytes (1 MiB)`, 'limit'); + } + } + const regions = regionSpecs.map((r) => { + let bytes; + if (r.text != null) bytes = Uint8Array.from([...String(r.text)].map((c) => c.charCodeAt(0) & 0xff)); + else if (r.bytes != null) bytes = Uint8Array.from(r.bytes); + else bytes = new Uint8Array(r.size || 0); + if (r.size != null && r.text != null && r.size > bytes.length) { + const padded = new Uint8Array(r.size); padded.set(bytes); bytes = padded; + } + return { + name: r.name || r.label || 'mem', + label: r.label || r.name || 'mem', + addr: Number(r.addr), + kind: r.kind || (r.text != null ? 'text' : 'bytes'), + hidden: r.kind === 'stack', + writable: r.writable !== false, + bytes, + }; + }); + return regions; +} + +function findRegion(regions, addr) { + for (let i = 0; i < regions.length; i++) { + const r = regions[i]; + if (addr >= r.addr && addr < r.addr + r.bytes.length) return i; + } + return -1; +} + +export function readByte(state, addr) { + const i = findRegion(state.mem, addr); + if (i < 0) throw new AsmError(`page fault: read from ${formatHex(BigInt(addr), 8)}, an address no region maps`, 'page-fault', { addr, access: 'read' }); + return state.mem[i].bytes[addr - state.mem[i].addr]; +} + +function writeByte(state, addr, byte) { + const i = findRegion(state.mem, addr); + if (i < 0) throw new AsmError(`page fault: write to ${formatHex(BigInt(addr), 8)}, an address no region maps`, 'page-fault', { addr, access: 'write' }); + if (!state.mem[i].writable) throw new AsmError(`page fault: write to read-only region "${state.mem[i].label}" at ${formatHex(BigInt(addr), 8)}`, 'page-fault', { addr, access: 'write' }); + // copy-on-write so earlier snapshots keep their values + if (!state.dirty.has(i)) { + state.copied = (state.copied || 0) + state.mem[i].bytes.length; + if (state.copied > MAX_COPIED) throw new AsmError(`stopped: the steps so far wrote to memory regions totaling ${MAX_COPIED >> 20} MiB, each step keeping its own copy for stepping back; write to a smaller region, or take fewer steps`, 'limit'); + state.mem = state.mem.slice(); + state.mem[i] = { ...state.mem[i], bytes: state.mem[i].bytes.slice() }; + state.dirty.add(i); + } + state.mem[i].bytes[addr - state.mem[i].addr] = byte & 0xff; +} + +function readSize(state, addr, size) { + let v = 0n; + for (let k = size - 1; k >= 0; k--) v = (v << 8n) | BigInt(readByte(state, addr + k)); + return v; +} + +function writeSize(state, addr, size, value) { + let v = BigInt(value) & MASK[size]; + for (let k = 0; k < size; k++) { + writeByte(state, addr + k, Number(v & 0xffn)); + v >>= 8n; + } +} + +/* ------------------------------------------------------------------ */ +/* Assembler */ +/* ------------------------------------------------------------------ */ + +const SIZE_WORDS = { byte: 1, word: 2, dword: 4, qword: 8, xmmword: 16, oword: 16 }; +const SIZE_NAME = { 1: 'byte', 2: 'word', 4: 'dword', 8: 'qword', 16: 'xmmword' }; + +function parseImmediate(tok) { + const t = tok.trim(); + if (/^'(.)'$/.test(t)) return BigInt(t.charCodeAt(1)); + if (/^"(.)"$/.test(t)) return BigInt(t.charCodeAt(1)); + if (/^-?0[bxo][0-9a-f_]+$/i.test(t)) { + const neg = t.startsWith('-'); + const v = BigInt((neg ? t.slice(1) : t).replace(/_/g, '')); + return neg ? -v : v; + } + if (/^-?[0-9a-f]+h$/i.test(t) && /^-?[0-9]/.test(t)) { + // 0ffh, as some listings write hexadecimal + const neg = t.startsWith('-'); + const v = BigInt('0x' + (neg ? t.slice(1) : t).slice(0, -1)); + return neg ? -v : v; + } + if (/^-?\d[\d_]*$/.test(t)) return BigInt(t.replace(/_/g, '')); + return null; +} + +/** `byte ptr [rdi + rcx*1 + 8]` and every simpler form. */ +function parseMemory(text) { + let size = null; + let s = text.trim(); + const pre = /^(byte|word|dword|qword|xmmword|oword)\s*(?:ptr)?\s*/i.exec(s); + if (pre) { + size = SIZE_WORDS[pre[1].toLowerCase()]; + s = s.slice(pre[0].length); + } + const inner = /^\[(.*)\]$/.exec(s.trim()); + if (!inner) return null; + const body = inner[1]; + const mem = { kind: 'mem', size, sizeGiven: size != null, base: null, index: null, scale: 1, disp: 0n, source: text.trim() }; + const parts = body.replace(/-/g, '+-').split('+').map((p) => p.trim()).filter((p) => p.length); + for (const part of parts) { + const neg = part.startsWith('-'); + const p = (neg ? part.slice(1) : part).trim(); + const scaled = /^([a-z][a-z0-9]*)\s*\*\s*(\d+)$/i.exec(p) || /^(\d+)\s*\*\s*([a-z][a-z0-9]*)$/i.exec(p); + if (scaled) { + const [reg, sc] = REGS[scaled[1].toLowerCase()] ? [scaled[1], scaled[2]] : [scaled[2], scaled[1]]; + if (REGS[reg.toLowerCase()]) { + if (mem.index !== null) throw new AsmError(`an address can hold only one scaled register: ${text}`); + mem.index = reg.toLowerCase(); + mem.scale = Number(sc); + continue; + } + } + if (REGS[p.toLowerCase()] || p.toLowerCase() === 'rip') { + if (neg) throw new AsmError(`an address can add a register, never subtract one: ${text}`); + if (mem.base === null) mem.base = p.toLowerCase(); + else if (mem.index === null) { mem.index = p.toLowerCase(); mem.scale = 1; } + else throw new AsmError(`too many registers in address: ${text}`); + continue; + } + const imm = parseImmediate(p); + if (imm === null) throw new AsmError(`cannot read address part "${p}" in ${text}`); + mem.disp += neg ? -imm : imm; + } + // [rcx*1 + rdi] names the same address as [rdi + rcx*1]; an unscaled + // lone index register is really a base. + if (mem.base === null && mem.index !== null && mem.scale === 1 && !/\*/.test(body)) { + mem.base = mem.index; mem.index = null; + } + return mem; +} + +function parseOperand(text) { + const t = text.trim(); + const lower = t.toLowerCase(); + if (REGS[lower]) return { kind: 'reg', name: lower, size: REGS[lower].size, source: t }; + if (/\[/.test(t)) { + const mem = parseMemory(t); + if (mem) return mem; + } + const imm = parseImmediate(t); + if (imm !== null) return { kind: 'imm', value: imm, source: t }; + if (/^ymm\d+$/.test(lower)) throw new AsmError(`${lower} is a 256-bit AVX2 register; the model machine has the sixteen 128-bit xmm registers and no ymm`); + if (!/^[.\w$]+$/.test(t)) throw new AsmError(`cannot read operand "${t}"`); + return { kind: 'label', name: t, source: t }; +} + +/** Split an operand list on commas that sit outside brackets and quotes. */ +function splitOperands(s) { + const out = []; + let depth = 0; + let quote = null; + let cur = ''; + for (const ch of s) { + if (quote) { cur += ch; if (ch === quote) quote = null; continue; } + if (ch === "'" || ch === '"') { quote = ch; cur += ch; continue; } + if (ch === '[') depth++; + if (ch === ']') depth--; + if (ch === ',' && depth === 0) { out.push(cur); cur = ''; continue; } + cur += ch; + } + if (cur.trim().length) out.push(cur); + return out.map((x) => x.trim()).filter((x) => x.length); +} + +/** Strip a comment, leaving a ';' inside a character constant alone. */ +export function splitComment(raw) { + let quote = null; + for (let i = 0; i < raw.length; i++) { + const ch = raw[i]; + if (quote) { if (ch === quote) quote = null; continue; } + if (ch === "'" || ch === '"') { quote = ch; continue; } + if (ch === ';' || ch === '#' || (ch === '/' && raw[i + 1] === '/')) { + return { code: raw.slice(0, i), comment: raw.slice(i).replace(/^(;|#|\/\/)\s*/, '') }; + } + } + return { code: raw, comment: '' }; +} + +/** + * Turn assembly text into a program: a flat list of instructions plus a + * label table. Labels are indexes into the instruction list; encode.js + * later gives every instruction a byte address as well. + * + * Every instruction is checked against the rules the real instruction set + * imposes, so a line that no x86-64 assembler would accept is refused here + * too, with a sentence saying which rule it broke. + */ +export function assemble(source) { + const rawLines = String(source).replace(/\r/g, '').split('\n'); + const instrs = []; + const labels = {}; + const lines = []; + const pendingLabels = []; + + rawLines.forEach((raw, lineNo) => { + const { code, comment } = splitComment(raw); + const text = code.trim(); + const record = { lineNo, raw, comment, kind: 'blank', instrIndex: null, labels: [] }; + lines.push(record); + if (!text.length) return; + + let rest = text; + let labelMatch; + while ((labelMatch = /^([.\w$]+)\s*:\s*/.exec(rest))) { + const name = labelMatch[1]; + if (name in labels) throw new AsmError(`line ${lineNo + 1}: the label "${name}" is defined twice`, 'assemble', { line: lineNo }); + labels[name] = instrs.length; + pendingLabels.push(name); + record.labels.push(name); + record.kind = 'label'; + rest = rest.slice(labelMatch[0].length).trim(); + } + if (!rest.length) return; + + // `lock` makes a read-modify-write atomic across cores; the model has one core, so it is noted and nothing more + let locked = false; + const lk = /^lock\s+/i.exec(rest); + if (lk) { locked = true; rest = rest.slice(lk[0].length); } + const m = /^([a-z][a-z0-9]*)\s*(.*)$/i.exec(rest); + if (!m) throw new AsmError(`line ${lineNo + 1}: cannot read "${text}"`, 'assemble', { line: lineNo }); + const op = m[1].toLowerCase(); + if (!OPS[op]) throw new AsmError(`line ${lineNo + 1}: this guide's subset has no instruction "${op}"${/^v/.test(op) && /ymm/i.test(m[2]) ? ' (it works on ymm registers: 256-bit AVX2, which the model machine does not have; it has the sixteen 128-bit xmm registers)' : UNMODELED[op] ? ` (${UNMODELED[op]})` : ''}`, 'assemble', { line: lineNo }); + if (locked && !OPS[op].cls.lockable) throw new AsmError(`line ${lineNo + 1}: lock goes only on a read-modify-write of memory (xadd, cmpxchg, add, sub, and, or, xor, inc, dec, neg, not, xchg, bts, btr, btc)`, 'assemble', { line: lineNo }); + let operands; + try { + operands = splitOperands(m[2]).map(parseOperand); + validate(op, operands); + } catch (err) { + throw new AsmError(`line ${lineNo + 1}: ${err.message}`, 'assemble', { line: lineNo }); + } + const instr = { + index: instrs.length, + op, + operands, + lineNo, + text: rest, + comment, + labelsHere: pendingLabels.slice(), + }; + pendingLabels.length = 0; + record.kind = 'instr'; + record.instrIndex = instrs.length; + instrs.push(instr); + }); + + for (const ins of instrs) { + for (const o of ins.operands) { + if (o.kind === 'label' && !(o.name in labels)) { + throw new AsmError(`line ${ins.lineNo + 1}: no label named "${o.name}"`, 'assemble', { line: ins.lineNo }); + } + } + } + return { instrs, labels, lines, source: String(source) }; +} + +/* ------------------------------------------------------------------ */ +/* The instruction set's own rules */ +/* ------------------------------------------------------------------ */ + +/** Does v fit in `width` bytes, read either as signed or as unsigned? */ +function fitsEither(v, width) { + const bits = BigInt(width * 8); + return v >= -(1n << (bits - 1n)) && v < (1n << bits); +} +function fitsSigned(v, bits) { + return v >= -(1n << BigInt(bits - 1)) && v < (1n << BigInt(bits - 1)); +} + +const isReg = (o) => o && o.kind === 'reg' && !REGS[o.name].vector; +const isVec = (o) => o && o.kind === 'reg' && REGS[o.name].vector; +const isMem = (o) => o && o.kind === 'mem'; +const isImm = (o) => o && o.kind === 'imm'; +const isRM = (o) => isReg(o) || isMem(o); + +function fail(msg) { throw new AsmError(msg); } + +function checkAddress(o) { + for (const r of [o.base, o.index]) { + if (r == null || r === 'rip') continue; + const info = REGS[r]; + if (info.vector) fail(`${r} cannot appear inside an address; addresses are built from ordinary registers`); + if (info.size !== 8) fail(`${r} is ${info.size * 8} bits wide; build addresses from 64-bit registers such as ${info.base}`); + } + if (o.index === 'rip' || (o.base === 'rip' && o.index)) fail('rip-relative addresses take only a displacement: [rip + label or number]'); + if (o.index === 'rsp') fail('rsp can be the base of an address, never the index; the encoding has no room for it'); + if (![1, 2, 4, 8].includes(o.scale)) fail(`the scale must be 1, 2, 4, or 8, not ${o.scale}`); + if (!fitsSigned(o.disp, 32)) fail(`a displacement must fit in 32 signed bits; ${o.disp} does not`); +} + +/** Settle the width of every operand, and refuse anything the ISA cannot encode. */ +function validate(op, ops) { + const def = OPS[op]; + const arity = def.arity ?? [0, 3]; + const [lo, hi] = Array.isArray(arity) ? arity : [arity, arity]; + if (ops.length < lo || ops.length > hi) { + fail(lo === hi ? `${op} takes ${lo} operand${lo === 1 ? '' : 's'}, not ${ops.length}` : `${op} takes ${lo} to ${hi} operands, not ${ops.length}`); + } + if (ops.filter(isMem).length > 1) fail('at most one operand may be in memory; load one of them into a register first'); + for (const o of ops) if (isMem(o)) checkAddress(o); + const labelsOk = def.cls.control && op !== 'ret'; + for (const o of ops) { + if (o.kind === 'label' && !labelsOk) fail(`"${o.name}" is not a register, a number, or an address; only jumps and calls take a label`); + } + if (labelsOk && op !== 'ret' && op !== 'syscall' && !def.cls.trap && !(ops[0]?.kind === 'label')) { + fail(`${op} needs a label to go to`); + } + // A REX prefix is what reaches r8-r15, spl-dil, and 64-bit operand size. + // Its presence repurposes the encodings of ah, ch, dh, and bh, so the two + // can never meet in one instruction. + const high = ops.some((o) => o.kind === 'reg' && REGS[o.name].high); + const extended = (r) => /^r\d/.test(r); + const needsRex = ops.some((o) => (o.kind === 'reg' && !REGS[o.name].vector + && (extended(o.name) || /^(spl|bpl|sil|dil)$/.test(o.name) || REGS[o.name].size === 8)) + || (isVec(o) && Number(o.name.slice(3)) >= 8) + || (isMem(o) && [o.base, o.index].some((r) => r && extended(r)))); + if (high && needsRex) fail('ah, bh, ch, and dh cannot appear in an instruction that needs a REX prefix, which is any with 64-bit operands or naming r8 to r15, spl, bpl, sil, or dil: that prefix takes over the encodings of the old high bytes'); + + if (def.check) def.check(ops, op); +} + +/** + * The shared rule for two-operand integer instructions: same width on both + * sides, the width settled by a register or a size word, and a constant + * that fits. + */ +function checkBinary(ops, op, { allowImm = true, destMem = true, imm64 = false } = {}) { + const [d, s] = ops; + if (isVec(d) || isVec(s)) fail(`${op} works on ordinary registers; xmm registers take the vector instructions`); + if (!isRM(d)) fail(`the first operand of ${op} must be a register or a memory address`); + if (!destMem && isMem(d)) fail(`the first operand of ${op} must be a register`); + if (isImm(s) && !allowImm) fail(`${op} cannot take a constant as its second operand`); + if (!isRM(s) && !isImm(s)) fail(`cannot read the second operand of ${op}`); + const regSize = [d, s].find(isReg)?.size; + const memSize = [d, s].find((o) => isMem(o) && o.size)?.size; + if (isReg(d) && isReg(s) && d.size !== s.size) fail(`${d.name} is ${d.size * 8} bits and ${s.name} is ${s.size * 8}; both operands must be the same width`); + if (regSize && memSize && regSize !== memSize) fail(`${SIZE_NAME[memSize]} ptr does not match ${[d, s].find(isReg).name}, which is ${regSize * 8} bits`); + const width = regSize || memSize; + if (!width) fail(`nothing says how many bytes to use; write byte ptr, word ptr, dword ptr, or qword ptr before the address`); + for (const o of ops) if (isMem(o)) o.size = width; + if (memSize === 16 || regSize === 16) fail(`${op} works on at most eight bytes`); + if (isImm(s)) { + if (!fitsEither(s.value, width)) fail(`${s.source} does not fit in ${width * 8} bits`); + if (width === 8 && !fitsSigned(s.value, 32) && !(imm64 && isReg(d))) { + fail(imm64 + ? `only a register can receive a full 64-bit constant; ${s.source} is too wide to store to memory directly` + : `no x86-64 instruction ${op}s a 64-bit constant; ${s.source} does not fit in 32 signed bits; mov it into a register first`); + } + } + return width; +} + +function checkUnary(ops, op) { + const [d] = ops; + if (isVec(d)) fail(`${op} works on ordinary registers`); + if (!isRM(d)) fail(`${op} needs a register or a memory address`); + if (isMem(d) && !d.size) fail(`nothing says how many bytes ${op} should use; write byte ptr, word ptr, dword ptr, or qword ptr`); + if (isMem(d) && d.size === 16) fail(`${op} works on at most eight bytes`); +} + +function checkShift(ops, op) { + const [d, n] = ops; + checkUnary([d], op); + if (ops.length === 1) return; + if (isImm(n)) { if (n.value < 0n || n.value > 255n) fail(`a shift count must be 0 to 255`); return; } + if (!(isReg(n) && n.name === 'cl')) fail(`${op} shifts by a constant or by cl, and by nothing else`); +} + +function checkDestReg(ops, op, sizes) { + if (!isReg(ops[0])) fail(`the first operand of ${op} must be an ordinary register`); + if (sizes && !sizes.includes(ops[0].size)) fail(`${op} cannot write ${ops[0].name}; its destination must be ${sizes.map((s) => s * 8).join(', ')} bits wide`); +} + +/* ------------------------------------------------------------------ */ +/* Operand access */ +/* ------------------------------------------------------------------ */ + +/** Work out base + index*scale + displacement, and keep every part for display. */ +function effectiveAddress(state, mem) { + let baseVal = 0n; + if (mem.base === 'rip') { + if (state.rip == null) throw new AsmError('rip-relative addressing needs instruction addresses, and this program could not be encoded'); + baseVal = BigInt(state.rip); + } else if (mem.base) baseVal = regValue(state, mem.base); + const indexVal = mem.index ? regValue(state, mem.index) : 0n; + const addr = baseVal + indexVal * BigInt(mem.scale) + mem.disp; + return { + base: mem.base, baseVal, + index: mem.index, indexVal, + scale: mem.scale, disp: mem.disp, + // A Number is exact only to 2^53, which covers every address a region + // can occupy; the full 64-bit sum is kept as well, for lea. + addr: Number(addr & MASK[8]), + value: addr & MASK[8], + size: mem.size, + source: mem.source, + }; +} + +function operandWidth(operands) { + for (const o of operands) { + if (o.kind === 'reg') return o.size; + if (o.kind === 'mem' && o.size) return o.size; + } + return 8; +} + +function readOperand(state, o, width, ctx) { + if (o.kind === 'reg') return regValue(state, o.name); + if (o.kind === 'imm') return o.value & MASK[width]; + if (o.kind === 'mem') { + const ea = effectiveAddress(state, o); + const size = o.size || width; + const value = readSize(state, ea.addr, size); + ctx.accesses.push({ kind: 'read', addr: ea.addr, size, value, ea }); + ctx.effective = ea; + return value; + } + throw new AsmError(`cannot read operand ${o.source}`); +} + +function writeOperand(state, o, width, value, ctx) { + if (o.kind === 'reg') return regWrite(state, o.name, value); + if (o.kind === 'mem') { + const ea = effectiveAddress(state, o); + const size = o.size || width; + writeSize(state, ea.addr, size, value); + ctx.accesses.push({ kind: 'write', addr: ea.addr, size, value: BigInt(value) & MASK[size], ea }); + ctx.effective = ea; + return; + } + throw new AsmError(`cannot write to operand ${o.source}`); +} + +function push64(st, v, ctx, why) { + st.regs.rsp = (st.regs.rsp - 8n) & MASK[8]; + writeSize(st, Number(st.regs.rsp), 8, v); + ctx.accesses.push({ kind: 'write', addr: Number(st.regs.rsp), size: 8, value: BigInt(v) & MASK[8], ea: null, stack: why }); +} + +function pop64(st, ctx, why) { + const addr = Number(st.regs.rsp); + const v = readSize(st, addr, 8); + ctx.accesses.push({ kind: 'read', addr, size: 8, value: v, ea: null, stack: why }); + st.regs.rsp = (st.regs.rsp + 8n) & MASK[8]; + return v; +} + +/* ------------------------------------------------------------------ */ +/* Flags */ +/* ------------------------------------------------------------------ */ + +export const FLAG_NAMES = ['ZF', 'SF', 'CF', 'OF', 'PF']; + +/** The flags as the 64-bit RFLAGS register lays them out, with IF set. */ +export function rflagsValue(f) { + return 0x202n | (f.CF ? 1n : 0n) | (f.PF ? 1n << 2n : 0n) | (f.ZF ? 1n << 6n : 0n) | (f.SF ? 1n << 7n : 0n) | (f.OF ? 1n << 11n : 0n); +} + +/** PF is set when the low byte of a result holds an even number of one bits: a leftover from serial lines, and the flag a floating-point compare uses to say "unordered". */ +export function parityOf(result) { + let b = Number(BigInt(result) & 0xffn); + b ^= b >> 4; b ^= b >> 2; b ^= b >> 1; + return (b & 1) === 0; +} + +function setLogicFlags(state, result, width) { + state.flags.PF = parityOf(result); + state.flags.ZF = (result & MASK[width]) === 0n; + state.flags.SF = (result & SIGNBIT[width]) !== 0n; + state.flags.CF = false; + state.flags.OF = false; +} + +function setAddFlags(state, a, b, width, cin = 0n) { + const full = (a & MASK[width]) + (b & MASK[width]) + cin; + const res = full & MASK[width]; + state.flags.PF = parityOf(res); + state.flags.ZF = res === 0n; + state.flags.SF = (res & SIGNBIT[width]) !== 0n; + state.flags.CF = full > MASK[width]; + const sa = (a & SIGNBIT[width]) !== 0n; + const sb = (b & SIGNBIT[width]) !== 0n; + const sr = (res & SIGNBIT[width]) !== 0n; + state.flags.OF = sa === sb && sr !== sa; + return res; +} + +function setSubFlags(state, a, b, width, cin = 0n) { + const av = a & MASK[width]; + const bv = b & MASK[width]; + const res = (av - bv - cin) & MASK[width]; + state.flags.PF = parityOf(res); + state.flags.ZF = res === 0n; + state.flags.SF = (res & SIGNBIT[width]) !== 0n; + state.flags.CF = av < bv + cin; + const sa = (av & SIGNBIT[width]) !== 0n; + const sb = (bv & SIGNBIT[width]) !== 0n; + const sr = (res & SIGNBIT[width]) !== 0n; + state.flags.OF = sa !== sb && sr !== sa; + return res; +} + +/** Every condition this subset can test, as a function of the flags. */ +export const CONDITIONS = { + e: (f) => f.ZF, + z: (f) => f.ZF, + ne: (f) => !f.ZF, + nz: (f) => !f.ZF, + b: (f) => f.CF, + nae: (f) => f.CF, + c: (f) => f.CF, + ae: (f) => !f.CF, + nb: (f) => !f.CF, + nc: (f) => !f.CF, + be: (f) => f.CF || f.ZF, + na: (f) => f.CF || f.ZF, + a: (f) => !f.CF && !f.ZF, + nbe: (f) => !f.CF && !f.ZF, + l: (f) => f.SF !== f.OF, + nge: (f) => f.SF !== f.OF, + ge: (f) => f.SF === f.OF, + nl: (f) => f.SF === f.OF, + le: (f) => f.ZF || f.SF !== f.OF, + ng: (f) => f.ZF || f.SF !== f.OF, + g: (f) => !f.ZF && f.SF === f.OF, + nle: (f) => !f.ZF && f.SF === f.OF, + s: (f) => f.SF, + ns: (f) => !f.SF, + o: (f) => f.OF, + no: (f) => !f.OF, + p: (f) => f.PF, + pe: (f) => f.PF, + np: (f) => !f.PF, + po: (f) => !f.PF, +}; + +/** Which flags each condition reads, for the bit-level and dependency views. */ +export const CONDITION_FLAGS = { + e: ['ZF'], z: ['ZF'], ne: ['ZF'], nz: ['ZF'], + b: ['CF'], nae: ['CF'], c: ['CF'], ae: ['CF'], nb: ['CF'], nc: ['CF'], + be: ['CF', 'ZF'], na: ['CF', 'ZF'], a: ['CF', 'ZF'], nbe: ['CF', 'ZF'], + l: ['SF', 'OF'], nge: ['SF', 'OF'], ge: ['SF', 'OF'], nl: ['SF', 'OF'], + le: ['ZF', 'SF', 'OF'], ng: ['ZF', 'SF', 'OF'], g: ['ZF', 'SF', 'OF'], nle: ['ZF', 'SF', 'OF'], + s: ['SF'], ns: ['SF'], o: ['OF'], no: ['OF'], + p: ['PF'], pe: ['PF'], np: ['PF'], po: ['PF'], +}; + +/** Plain-English gloss for each condition, shown in the widgets. */ +export const CONDITION_TEXT = { + e: 'equal (ZF set)', z: 'zero (ZF set)', + ne: 'not equal (ZF clear)', nz: 'not zero (ZF clear)', + b: 'below, unsigned (CF set)', nae: 'below, unsigned (CF set)', c: 'carry set', + ae: 'at or above, unsigned (CF clear)', nb: 'at or above, unsigned (CF clear)', nc: 'carry clear', + be: 'below or equal, unsigned', na: 'below or equal, unsigned', + a: 'above, unsigned', nbe: 'above, unsigned', + l: 'less, signed', nge: 'less, signed', + ge: 'greater or equal, signed', nl: 'greater or equal, signed', + le: 'less or equal, signed', ng: 'less or equal, signed', + g: 'greater, signed', nle: 'greater, signed', + s: 'negative (SF set)', ns: 'not negative (SF clear)', + o: 'overflowed (OF set)', no: 'did not overflow (OF clear)', + p: 'parity even (PF set); after a floating-point compare, unordered: one side was NaN', pe: 'parity even (PF set); after a floating-point compare, unordered: one side was NaN', + np: 'parity odd (PF clear); after a floating-point compare, ordered: neither side was NaN', po: 'parity odd (PF clear); after a floating-point compare, ordered: neither side was NaN', +}; + +/* ------------------------------------------------------------------ */ +/* Instructions */ +/* ------------------------------------------------------------------ */ + +/** Record what the arithmetic unit saw, for the bit-level view. */ +function aluNote(ctx, kind, a, b, width, result, extra = {}) { + ctx.alu = { kind, a: BigInt(a) & MASK[width], b: b == null ? null : BigInt(b) & MASK[width], width, result: BigInt(result) & MASK[width], ...extra }; +} + +/** add, sub, and the rest share this: read both, combine, write the first. */ +function binaryOp(kind, combine, gloss, extra = {}) { + return { + cls: { alu: true, write: true, flags: true }, + gloss, + arity: 2, + check: (ops, op) => checkBinary(ops, op), + exec(st, ops, ctx) { + const w = operandWidth(ops); + const a = readOperand(st, ops[0], w, ctx); + const b = readOperand(st, ops[1], w, ctx); + const cin = extra.carry ? (st.flags.CF ? 1n : 0n) : 0n; + const res = combine(st, a, b, w, cin); + aluNote(ctx, kind, a, b, w, res, { cin }); + writeOperand(st, ops[0], w, res, ctx); + }, + ...extra.def, + }; +} + +function logicOp(kind, fn, gloss) { + return binaryOp(kind, (st, a, b, w) => { const r = fn(a, b) & MASK[w]; setLogicFlags(st, r, w); return r; }, gloss); +} + +/** Shifts and rotates: the count is masked to 5 bits, or 6 for 64-bit operands. */ +function shiftCount(st, ops, w, ctx) { + const raw = ops.length > 1 ? readOperand(st, ops[1], 1, ctx) : 1n; + return Number(raw & (w === 8 ? 63n : 31n)); +} + +function shiftOp(kind, gloss) { + return { + cls: { alu: true, write: true, flags: true, shift: true }, + gloss, + arity: [1, 2], + check: checkShift, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const bits = w * 8; + const a = readOperand(st, ops[0], w, ctx) & MASK[w]; + const n = shiftCount(st, ops, w, ctx); + let res = a; let out = null; + if (kind === 'shl') { + res = (a << BigInt(n)) & MASK[w]; + if (n) out = n <= bits ? Boolean((a >> BigInt(bits - n)) & 1n) : false; + } else if (kind === 'shr') { + res = a >> BigInt(n); + if (n) out = Boolean((a >> BigInt(n - 1)) & 1n); + } else if (kind === 'sar') { + const s = toSigned(a, w); + res = (s >> BigInt(n)) & MASK[w]; + if (n) out = Boolean((s >> BigInt(n - 1)) & 1n); + } else { + const r = n % bits; // rotates wrap around the operand's own width + res = r === 0 ? a : kind === 'rol' + ? ((a << BigInt(r)) | (a >> BigInt(bits - r))) & MASK[w] + : ((a >> BigInt(r)) | (a << BigInt(bits - r))) & MASK[w]; + if (n) out = kind === 'rol' ? Boolean(res & 1n) : Boolean(res & SIGNBIT[w]); + } + aluNote(ctx, kind, a, BigInt(n), w, res, { count: n, out }); + // A count of zero leaves every flag alone. Rotates touch only CF and OF. + if (n) { + st.flags.CF = out; + const msb = (res & SIGNBIT[w]) !== 0n; + if (kind === 'shl') st.flags.OF = msb !== out; + else if (kind === 'shr') st.flags.OF = (a & SIGNBIT[w]) !== 0n; + else if (kind === 'sar') st.flags.OF = false; + else if (kind === 'rol') st.flags.OF = msb !== out; + else st.flags.OF = msb !== ((res & (SIGNBIT[w] >> 1n)) !== 0n); + if (kind !== 'rol' && kind !== 'ror') { + st.flags.PF = parityOf(res); + st.flags.ZF = res === 0n; + st.flags.SF = msb; + } + } + writeOperand(st, ops[0], w, res, ctx); + }, + }; +} + +/** The accumulator pair each operand width multiplies and divides into. */ +const ACC = { + 1: { lo: 'al', hi: 'ah', wide: 'ax' }, + 2: { lo: 'ax', hi: 'dx' }, + 4: { lo: 'eax', hi: 'edx' }, + 8: { lo: 'rax', hi: 'rdx' }, +}; + +function wideDividend(st, w) { + if (w === 1) return regValue(st, 'ax'); + return (regValue(st, ACC[w].hi) << BigInt(w * 8)) | regValue(st, ACC[w].lo); +} + +function writeWide(st, w, lo, hi) { + if (w === 1) regWrite(st, 'ax', ((hi & 0xffn) << 8n) | (lo & 0xffn)); + else { regWrite(st, ACC[w].lo, lo); regWrite(st, ACC[w].hi, hi); } +} + +function bitCounter(kind, gloss) { + return { + cls: { alu: true, write: true, flags: true, bitcount: true }, + gloss, + arity: 2, + check(ops, op) { + checkDestReg(ops, op, kind === 'popcnt' || kind === 'tzcnt' || kind === 'lzcnt' || kind.startsWith('bs') ? [2, 4, 8] : null); + checkBinary(ops, op, { allowImm: false, destMem: false }); + }, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const bits = w * 8; + const v = readOperand(st, ops[1], w, ctx) & MASK[w]; + let lowest = -1; let highest = -1; let count = 0; + for (let k = 0; k < bits; k++) { + if ((v >> BigInt(k)) & 1n) { count++; if (lowest < 0) lowest = k; highest = k; } + } + let res = null; + if (kind === 'popcnt') { + res = BigInt(count); + st.flags.ZF = v === 0n; st.flags.CF = false; st.flags.OF = false; st.flags.SF = false; st.flags.PF = false; + } else if (kind === 'tzcnt') { + res = BigInt(v === 0n ? bits : lowest); + st.flags.CF = v === 0n; st.flags.ZF = res === 0n; + } else if (kind === 'lzcnt') { + res = BigInt(v === 0n ? bits : bits - 1 - highest); + st.flags.CF = v === 0n; st.flags.ZF = res === 0n; + } else { + st.flags.ZF = v === 0n; + if (v !== 0n) res = BigInt(kind === 'bsf' ? lowest : highest); + } + aluNote(ctx, kind, v, null, w, res ?? regValue(st, ops[0].name), { count, lowest, highest }); + if (res !== null) writeOperand(st, ops[0], w, res, ctx); + }, + }; +} + +/** + * Each entry describes one instruction. `cls` groups instructions for the + * datapath view: does this one use the ALU, touch memory, write a register, + * change the flags, redirect control? `arity` and `check` hold the rules the + * assembler enforces; `exec` is what the instruction does. + */ +export const OPS = { + mov: { + cls: { move: true, write: true }, + gloss: 'copy the second operand into the first', + arity: 2, + check: (ops, op) => checkBinary(ops, op, { imm64: true }), + exec(st, ops, ctx) { + const w = operandWidth(ops); + const v = readOperand(st, ops[1], w, ctx); + writeOperand(st, ops[0], w, v, ctx); + }, + }, + movzx: { + cls: { move: true, write: true, extend: true }, + gloss: 'copy a smaller value into a larger register and fill the rest with zeros', + arity: 2, + check(ops, op) { + checkDestReg(ops, op, [2, 4, 8]); + const s = ops[1]; + if (!isRM(s)) fail(`${op} copies from a register or from memory`); + if (isMem(s) && !s.size) fail(`${op} needs to know how much to read; write byte ptr or word ptr`); + if (![1, 2].includes(s.size)) fail(`${op} reads one or two bytes; for four, a plain 32-bit mov already zero-fills`); + if (s.size >= ops[0].size) fail(`${op} must widen: ${ops[0].name} is no wider than the source`); + }, + exec(st, ops, ctx) { + const src = ops[1]; + const srcWidth = src.kind === 'reg' ? src.size : (src.size || 1); + const v = readOperand(st, src, srcWidth, ctx) & MASK[srcWidth]; + writeOperand(st, ops[0], ops[0].size, v, ctx); + }, + }, + movsx: { + cls: { move: true, write: true, extend: true }, + gloss: 'copy a smaller value into a larger register and repeat its sign bit across the rest', + arity: 2, + check(ops, op) { OPS.movzx.check(ops, op); }, + exec(st, ops, ctx) { + const src = ops[1]; + const sw = src.kind === 'reg' ? src.size : (src.size || 1); + let v = readOperand(st, src, sw, ctx) & MASK[sw]; + if (v & SIGNBIT[sw]) v |= ~MASK[sw] & MASK[8]; + writeOperand(st, ops[0], ops[0].size, v, ctx); + }, + }, + movsxd: { + cls: { move: true, write: true, extend: true }, + gloss: 'copy a 32-bit value into a 64-bit register, repeating its sign bit across the top half', + arity: 2, + check(ops, op) { + checkDestReg(ops, op, [8]); + if (!isRM(ops[1])) fail(`${op} copies from a register or from memory`); + if (isMem(ops[1])) ops[1].size = ops[1].size || 4; + if (ops[1].size !== 4) fail(`${op} reads exactly four bytes`); + }, + exec(st, ops, ctx) { + const v = readOperand(st, ops[1], 4, ctx); + writeOperand(st, ops[0], 8, toSigned(v, 4) & MASK[8], ctx); + }, + }, + lea: { + cls: { alu: true, write: true, address: true }, + gloss: 'work out an address and store the address itself, reading no memory at all', + arity: 2, + check(ops, op) { + checkDestReg(ops, op, [2, 4, 8]); + if (!isMem(ops[1])) fail(`${op} needs an address in square brackets as its second operand`); + }, + exec(st, ops, ctx) { + const ea = effectiveAddress(st, ops[1]); + ctx.effective = ea; + ctx.addressOnly = true; + writeOperand(st, ops[0], ops[0].size, ea.value, ctx); + }, + }, + add: binaryOp('add', (st, a, b, w) => setAddFlags(st, a, b, w), 'add the second operand into the first'), + sub: binaryOp('sub', (st, a, b, w) => setSubFlags(st, a, b, w), 'subtract the second operand from the first'), + adc: binaryOp('add', (st, a, b, w, c) => setAddFlags(st, a, b, w, c), 'add the second operand and the carry flag into the first', { carry: true }), + sbb: binaryOp('sub', (st, a, b, w, c) => setSubFlags(st, a, b, w, c), 'subtract the second operand and the carry flag from the first', { carry: true }), + inc: { + cls: { alu: true, write: true, flags: true }, + gloss: 'add one, leaving the carry flag alone', + arity: 1, + check: checkUnary, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const a = readOperand(st, ops[0], w, ctx); + const carry = st.flags.CF; + const res = setAddFlags(st, a, 1n, w); + aluNote(ctx, 'add', a, 1n, w, res, { cin: 0n }); + writeOperand(st, ops[0], w, res, ctx); + st.flags.CF = carry; // inc leaves CF alone on real hardware + }, + }, + dec: { + cls: { alu: true, write: true, flags: true }, + gloss: 'subtract one, leaving the carry flag alone', + arity: 1, + check: checkUnary, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const a = readOperand(st, ops[0], w, ctx); + const carry = st.flags.CF; + const res = setSubFlags(st, a, 1n, w); + aluNote(ctx, 'sub', a, 1n, w, res, { cin: 0n }); + writeOperand(st, ops[0], w, res, ctx); + st.flags.CF = carry; + }, + }, + imul: { + cls: { alu: true, write: true, flags: true, multiply: true }, + gloss: 'multiply, reading both values as signed', + arity: [1, 3], + check(ops, op) { + if (ops.length === 1) return checkUnary(ops, op); + checkDestReg(ops, op, [2, 4, 8]); + if (ops.length === 2) return checkBinary(ops, op, { destMem: false }); + if (!isRM(ops[1])) fail(`the middle operand of three-operand ${op} must be a register or memory`); + if (!isImm(ops[2])) fail(`the last operand of three-operand ${op} must be a constant`); + checkBinary([ops[0], ops[1]], op, { allowImm: false, destMem: false }); + if (!fitsSigned(ops[2].value, 32) || !fitsEither(ops[2].value, ops[0].size)) fail(`${ops[2].source} does not fit`); + }, + exec(st, ops, ctx) { + const w = operandWidth(ops); + if (ops.length === 1) { + const a = toSigned(regValue(st, ACC[w].lo), w); + const b = toSigned(readOperand(st, ops[0], w, ctx), w); + const full = a * b; + const lo = full & MASK[w]; + const hi = (full >> BigInt(w * 8)) & MASK[w]; + writeWide(st, w, lo, hi); + const fits = toSigned(lo, w) === full; + st.flags.CF = !fits; st.flags.OF = !fits; + st.flags.SF = (lo & SIGNBIT[w]) !== 0n; st.flags.ZF = lo === 0n; + aluNote(ctx, 'imul', a, b, w, lo, { hi, overflow: !fits, wide: true }); + return; + } + const a = readOperand(st, ops[ops.length === 2 ? 0 : 1], w, ctx); + const b = readOperand(st, ops[ops.length === 2 ? 1 : 2], w, ctx); + const full = toSigned(a, w) * toSigned(b, w); + const res = full & MASK[w]; + const fits = toSigned(res, w) === full; + // CF and OF report whether the signed product fitted; ZF and SF are + // officially undefined and are set from the result here. + st.flags.CF = !fits; st.flags.OF = !fits; + st.flags.ZF = res === 0n; st.flags.SF = (res & SIGNBIT[w]) !== 0n; + aluNote(ctx, 'imul', a, b, w, res, { overflow: !fits }); + writeOperand(st, ops[0], w, res, ctx); + }, + }, + mul: { + cls: { alu: true, write: true, flags: true, multiply: true }, + gloss: 'multiply rax by the operand, unsigned, with the double-width answer landing in rdx:rax', + arity: 1, + check: checkUnary, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const a = regValue(st, ACC[w].lo); + const b = readOperand(st, ops[0], w, ctx); + const full = a * b; + const lo = full & MASK[w]; const hi = full >> BigInt(w * 8); + writeWide(st, w, lo, hi); + st.flags.CF = hi !== 0n; st.flags.OF = hi !== 0n; + st.flags.SF = (lo & SIGNBIT[w]) !== 0n; st.flags.ZF = lo === 0n; + aluNote(ctx, 'mul', a, b, w, lo, { hi, wide: true }); + }, + }, + div: { + cls: { alu: true, write: true, flags: true, divide: true }, + gloss: 'divide rdx:rax by the operand, unsigned: quotient to rax, remainder to rdx', + arity: 1, + check: checkUnary, + exec(st, ops, ctx) { divide(st, ops, ctx, false); }, + }, + idiv: { + cls: { alu: true, write: true, flags: true, divide: true }, + gloss: 'divide rdx:rax by the operand, signed: quotient to rax, remainder to rdx', + arity: 1, + check: checkUnary, + exec(st, ops, ctx) { divide(st, ops, ctx, true); }, + }, + cqo: { + cls: { alu: true, write: true, extend: true }, + gloss: 'fill rdx with copies of the sign bit of rax, ready for a signed division', + arity: 0, + exec(st) { regWrite(st, 'rdx', (st.regs.rax & SIGNBIT[8]) ? MASK[8] : 0n); }, + }, + cdq: { + cls: { alu: true, write: true, extend: true }, + gloss: 'fill edx with copies of the sign bit of eax, ready for a 32-bit signed division', + arity: 0, + exec(st) { regWrite(st, 'edx', (st.regs.rax & SIGNBIT[4]) ? MASK[4] : 0n); }, + }, + cdqe: { + cls: { alu: true, write: true, extend: true }, + gloss: 'widen eax to rax, repeating its sign bit', + arity: 0, + exec(st) { regWrite(st, 'rax', toSigned(st.regs.rax, 4) & MASK[8]); }, + }, + and: logicOp('and', (a, b) => a & b, 'keep only the bits set in both operands'), + or: logicOp('or', (a, b) => a | b, 'set every bit that is set in either operand'), + xor: logicOp('xor', (a, b) => a ^ b, 'set every bit that is set in exactly one operand; with the same register twice, this clears it'), + shl: shiftOp('shl', 'shift bits left, which doubles the value once per place; the last bit pushed out lands in CF'), + sal: null, // filled in below: sal is another name for shl + shr: shiftOp('shr', 'shift bits right, filling with zeros; the last bit pushed out lands in CF'), + sar: shiftOp('sar', 'shift bits right, repeating the sign bit, which halves a signed value'), + rol: shiftOp('rol', 'rotate bits left: whatever leaves the top comes back in at the bottom'), + ror: shiftOp('ror', 'rotate bits right: whatever leaves the bottom comes back in at the top'), + cmp: { + cls: { alu: true, flags: true, compare: true }, + gloss: 'subtract, set the flags, and throw the answer away', + arity: 2, + check: (ops, op) => checkBinary(ops, op), + exec(st, ops, ctx) { + const w = operandWidth(ops); + const a = readOperand(st, ops[0], w, ctx); + const b = readOperand(st, ops[1], w, ctx); + ctx.compared = { a, b, width: w }; + const res = setSubFlags(st, a, b, w); + aluNote(ctx, 'sub', a, b, w, res, { cin: 0n, discarded: true }); + }, + }, + test: { + cls: { alu: true, flags: true, compare: true }, + gloss: 'AND, set the flags, and throw the answer away', + arity: 2, + check: (ops, op) => checkBinary(ops, op), + exec(st, ops, ctx) { + const w = operandWidth(ops); + const a = readOperand(st, ops[0], w, ctx); + const b = readOperand(st, ops[1], w, ctx); + ctx.compared = { a, b, width: w }; + setLogicFlags(st, a & b, w); + aluNote(ctx, 'and', a, b, w, a & b, { discarded: true }); + }, + }, + bt: { + cls: { alu: true, flags: true, compare: true }, + gloss: 'copy one chosen bit into the carry flag', + arity: 2, + check(ops, op) { + if (!isReg(ops[0])) fail(`this subset's ${op} tests a bit of a register`); + if (ops[0].size === 1) fail(`${op} works on 16, 32, or 64-bit registers`); + if (isImm(ops[1])) { if (ops[1].value < 0n || ops[1].value > 255n) fail('a bit number must be 0 to 255'); return; } + checkBinary(ops, op, { destMem: false }); + }, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const a = readOperand(st, ops[0], w, ctx); + const n = Number(readOperand(st, ops[1], w, ctx) & BigInt(w * 8 - 1)); + st.flags.CF = Boolean((a >> BigInt(n)) & 1n); + aluNote(ctx, 'bt', a, BigInt(n), w, a, { count: n, out: st.flags.CF, discarded: true }); + }, + }, + stc: { cls: { flags: true }, gloss: 'set the carry flag', arity: 0, exec(st) { st.flags.CF = true; } }, + clc: { cls: { flags: true }, gloss: 'clear the carry flag', arity: 0, exec(st) { st.flags.CF = false; } }, + cmc: { cls: { flags: true }, gloss: 'flip the carry flag', arity: 0, exec(st) { st.flags.CF = !st.flags.CF; } }, + push: { + cls: { move: true, stack: true }, + gloss: 'move the stack pointer down by eight and store a value there', + arity: 1, + check(ops, op) { + const [s] = ops; + if (isImm(s)) { if (!fitsSigned(s.value, 32)) fail(`push can carry a 32-bit signed constant at most`); return; } + if (isVec(s)) fail(`${op} moves ordinary registers; xmm registers cannot be pushed`); + if (isMem(s)) s.size = s.size || 8; + if (!isRM(s) || s.size !== 8) fail(`${op} moves eight bytes in 64-bit mode; use the 64-bit name, such as ${isReg(s) ? REGS[s.name].base : 'qword ptr'}`); + }, + exec(st, ops, ctx) { + const v = readOperand(st, ops[0], 8, ctx); + push64(st, ops[0].kind === 'imm' ? BigInt.asUintN(64, ops[0].value) : v, ctx, { kind: 'push', what: ops[0].source }); + }, + }, + pop: { + cls: { move: true, stack: true, write: true }, + gloss: 'load a value from the stack pointer and move the pointer up by eight', + arity: 1, + check(ops, op) { + const [d] = ops; + if (isMem(d)) d.size = d.size || 8; + if (!isRM(d) || d.size !== 8 || isVec(d)) fail(`${op} moves eight bytes in 64-bit mode; name a 64-bit register or a qword in memory`); + }, + exec(st, ops, ctx) { + const v = pop64(st, ctx, { kind: 'pop', what: ops[0].source }); + writeOperand(st, ops[0], 8, v, ctx); + }, + }, + leave: { + cls: { move: true, stack: true, write: true }, + gloss: 'undo a stack frame: rsp takes rbp’s value, then rbp is popped', + arity: 0, + exec(st, ops, ctx) { + st.regs.rsp = st.regs.rbp; + st.regs.rbp = pop64(st, ctx, { kind: 'pop', what: 'rbp' }); + }, + }, + jmp: { + cls: { control: true }, + gloss: 'carry on from a different place, always', + arity: 1, + exec(st, ops, ctx) { + ctx.jumpTo = ctx.labels[ops[0].name]; + ctx.branch = { kind: 'unconditional', taken: true, target: ops[0].name }; + }, + }, + nop: { cls: {}, gloss: 'do nothing', arity: [0, 3], check() { }, exec() { } }, + ret: { + cls: { control: true, stack: true }, + gloss: 'hand the answer back to whoever called', + arity: 0, + exec(st, ops, ctx) { + if (st.depth > 0) { + const back = pop64(st, ctx, { kind: 'return' }); + st.depth--; + ctx.jumpTo = ctx.resolveReturn(back); + ctx.branch = { kind: 'return', taken: true, target: 'caller', address: back }; + } else { + ctx.halt = true; + } + }, + }, + call: { + cls: { control: true, stack: true }, + gloss: 'remember where to come back to, then carry on somewhere else', + arity: 1, + exec(st, ops, ctx) { + push64(st, ctx.returnAddress, ctx, { kind: 'call', returnTo: ctx.pc + 1 }); + st.depth++; + ctx.jumpTo = ctx.labels[ops[0].name]; + ctx.branch = { kind: 'call', taken: true, target: ops[0].name }; + }, + }, + syscall: { + cls: { control: true, system: true, write: true }, + gloss: 'ask the operating system to do something: rax says what, rdi, rsi, and rdx carry the details', + arity: 0, + exec(st, ops, ctx) { systemCall(st, ctx); }, + }, + neg: { + cls: { alu: true, write: true, flags: true }, + gloss: 'replace a value with zero minus that value', + arity: 1, + check: checkUnary, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const a = readOperand(st, ops[0], w, ctx); + const res = setSubFlags(st, 0n, a, w); + aluNote(ctx, 'sub', 0n, a, w, res, { cin: 0n }); + writeOperand(st, ops[0], w, res, ctx); + }, + }, + not: { + cls: { alu: true, write: true }, + gloss: 'flip every bit, leaving the flags alone', + arity: 1, + check: checkUnary, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const a = readOperand(st, ops[0], w, ctx); + aluNote(ctx, 'not', a, null, w, ~a & MASK[w]); + writeOperand(st, ops[0], w, ~a & MASK[w], ctx); + }, + }, + xchg: { + cls: { move: true, write: true }, + gloss: 'swap two values', + arity: 2, + check(ops, op) { + if (isImm(ops[1]) || isImm(ops[0])) fail(`${op} swaps two places; a constant is not a place`); + checkBinary(ops, op, { allowImm: false }); + }, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const a = readOperand(st, ops[0], w, ctx); + const b = readOperand(st, ops[1], w, ctx); + writeOperand(st, ops[0], w, b, ctx); + writeOperand(st, ops[1], w, a, ctx); + }, + }, + popcnt: bitCounter('popcnt', 'count how many bits are set'), + tzcnt: bitCounter('tzcnt', 'count the zero bits below the lowest set bit, which gives its position'), + lzcnt: bitCounter('lzcnt', 'count the zero bits above the highest set bit'), + bsf: bitCounter('bsf', 'find the position of the lowest set bit; leaves the destination alone when there is none'), + bsr: bitCounter('bsr', 'find the position of the highest set bit; leaves the destination alone when there is none'), +}; + +OPS.sal = { ...OPS.shl, gloss: 'another name for shl' }; + +/** div and idiv share everything except how they read the numbers. */ +function divide(st, ops, ctx, signed) { + const w = operandWidth(ops); + const bits = BigInt(w * 8); + const raw = readOperand(st, ops[0], w, ctx) & MASK[w]; + const wide = wideDividend(st, w); + const divisor = signed ? toSigned(raw, w) : raw; + const dividend = signed ? BigInt.asIntN(Number(bits) * 2, wide) : wide; + if (divisor === 0n) throw new AsmError('divide error (#DE): division by zero', 'divide-error'); + const q = dividend / divisor; // BigInt division truncates toward zero, as idiv does + const r = dividend % divisor; // and the remainder takes the dividend's sign, as idiv does + const fits = signed ? fitsSigned(q, Number(bits)) : q <= MASK[w]; + if (!fits) throw new AsmError(`divide error (#DE): the quotient ${q} does not fit in ${bits} bits`, 'divide-error'); + writeWide(st, w, q & MASK[w], r & MASK[w]); + aluNote(ctx, signed ? 'idiv' : 'div', wide, raw, w, q & MASK[w], { remainder: r & MASK[w], dividend, divisor, wide: true }); +} + +/** + * A Linux-shaped system call. The instruction itself is the boundary between + * the program and the kernel: the processor switches privilege level, saves + * where to come back to in rcx and the flags in r11, and runs kernel code. + * That is why syscall clobbers rcx and r11, which surprises everyone once. + */ +export const SYSCALLS = { + 0: 'read', 1: 'write', 39: 'getpid', 60: 'exit', 231: 'exit_group', +}; + +function systemCall(st, ctx) { + const nr = Number(st.regs.rax & 0xffffffffn); + const name = SYSCALLS[nr] || null; + const args = [st.regs.rdi, st.regs.rsi, st.regs.rdx]; + const info = { nr, name: name || `unknown (${nr})`, args, ret: null, text: '' }; + st.regs.rcx = BigInt(ctx.returnAddress); + st.regs.r11 = rflagsValue(st.flags); + // The kernel checks every pointer a program hands it. A bad one does not + // crash the program: the call fails with -EFAULT, or, if some bytes were + // good, succeeds for those and reports how many. + const EFAULT = -14n; const EBADF = -9n; + if (nr === 1) { + const [fd, buf, count] = args; + let text = ''; + const n = Number(count & 0xffffffffn); + for (let k = 0; k < n; k++) { + let b; + try { b = readByte(st, Number(buf) + k); } catch { info.fault = true; break; } + text += String.fromCharCode(b); + } + if (text.length) ctx.accesses.push({ kind: 'read', addr: Number(buf), size: text.length, value: 0n, ea: null, kernel: true }); + if (fd !== 1n && fd !== 2n) info.ret = EBADF; + else if (info.fault && !text.length) info.ret = EFAULT; + else { st.out += text; info.ret = BigInt(text.length); } + info.text = text; info.fd = Number(fd); + } else if (nr === 0) { + const [fd, buf, count] = args; + if (fd !== 0n) info.ret = EBADF; + else { + const avail = (st.stdin || '').slice(st.inPos, st.inPos + Number(count & 0xffffffffn)); + let done = 0; + for (; done < avail.length; done++) { + try { writeByte(st, Number(buf) + done, avail.charCodeAt(done)); } catch { info.fault = true; break; } + } + if (done) ctx.accesses.push({ kind: 'write', addr: Number(buf), size: done, value: 0n, ea: null, kernel: true }); + st.inPos += done; + info.ret = info.fault && !done ? EFAULT : BigInt(done); info.text = avail.slice(0, done); + } + } else if (nr === 39) { + info.ret = 4242n; + } else if (nr === 60 || nr === 231) { + st.exitCode = Number(st.regs.rdi & 0xffn); + ctx.halt = true; + info.ret = null; + } else { + info.ret = -38n; // ENOSYS + } + if (info.ret !== null) st.regs.rax = BigInt.asUintN(64, info.ret); + ctx.syscall = info; + ctx.branch = { kind: 'system', taken: true, target: 'kernel' }; +} + +/* ------------------------------------------------------------------ */ +/* Vector instructions */ +/* ------------------------------------------------------------------ */ + +function checkVecBinary(ops, op) { + if (!isVec(ops[0])) fail(`the first operand of ${op} must be an xmm register`); + if (!(isVec(ops[1]) || isMem(ops[1]))) fail(`the second operand of ${op} must be an xmm register or sixteen bytes of memory`); + if (isMem(ops[1])) { if (ops[1].size && ops[1].size !== 16) fail(`${op} reads sixteen bytes; use xmmword ptr`); ops[1].size = 16; } +} + +function checkVecMove(ops, op) { + const [d, s] = ops; + if (isVec(d) && (isVec(s) || isMem(s))) { if (isMem(s)) { if (s.size && s.size !== 16) fail(`${op} moves sixteen bytes; use xmmword ptr`); s.size = 16; } return; } + if (isMem(d) && isVec(s)) { if (d.size && d.size !== 16) fail(`${op} moves sixteen bytes; use xmmword ptr`); d.size = 16; return; } + fail(`${op} moves sixteen bytes between xmm registers and memory`); +} + +/** + * Sixteen bytes at a time. Each of these treats an xmm register as sixteen + * independent byte lanes and does the same thing to every lane at once, + * which is the whole idea students need to see. + */ +const VECTOR_OPS = { + movdqu: { + cls: { move: true, write: true, vector: true }, + gloss: 'copy sixteen bytes, from any address', + arity: 2, + check: checkVecMove, + exec(st, ops, ctx) { + const v = readOperand(st, ops[1], 16, ctx); + writeOperand(st, ops[0], 16, v, ctx); + }, + }, + movdqa: { + cls: { move: true, write: true, vector: true }, + gloss: 'copy sixteen bytes, from an address that must be a multiple of sixteen', + arity: 2, + check: checkVecMove, + exec(st, ops, ctx) { + const m = ops.find((o) => o.kind === 'mem'); + if (m) { + const ea = effectiveAddress(st, m); + if (ea.addr % 16 !== 0) throw new AsmError(`general protection fault: movdqa needs an address that is a multiple of 16; this one is ${formatHex(BigInt(ea.addr), 4)}`, 'alignment'); + } + const v = readOperand(st, ops[1], 16, ctx); + writeOperand(st, ops[0], 16, v, ctx); + }, + }, + movd: { + cls: { move: true, write: true, vector: true }, + gloss: 'copy four bytes between a vector register and an ordinary one', + arity: 2, + check(ops, op) { + const [d, s] = ops; + const other = isVec(d) ? s : d; + if (isVec(d) === isVec(s)) fail(`${op} needs exactly one xmm register`); + if (isMem(other)) other.size = other.size || 4; + if (!isRM(other) || other.size !== 4) fail(`${op} moves four bytes; name a 32-bit register or a dword in memory`); + }, + exec(st, ops, ctx) { + const v = readOperand(st, ops[1], 4, ctx) & MASK[4]; + if (REGS[ops[0].name?.toLowerCase()]?.vector) writeOperand(st, ops[0], 16, v, ctx); + else writeOperand(st, ops[0], 4, v, ctx); + }, + }, + movq: { + cls: { move: true, write: true, vector: true }, + gloss: 'copy eight bytes between a vector register and an ordinary one', + arity: 2, + check(ops, op) { + const [d, s] = ops; + if (isVec(d) && isVec(s)) return; + const other = isVec(d) ? s : d; + if (!isVec(d) && !isVec(s)) fail(`${op} needs an xmm register on at least one side`); + if (isMem(other)) other.size = other.size || 8; + if (!isRM(other) || other.size !== 8) fail(`${op} moves eight bytes; name a 64-bit register or a qword in memory`); + }, + exec(st, ops, ctx) { + const v = readOperand(st, ops[1], 8, ctx) & MASK[8]; + if (REGS[ops[0].name?.toLowerCase()]?.vector) writeOperand(st, ops[0], 16, v, ctx); + else writeOperand(st, ops[0], 8, v, ctx); + }, + }, + pcmpeqb: { + cls: { alu: true, write: true, vector: true, compare: true }, + gloss: 'compare sixteen pairs of bytes at once, filling each lane with ones where they matched', + arity: 2, + check: checkVecBinary, + exec(st, ops, ctx) { + const a = readOperand(st, ops[0], 16, ctx); + const b = readOperand(st, ops[1], 16, ctx); + ctx.vectorCompare = { a, b }; + writeOperand(st, ops[0], 16, perLane(a, b, (x, y) => (x === y ? 0xff : 0x00)), ctx); + }, + }, + pmovmskb: { + cls: { write: true, vector: true, extract: true }, + gloss: 'collect the top bit of each of sixteen lanes into a sixteen-bit number', + arity: 2, + check(ops, op) { + checkDestReg(ops, op, [4, 8]); + if (!isVec(ops[1])) fail(`${op} reads an xmm register`); + }, + exec(st, ops, ctx) { + const v = readOperand(st, ops[1], 16, ctx); + const bits = lanes(v).reduce((acc, byte, k) => acc | (byte & 0x80 ? (1 << k) : 0), 0); + writeOperand(st, ops[0], ops[0].size, BigInt(bits), ctx); + }, + }, + pand: { + cls: { alu: true, write: true, vector: true }, + gloss: 'bitwise AND across sixteen bytes', + arity: 2, + check: checkVecBinary, + exec(st, ops, ctx) { + const a = readOperand(st, ops[0], 16, ctx); + const b = readOperand(st, ops[1], 16, ctx); + writeOperand(st, ops[0], 16, a & b, ctx); + }, + }, + por: { + cls: { alu: true, write: true, vector: true }, + gloss: 'bitwise OR across sixteen bytes', + arity: 2, + check: checkVecBinary, + exec(st, ops, ctx) { + const a = readOperand(st, ops[0], 16, ctx); + const b = readOperand(st, ops[1], 16, ctx); + writeOperand(st, ops[0], 16, a | b, ctx); + }, + }, + pxor: { + cls: { alu: true, write: true, vector: true }, + gloss: 'bitwise exclusive OR across sixteen bytes; with the same register twice, this clears it', + arity: 2, + check: checkVecBinary, + exec(st, ops, ctx) { + const a = readOperand(st, ops[0], 16, ctx); + const b = readOperand(st, ops[1], 16, ctx); + writeOperand(st, ops[0], 16, a ^ b, ctx); + }, + }, + psubb: { + cls: { alu: true, write: true, vector: true }, + gloss: 'subtract sixteen pairs of bytes at once, each lane on its own', + arity: 2, + check: checkVecBinary, + exec(st, ops, ctx) { + const a = readOperand(st, ops[0], 16, ctx); + const b = readOperand(st, ops[1], 16, ctx); + writeOperand(st, ops[0], 16, perLane(a, b, (x, y) => x - y), ctx); + }, + }, + paddb: { + cls: { alu: true, write: true, vector: true }, + gloss: 'add sixteen pairs of bytes at once, each lane on its own', + arity: 2, + check: checkVecBinary, + exec(st, ops, ctx) { + const a = readOperand(st, ops[0], 16, ctx); + const b = readOperand(st, ops[1], 16, ctx); + writeOperand(st, ops[0], 16, perLane(a, b, (x, y) => x + y), ctx); + }, + }, +}; + +Object.assign(OPS, VECTOR_OPS); + +/* ------------------------------------------------------------------ */ +/* What compiled Mojo also uses */ +/* ------------------------------------------------------------------ */ +/* + * Programming+ sends functions from the real compiler's listings to this + * machine (src/mojo/att.ts). A census of every listing in the Mojo corpus + * (docs/asm-coverage.md) found these beyond the course's subset: the BMI + * bit instructions the optimizer likes (andn, mulx, the x-shifts, bzhi, + * blsr, blsi, blsmsk, rorx, bextr), the atomic read-modify-writes behind + * reference counts (xadd, cmpxchg, with lock), bit tests that write, bswap, + * the trap the compiler plants on paths that must never run (ud2), and the + * 128-bit AVX spellings of the vector moves and logic the subset already + * had. 256-bit AVX2 (ymm) stays outside the model, and says so. + */ + +/** Why an instruction outside the subset is outside it, when the reason is worth a sentence. */ +const UNMODELED = { + vzeroall: 'AVX2 state the model does not keep', + vpinsrb: 'inserting into one lane of a vector is not modeled yet', vpextrb: 'extracting one lane of a vector is not modeled yet', vpextrw: 'extracting one lane of a vector is not modeled yet', vpextrd: 'extracting one lane of a vector is not modeled yet', vpextrq: 'extracting one lane of a vector is not modeled yet', + vpshufb: 'byte shuffles are not modeled yet', vpshufd: 'lane shuffles are not modeled yet', vpalignr: 'lane shuffles are not modeled yet', vperm2i128: 'AVX2 lane permutes are not modeled', vextracti128: 'AVX2 256-bit registers are not modeled', vinserti128: 'AVX2 256-bit registers are not modeled', vpbroadcastd: 'AVX2 broadcasts are not modeled', + vpminub: 'per-lane minimum is not modeled yet', vpminud: 'per-lane minimum is not modeled yet', vpcmpeqd: 'four-byte lanes are not modeled yet; the model compares byte lanes (pcmpeqb)', vpsrld: 'per-lane shifts are not modeled yet', vpsllw: 'per-lane shifts are not modeled yet', vpsrlw: 'per-lane shifts are not modeled yet', vpsubusb: 'saturating lane arithmetic is not modeled yet', vpblendvb: 'lane blends are not modeled yet', + pdep: 'BMI2 bit deposit is not modeled yet', pext: 'BMI2 bit extract is not modeled yet', +}; + +/** d = f(a, b) over ordinary registers, three operands, no carry in: the BMI shape. */ +function threeOp(kind, gloss, fn, { flags = 'logic', third = 'rm' } = {}) { + return { + cls: { alu: true, write: true, flags: flags !== 'none' }, + gloss, + arity: 3, + check(ops, op) { + const [d, a, b] = ops; + if (!isReg(d) || isVec(d) || ![4, 8].includes(d.size)) fail(`the first operand of ${op} must be a 32- or 64-bit register`); + if (third === 'rm' ? !(isReg(a) && !isVec(a) && a.size === d.size) : !(isRM(a) && !isVec(a))) fail(`the second operand of ${op} must be a register as wide as the first`); + if (third === 'rm') { if (!(isRM(b) && !isVec(b))) fail(`the third operand of ${op} must be a register or a memory address`); if (isMem(b)) b.size = b.size || d.size; else if (b.size !== d.size) fail(`every register in ${op} must be the same width`); } + else if (third === 'reg') { if (!(isReg(b) && !isVec(b) && b.size === d.size)) fail(`the third operand of ${op} must be a register as wide as the first`); if (isMem(a)) a.size = a.size || d.size; else if (a.size !== d.size) fail(`every register in ${op} must be the same width`); } + else { if (!isImm(b)) fail(`the third operand of ${op} must be a constant`); if (isMem(a)) a.size = a.size || d.size; } + }, + exec(st, ops, ctx) { + const w = ops[0].size; + const a = readOperand(st, ops[1], w, ctx) & MASK[w]; + const b = isImm(ops[2]) ? ops[2].value & MASK[w] : readOperand(st, ops[2], w, ctx) & MASK[w]; + const res = fn(st, a, b, w) & MASK[w]; + aluNote(ctx, kind, a, b, w, res); + if (flags === 'logic') setLogicFlags(st, res, w); + writeOperand(st, ops[0], w, res, ctx); + }, + }; +} + +/** d = f(s) over ordinary registers: the BMI1 "lowest set bit" family. */ +function twoOpBmi(kind, gloss, fn, carry) { + return { + cls: { alu: true, write: true, flags: true }, + gloss, + arity: 2, + check(ops, op) { + checkDestReg(ops, op, [4, 8]); + if (!(isRM(ops[1]) && !isVec(ops[1]))) fail(`the second operand of ${op} must be a register or a memory address`); + if (isMem(ops[1])) ops[1].size = ops[1].size || ops[0].size; else if (ops[1].size !== ops[0].size) fail(`both operands of ${op} must be the same width`); + }, + exec(st, ops, ctx) { + const w = ops[0].size; + const v = readOperand(st, ops[1], w, ctx) & MASK[w]; + const res = fn(v, w) & MASK[w]; + aluNote(ctx, kind, v, null, w, res); + st.flags.ZF = res === 0n; st.flags.SF = (res & SIGNBIT[w]) !== 0n; st.flags.OF = false; st.flags.CF = carry(v); + writeOperand(st, ops[0], w, res, ctx); + }, + }; +} + +const bitWrite = (kind, gloss, fn) => ({ + cls: { alu: true, write: true, flags: true, lockable: true }, + gloss, + arity: 2, + check(ops, op) { OPS.bt.check(ops, op); }, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const a = readOperand(st, ops[0], w, ctx) & MASK[w]; + const n = Number((isImm(ops[1]) ? ops[1].value : readOperand(st, ops[1], w, ctx)) & BigInt(w * 8 - 1)); + const bit = (a >> BigInt(n)) & 1n; + st.flags.CF = bit === 1n; + const res = fn(a, 1n << BigInt(n)) & MASK[w]; + aluNote(ctx, kind, a, BigInt(n), w, res, { bit: Number(bit) }); + writeOperand(st, ops[0], w, res, ctx); + }, +}); + +const MOJO_OPS = { + andn: threeOp('andn', 'AND the third operand with the complement of the second: keep the bits set in one and clear in the other', (st, a, b) => ~a & b), + mulx: { + cls: { alu: true, write: true, multiply: true }, + gloss: 'multiply rdx (or edx) by the third operand, unsigned, without touching the flags: the high half lands in the first operand, the low half in the second', + arity: 3, + check(ops, op) { + const [hi, lo, src] = ops; + if (!isReg(hi) || isVec(hi) || ![4, 8].includes(hi.size)) fail(`the first operand of ${op} must be a 32- or 64-bit register`); + if (!isReg(lo) || isVec(lo) || lo.size !== hi.size) fail(`the second operand of ${op} must be a register as wide as the first`); + if (!(isRM(src) && !isVec(src))) fail(`the third operand of ${op} must be a register or a memory address`); + if (isMem(src)) src.size = src.size || hi.size; else if (src.size !== hi.size) fail(`every register in ${op} must be the same width`); + }, + exec(st, ops, ctx) { + const w = ops[0].size; + const a = (w === 8 ? st.regs.rdx : st.regs.rdx & MASK[4]) & MASK[w]; + const b = readOperand(st, ops[2], w, ctx) & MASK[w]; + const full = a * b; + const lo = full & MASK[w], hi = (full >> BigInt(w * 8)) & MASK[w]; + aluNote(ctx, 'mulx', a, b, w, lo, { high: hi }); + writeOperand(st, ops[1], w, lo, ctx); + writeOperand(st, ops[0], w, hi, ctx); + }, + }, + shlx: threeOp('shlx', 'shift the second operand left by the count in the third, into the first, leaving the flags alone', (st, a, b, w) => a << (b & BigInt(w * 8 - 1)), { flags: 'none', third: 'reg' }), + shrx: threeOp('shrx', 'shift the second operand right by the count in the third, into the first, leaving the flags alone', (st, a, b, w) => a >> (b & BigInt(w * 8 - 1)), { flags: 'none', third: 'reg' }), + sarx: threeOp('sarx', 'shift the second operand right, keeping its sign, by the count in the third, into the first, leaving the flags alone', (st, a, b, w) => toSigned(a, w) >> (b & BigInt(w * 8 - 1)), { flags: 'none', third: 'reg' }), + bzhi: threeOp('bzhi', 'copy the second operand with every bit from the position in the third upward cleared', (st, a, b, w) => { const n = Number(b & 0xffn); st.flags.CF = n > w * 8 - 1; return n >= w * 8 ? a : a & ((1n << BigInt(n)) - 1n); }, { third: 'reg' }), + rorx: threeOp('rorx', 'rotate the second operand right by a constant, into the first, leaving the flags alone', (st, a, b, w) => { const bits = w * 8; const r = Number(b) % bits; return r ? ((a >> BigInt(r)) | (a << BigInt(bits - r))) & MASK[w] : a; }, { flags: 'none', third: 'imm' }), + bextr: threeOp('bextr', 'take a field of bits out of the second operand: the third holds the start position in its low byte and the length in the next', (st, a, b, w) => { const start = Number(b & 0xffn), len = Number((b >> 8n) & 0xffn); return len === 0 || start >= w * 8 ? 0n : (a >> BigInt(start)) & ((1n << BigInt(Math.min(len, w * 8))) - 1n); }, { third: 'reg' }), + blsr: twoOpBmi('blsr', 'clear the lowest set bit', (v) => v & (v - 1n), (v) => v === 0n), + blsi: twoOpBmi('blsi', 'keep only the lowest set bit', (v, w) => v & (-v & MASK[w]), (v) => v !== 0n), + blsmsk: twoOpBmi('blsmsk', 'a mask of ones from bit 0 up to and including the lowest set bit', (v) => v ^ (v - 1n), (v) => v === 0n), + xadd: { + cls: { alu: true, write: true, flags: true, lockable: true }, + gloss: 'exchange and add: the first operand becomes the sum, the second receives what the first held (with lock, the atomic increment behind a reference count)', + arity: 2, + check(ops, op) { if (!isReg(ops[1]) || isVec(ops[1])) fail(`the second operand of ${op} must be an ordinary register`); checkBinary(ops, op, { allowImm: false }); }, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const a = readOperand(st, ops[0], w, ctx), b = readOperand(st, ops[1], w, ctx); + const res = setAddFlags(st, a, b, w); + aluNote(ctx, 'add', a, b, w, res); + writeOperand(st, ops[1], w, a, ctx); + writeOperand(st, ops[0], w, res, ctx); + }, + }, + cmpxchg: { + cls: { alu: true, write: true, flags: true, compare: true, lockable: true }, + gloss: 'compare the accumulator (rax, eax, ax, or al) with the first operand: when they are equal, store the second operand there and set ZF; otherwise load the first into the accumulator and clear ZF', + arity: 2, + check(ops, op) { if (!isReg(ops[1]) || isVec(ops[1])) fail(`the second operand of ${op} must be an ordinary register`); checkBinary(ops, op, { allowImm: false }); }, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const acc = { kind: 'reg', name: { 1: 'al', 2: 'ax', 4: 'eax', 8: 'rax' }[w], size: w }; + const expected = readOperand(st, acc, w, ctx), current = readOperand(st, ops[0], w, ctx); + setSubFlags(st, expected, current, w); + aluNote(ctx, 'cmp', expected, current, w, (expected - current) & MASK[w]); + if (st.flags.ZF) writeOperand(st, ops[0], w, readOperand(st, ops[1], w, ctx), ctx); + else writeOperand(st, acc, w, current, ctx); + }, + }, + bts: bitWrite('bts', 'copy a bit into CF, then set it', (a, m) => a | m), + btr: bitWrite('btr', 'copy a bit into CF, then clear it', (a, m) => a & ~m), + btc: bitWrite('btc', 'copy a bit into CF, then flip it', (a, m) => a ^ m), + bswap: { + cls: { alu: true, write: true }, + gloss: 'reverse the order of the bytes: little-endian to big-endian and back', + arity: 1, + check(ops, op) { checkDestReg(ops, op, [4, 8]); }, + exec(st, ops, ctx) { + const w = ops[0].size; + const v = readOperand(st, ops[0], w, ctx) & MASK[w]; + let res = 0n; + for (let k = 0; k < w; k++) res = (res << 8n) | ((v >> BigInt(8 * k)) & 0xffn); + aluNote(ctx, 'bswap', v, null, w, res); + writeOperand(st, ops[0], w, res, ctx); + }, + }, + cwd: { cls: { alu: true, write: true, extend: true }, gloss: 'widen ax into dx:ax, repeating its sign bit', arity: 0, exec(st) { regWrite(st, 'dx', (toSigned(st.regs.rax, 2) >> 16n) & MASK[2]); } }, + cwde: { cls: { alu: true, write: true, extend: true }, gloss: 'widen ax to eax, repeating its sign bit', arity: 0, exec(st) { regWrite(st, 'eax', toSigned(st.regs.rax, 2) & MASK[4]); } }, + cbw: { cls: { alu: true, write: true, extend: true }, gloss: 'widen al to ax, repeating its sign bit', arity: 0, exec(st) { regWrite(st, 'ax', toSigned(st.regs.rax, 1) & MASK[2]); } }, + ud2: { + cls: { control: true, trap: true }, + gloss: 'an instruction that is defined to be undefined: the compiler plants it where a path must never be taken, so reaching it is a crash', + arity: 0, + check() { }, + exec(st, ops, ctx) { + // a stub for code that is not on the model machine says so in its comment; the message carries it + const note = ctx.instr?.comment?.trim(); + throw new AsmError(note && /not on the model machine/.test(note) ? `stopped: ${note}` : 'ud2: the processor raised an invalid-opcode fault. The compiler plants ud2 where a path must never be taken (a failed check, an unreachable branch); reaching it means the program crashed here.', 'trap'); + }, + }, + vzeroupper: { cls: { vector: true }, gloss: 'clear the upper halves of the 256-bit registers; with only the 128-bit halves modeled, nothing changes', arity: 0, check() { }, exec() { } }, +}; + +// the lockable read-modify-writes the subset already has +for (const op of ['add', 'sub', 'and', 'or', 'xor', 'inc', 'dec', 'neg', 'not', 'xchg']) if (OPS[op]) OPS[op].cls.lockable = true; + +/** The AVX spelling of a vector instruction the subset has: `vop d, a, b` is `op` with the result in d, the sources untouched. */ +function avx3(base, gloss) { + const inner = VECTOR_OPS[base]; + return { + cls: { ...inner.cls, avx: true }, + gloss, + arity: 3, + check(ops, op) { + const [d, a, b] = ops; + if (!isVec(d) || !isVec(a)) fail(`the first two operands of ${op} must be xmm registers`); + checkVecBinary([a, b], op); + }, + exec(st, ops, ctx) { + const [d, a, b] = ops; + const save = regValue(st, a.name); + inner.exec(st, [a, b], ctx); + const res = regValue(st, a.name); + if (d.name !== a.name) { regWrite(st, a.name, save); writeOperand(st, d, 16, res, ctx); } + }, + }; +} +Object.assign(MOJO_OPS, { + vmovdqu: { ...VECTOR_OPS.movdqu, gloss: 'the AVX spelling of movdqu' }, + vmovdqa: { ...VECTOR_OPS.movdqa, gloss: 'the AVX spelling of movdqa' }, + vmovups: { ...VECTOR_OPS.movdqu, gloss: 'copy sixteen bytes (four floats, to the compiler) from any address; the same bytes move either way' }, + vmovaps: { ...VECTOR_OPS.movdqa, gloss: 'copy sixteen bytes (four floats, to the compiler) from an address that is a multiple of sixteen' }, + vmovd: { ...VECTOR_OPS.movd, gloss: 'the AVX spelling of movd' }, + vmovq: { ...VECTOR_OPS.movq, gloss: 'the AVX spelling of movq' }, + vpmovmskb: { ...VECTOR_OPS.pmovmskb, gloss: 'the AVX spelling of pmovmskb' }, + vpxor: avx3('pxor', 'exclusive OR of the second and third operands, sixteen bytes at once, into the first'), + vxorps: avx3('pxor', 'exclusive OR of sixteen bytes into the first operand; with one register three times, this clears it'), + vpor: avx3('por', 'OR of the second and third operands, sixteen bytes at once, into the first'), + vpand: avx3('pand', 'AND of the second and third operands, sixteen bytes at once, into the first'), + vpaddb: avx3('paddb', 'add the second and third operands lane by lane, sixteen bytes at once, into the first'), + vpsubb: avx3('psubb', 'subtract the third operand from the second lane by lane, sixteen bytes at once, into the first'), + vpcmpeqb: avx3('pcmpeqb', 'compare the second and third operands lane by lane, filling each lane of the first with ones where they matched'), +}); + +/* lanes of other widths: a vector as 8 words, 4 dwords, or 2 qwords */ +function lanesOf(v, size) { const n = 16 / size, m = MASK[size]; const out = []; for (let k = 0; k < n; k++) out.push((v >> BigInt(8 * size * k)) & m); return out; } +function fromLanesOf(parts, size) { let v = 0n; for (let k = parts.length - 1; k >= 0; k--) v = (v << BigInt(8 * size)) | (parts[k] & MASK[size]); return v; } +const LANE_WORD = { 1: 'byte', 2: 'word', 4: 'dword', 8: 'qword' }; + +/** vpinsr{b,w,d,q} d, a, src, n: a with lane n replaced by src. */ +function laneInsert(size) { + const op = `vpinsr${LANE_WORD[size][0]}`; + return { + cls: { write: true, vector: true, avx: true }, + gloss: `copy the second operand into the first with one ${LANE_WORD[size]} lane (the last operand says which) replaced by the third`, + arity: 4, + check(ops) { + const [d, a, src, n] = ops; + if (!isVec(d) || !isVec(a)) fail(`the first two operands of ${op} must be xmm registers`); + if (!isImm(n)) fail(`the last operand of ${op} is the lane number, a constant`); + if (isMem(src)) src.size = src.size || size; else if (!isReg(src) || src.size !== (size <= 4 ? 4 : 8)) fail(`the third operand of ${op} must be a ${size <= 4 ? '32' : '64'}-bit register or ${LANE_WORD[size]} in memory`); + }, + exec(st, ops, ctx) { + const [d, a, src, n] = ops; + const parts = lanesOf(regValue(st, a.name), size); + parts[Number(n.value) % parts.length] = readOperand(st, src, isMem(src) ? size : src.size, ctx) & MASK[size]; + writeOperand(st, d, 16, fromLanesOf(parts, size), ctx); + }, + }; +} + +/** vpextr{b,w,d,q} dst, a, n: lane n of a into a register or memory. */ +function laneExtract(size) { + const op = `vpextr${LANE_WORD[size][0]}`; + return { + cls: { write: true, vector: true, extract: true, avx: true }, + gloss: `copy one ${LANE_WORD[size]} lane (the last operand says which) of the second operand into the first`, + arity: 3, + check(ops) { + const [dst, a, n] = ops; + if (!isVec(a)) fail(`the second operand of ${op} must be an xmm register`); + if (!isImm(n)) fail(`the last operand of ${op} is the lane number, a constant`); + if (isMem(dst)) dst.size = dst.size || size; else if (!isReg(dst) || dst.size !== (size <= 4 ? 4 : 8)) fail(`the first operand of ${op} must be a ${size <= 4 ? '32' : '64'}-bit register or ${LANE_WORD[size]} in memory`); + }, + exec(st, ops, ctx) { + const [dst, a, n] = ops; + const parts = lanesOf(regValue(st, a.name), size); + const v = parts[Number(n.value) % parts.length]; + writeOperand(st, dst, isMem(dst) ? size : dst.size, v, ctx); + }, + }; +} + +/** d = f(a, b) lane by lane at a given lane width, three operands, AVX shape. */ +function laneBinary(op, size, gloss, fn) { + return { + cls: { alu: true, write: true, vector: true, avx: true }, + gloss, + arity: 3, + check(ops) { const [d, a, b] = ops; if (!isVec(d) || !isVec(a)) fail(`the first two operands of ${op} must be xmm registers`); checkVecBinary([a, b], op); }, + exec(st, ops, ctx) { + const [d, a, b] = ops; + const la = lanesOf(regValue(st, a.name), size), lb = lanesOf(readOperand(st, b, 16, ctx), size); + writeOperand(st, d, 16, fromLanesOf(la.map((x, k) => fn(x, lb[k]) & MASK[size]), size), ctx); + }, + }; +} + +/** d = each lane of a shifted by a constant. */ +function laneShift(op, size, gloss, fn) { + return { + cls: { alu: true, write: true, vector: true, shift: true, avx: true }, + gloss, + arity: 3, + check(ops) { const [d, a, n] = ops; if (!isVec(d) || !isVec(a)) fail(`the first two operands of ${op} must be xmm registers`); if (!isImm(n)) fail(`${op} shifts by a constant here`); }, + exec(st, ops, ctx) { + const [d, a, n] = ops; + const k = Number(n.value); + const parts = lanesOf(regValue(st, a.name), size).map((x) => (k >= size * 8 ? 0n : fn(x, BigInt(k)) & MASK[size])); + writeOperand(st, d, 16, fromLanesOf(parts, size), ctx); + }, + }; +} + +Object.assign(MOJO_OPS, { + vpinsrb: laneInsert(1), vpinsrw: laneInsert(2), vpinsrd: laneInsert(4), vpinsrq: laneInsert(8), + vpextrb: laneExtract(1), vpextrw: laneExtract(2), vpextrd: laneExtract(4), vpextrq: laneExtract(8), + vpminub: laneBinary('vpminub', 1, 'the smaller of each pair of bytes, unsigned', (x, y) => (x < y ? x : y)), + vpmaxub: laneBinary('vpmaxub', 1, 'the larger of each pair of bytes, unsigned', (x, y) => (x > y ? x : y)), + vpminud: laneBinary('vpminud', 4, 'the smaller of each pair of dwords, unsigned', (x, y) => (x < y ? x : y)), + vpmaxud: laneBinary('vpmaxud', 4, 'the larger of each pair of dwords, unsigned', (x, y) => (x > y ? x : y)), + vpcmpeqd: laneBinary('vpcmpeqd', 4, 'compare four pairs of dwords, filling each lane with ones where they matched', (x, y) => (x === y ? MASK[4] : 0n)), + vpcmpeqw: laneBinary('vpcmpeqw', 2, 'compare eight pairs of words, filling each lane with ones where they matched', (x, y) => (x === y ? MASK[2] : 0n)), + vpcmpeqq: laneBinary('vpcmpeqq', 8, 'compare two pairs of qwords, filling each lane with ones where they matched', (x, y) => (x === y ? MASK[8] : 0n)), + // signed greater-than: each lane read as a two's-complement integer of its width + vpcmpgtb: laneBinary('vpcmpgtb', 1, 'compare sixteen pairs of signed bytes, filling each lane with ones where the first was greater', (x, y) => (BigInt.asIntN(8, x) > BigInt.asIntN(8, y) ? MASK[1] : 0n)), + vpcmpgtw: laneBinary('vpcmpgtw', 2, 'compare eight pairs of signed words, filling each lane with ones where the first was greater', (x, y) => (BigInt.asIntN(16, x) > BigInt.asIntN(16, y) ? MASK[2] : 0n)), + vpcmpgtd: laneBinary('vpcmpgtd', 4, 'compare four pairs of signed dwords, filling each lane with ones where the first was greater', (x, y) => (BigInt.asIntN(32, x) > BigInt.asIntN(32, y) ? MASK[4] : 0n)), + vpcmpgtq: laneBinary('vpcmpgtq', 8, 'compare two pairs of signed qwords, filling each lane with ones where the first was greater', (x, y) => (BigInt.asIntN(64, x) > BigInt.asIntN(64, y) ? MASK[8] : 0n)), + vpaddd: laneBinary('vpaddd', 4, 'add four pairs of dwords', (x, y) => x + y), + vpsubd: laneBinary('vpsubd', 4, 'subtract four pairs of dwords', (x, y) => x - y), + vpaddw: laneBinary('vpaddw', 2, 'add eight pairs of words', (x, y) => x + y), + vpsubusb: laneBinary('vpsubusb', 1, 'subtract sixteen pairs of bytes, stopping at zero instead of wrapping', (x, y) => (x > y ? x - y : 0n)), + vpandn: laneBinary('vpandn', 8, 'AND the third operand with the complement of the second, sixteen bytes at once', (x, y) => ~x & y), + vpsrld: laneShift('vpsrld', 4, 'shift each dword right by a constant', (x, k) => x >> k), + vpslld: laneShift('vpslld', 4, 'shift each dword left by a constant', (x, k) => x << k), + vpsrlw: laneShift('vpsrlw', 2, 'shift each word right by a constant', (x, k) => x >> k), + vpsllw: laneShift('vpsllw', 2, 'shift each word left by a constant', (x, k) => x << k), + vpsrlq: laneShift('vpsrlq', 8, 'shift each qword right by a constant', (x, k) => x >> k), + vpsllq: laneShift('vpsllq', 8, 'shift each qword left by a constant', (x, k) => x << k), + vpshufd: { + cls: { write: true, vector: true, avx: true }, + gloss: 'rearrange the four dwords of the second operand: each pair of bits of the constant picks the source dword for one lane', + arity: 3, + check(ops) { const [d, a, n] = ops; if (!isVec(d)) fail('the first operand of vpshufd must be an xmm register'); checkVecBinary([d, a], 'vpshufd'); if (!isImm(n)) fail('the last operand of vpshufd is a constant'); }, + exec(st, ops, ctx) { + const [d, a, n] = ops; + const src = lanesOf(readOperand(st, a, 16, ctx), 4); + const sel = Number(n.value) & 0xff; + writeOperand(st, d, 16, fromLanesOf([0, 1, 2, 3].map((k) => src[(sel >> (2 * k)) & 3]), 4), ctx); + }, + }, + vpshufb: { + cls: { write: true, vector: true, avx: true }, + gloss: 'rearrange the bytes of the second operand: each byte of the third names the source lane for that position, or clears it when its top bit is set', + arity: 3, + check(ops) { const [d, a, b] = ops; if (!isVec(d) || !isVec(a)) fail('the first two operands of vpshufb must be xmm registers'); checkVecBinary([a, b], 'vpshufb'); }, + exec(st, ops, ctx) { + const [d, a, b] = ops; + const src = lanes(regValue(st, a.name)), idx = lanes(readOperand(st, b, 16, ctx)); + writeOperand(st, d, 16, fromLanes(idx.map((i) => (i & 0x80 ? 0 : src[i & 15]))), ctx); + }, + }, + vpbroadcastd: { + cls: { write: true, vector: true, avx: true }, + gloss: 'copy one dword into all four lanes', + arity: 2, + check(ops) { const [d, s] = ops; if (!isVec(d)) fail('the first operand of vpbroadcastd must be an xmm register'); if (isMem(s)) s.size = s.size || 4; else if (!isVec(s)) fail('vpbroadcastd reads an xmm register or a dword in memory'); }, + exec(st, ops, ctx) { + const [d, s] = ops; + const v = isMem(s) ? readOperand(st, s, 4, ctx) & MASK[4] : regValue(st, s.name) & MASK[4]; + writeOperand(st, d, 16, fromLanesOf([v, v, v, v], 4), ctx); + }, + }, + vpbroadcastb: { + cls: { write: true, vector: true, avx: true }, + gloss: 'copy one byte into all sixteen lanes', + arity: 2, + check(ops) { const [d, s] = ops; if (!isVec(d)) fail('the first operand of vpbroadcastb must be an xmm register'); if (isMem(s)) s.size = s.size || 1; else if (!isVec(s)) fail('vpbroadcastb reads an xmm register or a byte in memory'); }, + exec(st, ops, ctx) { + const [d, s] = ops; + const v = Number((isMem(s) ? readOperand(st, s, 1, ctx) : regValue(st, s.name)) & 0xffn); + writeOperand(st, d, 16, fromLanes(Array(16).fill(v)), ctx); + }, + }, +}); +for (const k of ['vpinsrb', 'vpinsrw', 'vpinsrd', 'vpinsrq', 'vpextrb', 'vpextrw', 'vpextrd', 'vpextrq', 'vpshufb', 'vpshufd', 'vpbroadcastd', 'vpminub', 'vpminud', 'vpcmpeqd', 'vpsrld', 'vpsllw', 'vpsrlw', 'vpsubusb']) delete UNMODELED[k]; +/* ------------------------------------------------------------------ */ +/* Scalar double precision, and the double-width shifts */ +/* ------------------------------------------------------------------ */ + +/* + * A Float64 lives in the low eight bytes of an xmm register as an IEEE 754 + * binary64 bit pattern. The upper eight bytes are not part of the number: + * each instruction below says whether it keeps them, copies them from the + * first source, or clears them. JavaScript numbers are binary64 and round to + * nearest even, the processor's default mode, so + - * / and sqrt agree + * with the hardware bit for bit. What differs is NaN, which the helpers below + * handle the way SSE does. The exception flags in MXCSR are not modeled; every + * exception is masked, which is the default, so none of them stops a run. + */ +const F64_VIEW = new DataView(new ArrayBuffer(8)); +const f64FromBits = (bits) => { F64_VIEW.setBigUint64(0, BigInt(bits) & MASK[8]); return F64_VIEW.getFloat64(0); }; +const f64ToBits = (x) => { F64_VIEW.setFloat64(0, x); return F64_VIEW.getBigUint64(0); }; +const F64_QUIET = 1n << 51n; // the top fraction bit: set for a quiet NaN +const F64_INDEFINITE = 0xfff8000000000000n; // what an invalid operation produces: "real indefinite" +const f64IsNaN = (bits) => ((bits >> 52n) & 0x7ffn) === 0x7ffn && (bits & 0xfffffffffffffn) !== 0n; +const low64 = (v) => BigInt(v) & MASK[8]; +const upper64 = (v) => BigInt(v) & (MASK[16] ^ MASK[8]); + +/** a op b on two bit patterns. A NaN operand comes through quieted (the first operand's, when both are NaN); an invalid operation (0/0, inf - inf) gives the default NaN. */ +function f64Binary(a, b, fn) { + if (f64IsNaN(a)) return a | F64_QUIET; + if (f64IsNaN(b)) return b | F64_QUIET; + const r = fn(f64FromBits(a), f64FromBits(b)); + return Number.isNaN(r) ? F64_INDEFINITE : f64ToBits(r); +} + +function f64Sqrt(b) { + if (f64IsNaN(b)) return b | F64_QUIET; + const r = Math.sqrt(f64FromBits(b)); + return Number.isNaN(r) ? F64_INDEFINITE : f64ToBits(r); +} + +/** The flags ucomisd leaves: unordered (a NaN) sets ZF, PF and CF; otherwise greater clears them, less sets CF, equal sets ZF. OF and SF are cleared. */ +function f64CompareFlags(st, a, b) { + const f = st.flags; + if (f64IsNaN(a) || f64IsNaN(b)) { f.ZF = true; f.PF = true; f.CF = true; } + else { + const x = f64FromBits(a), y = f64FromBits(b); + f.ZF = x === y; f.PF = false; f.CF = x < y; + } + f.OF = false; f.SF = false; +} + +function checkF64Source(o, op) { + if (isMem(o)) { + if (o.size && o.size !== 8) fail(`${op} reads eight bytes, one double; use qword ptr`); + o.size = 8; + } else if (!isVec(o)) fail(`${op} reads an xmm register or eight bytes of memory`); +} + +/** addsd/vaddsd and the rest: the SSE form is `op d, s` (d = d op s); the AVX form is `vop d, a, b` (d = a op b, with a's upper half carried over). */ +function scalarDouble(what, fn, cost) { + const sse = { + cls: { alu: true, write: true, vector: true, fp: cost }, + gloss: `${what}, on the low double of an xmm register; its upper eight bytes stay as they were`, + arity: 2, + check(ops, op) { if (!isVec(ops[0])) fail(`the first operand of ${op} must be an xmm register`); checkF64Source(ops[1], op); }, + exec(st, ops, ctx) { + const d = regValue(st, ops[0].name); + const b = low64(readOperand(st, ops[1], 8, ctx)); + writeOperand(st, ops[0], 16, upper64(d) | fn(low64(d), b), ctx); + }, + }; + const avx = { + cls: { alu: true, write: true, vector: true, avx: true, fp: cost }, + gloss: `${what}, on the low double: the second operand's low double with the third's, the result in the first, whose upper eight bytes are copied from the second`, + arity: 3, + check(ops, op) { + if (!isVec(ops[0]) || !isVec(ops[1])) fail(`the first two operands of ${op} must be xmm registers`); + checkF64Source(ops[2], op); + }, + exec(st, ops, ctx) { + const a = regValue(st, ops[1].name); + const b = low64(readOperand(st, ops[2], 8, ctx)); + writeOperand(st, ops[0], 16, upper64(a) | fn(low64(a), b), ctx); + }, + }; + return [sse, avx]; +} + +function scalarSqrt() { + const gloss = 'the square root of a double'; + const sse = { + cls: { alu: true, write: true, vector: true, fp: 'sqrt' }, + gloss: `${gloss}: the low double of the second operand, into the low double of the first; the first's upper eight bytes stay`, + arity: 2, + check(ops, op) { if (!isVec(ops[0])) fail(`the first operand of ${op} must be an xmm register`); checkF64Source(ops[1], op); }, + exec(st, ops, ctx) { + const d = regValue(st, ops[0].name); + writeOperand(st, ops[0], 16, upper64(d) | f64Sqrt(low64(readOperand(st, ops[1], 8, ctx))), ctx); + }, + }; + const avx = { + cls: { alu: true, write: true, vector: true, avx: true, fp: 'sqrt' }, + gloss: `${gloss}: the low double of the third operand, into the low double of the first; the first's upper eight bytes are copied from the second`, + arity: 3, + check(ops, op) { + if (!isVec(ops[0]) || !isVec(ops[1])) fail(`the first two operands of ${op} must be xmm registers`); + checkF64Source(ops[2], op); + }, + exec(st, ops, ctx) { + const a = regValue(st, ops[1].name); + writeOperand(st, ops[0], 16, upper64(a) | f64Sqrt(low64(readOperand(st, ops[2], 8, ctx))), ctx); + }, + }; + return [sse, avx]; +} + +/** movsd: a double between an xmm register and memory, or between two xmm registers. */ +function scalarMove(avx) { + return { + cls: { move: true, write: true, vector: true, ...(avx ? { avx: true } : {}) }, + gloss: avx + ? 'copy one double: from memory into the low eight bytes of an xmm register (clearing the rest), from a register to memory, or, with three registers, the third one\'s double with the second one\'s upper eight bytes' + : 'copy one double: from memory into the low eight bytes of an xmm register (clearing the rest), from a register to memory, or between registers (the destination keeps its upper eight bytes)', + arity: avx ? [2, 3] : 2, + check(ops, op) { + if (ops.length === 3) { + if (!(ops.every(isVec))) fail(`${op} with three operands takes three xmm registers`); + return; + } + const [d, s] = ops; + if (isVec(d) && isMem(s)) { checkF64Source(s, op); return; } + if (isMem(d) && isVec(s)) { checkF64Source(d, op); return; } + if (isVec(d) && isVec(s)) { if (avx) fail(`${op} between registers takes three xmm registers: the destination, the register whose upper half is kept, and the one whose double is copied`); return; } + fail(`${op} moves a double between an xmm register and eight bytes of memory, or between xmm registers`); + }, + exec(st, ops, ctx) { + if (ops.length === 3) { + writeOperand(st, ops[0], 16, upper64(regValue(st, ops[1].name)) | low64(regValue(st, ops[2].name)), ctx); + return; + } + const [d, s] = ops; + if (isMem(d)) { writeOperand(st, d, 8, low64(regValue(st, s.name)), ctx); return; } + if (isMem(s)) { writeOperand(st, d, 16, low64(readOperand(st, s, 8, ctx)), ctx); return; } + writeOperand(st, d, 16, upper64(regValue(st, d.name)) | low64(regValue(st, s.name)), ctx); + }, + }; +} + +/** cvtsi2sd / vcvtsi2sd: a signed 32- or 64-bit integer, as a double. */ +function convertIntToDouble(avx) { + return { + cls: { alu: true, write: true, vector: true, fp: 'cvt', ...(avx ? { avx: true } : {}) }, + gloss: avx + ? 'convert a signed integer (in a register or memory) to a double in the low eight bytes of the first operand; its upper eight bytes are copied from the second' + : 'convert a signed integer (in a register or memory) to a double in the low eight bytes of the first operand; its upper eight bytes stay', + arity: avx ? 3 : 2, + check(ops, op) { + const src = ops[ops.length - 1]; + if (!isVec(ops[0]) || (avx && !isVec(ops[1]))) fail(`the xmm operands of ${op} come first`); + if (isMem(src)) { if (src.size !== 4 && src.size !== 8) fail(`${op} reads a 32- or 64-bit integer from memory: say dword ptr or qword ptr`); } + else if (!(isReg(src) && (src.size === 4 || src.size === 8))) fail(`the integer operand of ${op} is a 32- or 64-bit register, or a dword or qword in memory`); + }, + exec(st, ops, ctx) { + const src = ops[ops.length - 1]; + const w = src.size; + const n = toSigned(readOperand(st, src, w, ctx), w); + const keep = upper64(regValue(st, ops[avx ? 1 : 0].name)); + writeOperand(st, ops[0], 16, keep | f64ToBits(Number(n)), ctx); + }, + }; +} + +/** ucomisd / comisd: compare two doubles into ZF, PF and CF. The two differ only in which NaNs raise an exception, and the exceptions are not modeled. */ +function compareDoubles(avx) { + return { + cls: { flags: true, compare: true, vector: true, fp: 'cmp', ...(avx ? { avx: true } : {}) }, + gloss: `compare the low doubles of two operands and report the answer in the flags: unordered (a NaN) sets ZF, PF and CF; greater clears all three; less sets CF; equal sets ZF; then use ja, jb, je, or jp to act on it`, + arity: 2, + check(ops, op) { if (!isVec(ops[0])) fail(`the first operand of ${op} must be an xmm register`); checkF64Source(ops[1], op); }, + exec(st, ops, ctx) { + const a = low64(regValue(st, ops[0].name)); + const b = low64(readOperand(st, ops[1], 8, ctx)); + f64CompareFlags(st, a, b); + }, + }; +} + +/** shld / shrd: shift the first operand, filling the vacated end with bits from the second, which is not changed. */ +function doubleShift(left) { + return { + cls: { alu: true, write: true, flags: true }, + gloss: left + ? 'shift the first operand left by a count (a constant or cl), filling from the right with the top bits of the second operand, which does not change; CF takes the last bit pushed out. A count of zero changes nothing, flags included' + : 'shift the first operand right by a count (a constant or cl), filling from the left with the bottom bits of the second operand, which does not change; CF takes the last bit pushed out. A count of zero changes nothing, flags included', + arity: 3, + check(ops, op) { + const [d, s, n] = ops; + if (!isRM(d)) fail(`the first operand of ${op} must be a register or a memory address`); + if (!isReg(s)) fail(`the second operand of ${op} must be an ordinary register: the bits shifted in come from it`); + if (s.size !== 4 && s.size !== 8) fail(`${op} works on 32- or 64-bit operands here`); + if (isMem(d)) d.size = d.size || s.size; + if (d.size !== s.size) fail(`both operands of ${op} must be the same width`); + if (!(isImm(n) || (isReg(n) && n.name === 'cl'))) fail(`${op} shifts by a constant or by cl, and by nothing else`); + }, + exec(st, ops, ctx) { + const w = ops[1].size, bits = BigInt(w * 8); + const d = readOperand(st, ops[0], w, ctx) & MASK[w]; + const s = regValue(st, ops[1].name) & MASK[w]; + const raw = ops[2].kind === 'imm' ? ops[2].value : regValue(st, 'cl'); + const n = raw & (w === 8 ? 63n : 31n); + if (n === 0n) return; + const res = left ? ((d << n) | (s >> (bits - n))) & MASK[w] : ((d >> n) | (s << (bits - n))) & MASK[w]; + st.flags.CF = left ? ((d >> (bits - n)) & 1n) === 1n : ((d >> (n - 1n)) & 1n) === 1n; + st.flags.OF = ((d ^ res) & SIGNBIT[w]) !== 0n; + st.flags.ZF = res === 0n; + st.flags.SF = (res & SIGNBIT[w]) !== 0n; + st.flags.PF = parityOf(res); + writeOperand(st, ops[0], w, res, ctx); + }, + }; +} + +{ + const [addsd, vaddsd] = scalarDouble('add two doubles', (a, b) => f64Binary(a, b, (x, y) => x + y), 'add'); + const [subsd, vsubsd] = scalarDouble('subtract the second double from the first', (a, b) => f64Binary(a, b, (x, y) => x - y), 'add'); + const [mulsd, vmulsd] = scalarDouble('multiply two doubles', (a, b) => f64Binary(a, b, (x, y) => x * y), 'mul'); + const [divsd, vdivsd] = scalarDouble('divide the first double by the second (division by zero gives infinity, and 0/0 gives NaN, without stopping the run)', (a, b) => f64Binary(a, b, (x, y) => x / y), 'div'); + const [sqrtsd, vsqrtsd] = scalarSqrt(); + Object.assign(MOJO_OPS, { + addsd, vaddsd, subsd, vsubsd, mulsd, vmulsd, divsd, vdivsd, sqrtsd, vsqrtsd, + movsd: scalarMove(false), vmovsd: scalarMove(true), + cvtsi2sd: convertIntToDouble(false), vcvtsi2sd: convertIntToDouble(true), + ucomisd: compareDoubles(false), vucomisd: compareDoubles(true), + comisd: compareDoubles(false), vcomisd: compareDoubles(true), + shld: doubleShift(true), shrd: doubleShift(false), + }); +} + +Object.assign(OPS, MOJO_OPS); + +// movabs is how a listing writes a mov carrying a full 64-bit constant. +OPS.movabs = { + ...OPS.mov, + gloss: 'copy a full sixty-four-bit constant into a register', + check(ops, op) { + if (!isReg(ops[0]) || ops[0].size !== 8 || !isImm(ops[1])) fail(`${op} puts a 64-bit constant into a 64-bit register`); + checkBinary(ops, op, { imm64: true }); + }, +}; + +// Conditional jumps, conditional moves, and flag-materializing setcc, +// generated from CONDITIONS. +for (const [cond, testFn] of Object.entries(CONDITIONS)) { + OPS['j' + cond] = { + cls: { control: true, conditional: true }, + gloss: `jump when ${CONDITION_TEXT[cond]}`, + cond, + arity: 1, + exec(st, ops, ctx) { + const taken = testFn(st.flags); + ctx.branch = { kind: 'conditional', cond, taken, target: ops[0].name }; + if (taken) ctx.jumpTo = ctx.labels[ops[0].name]; + }, + }; + OPS['cmov' + cond] = { + cls: { move: true, write: true, conditional: true }, + gloss: `copy the second operand into the first when ${CONDITION_TEXT[cond]}, and do nothing otherwise`, + cond, + arity: 2, + check(ops, op) { + checkDestReg(ops, op, [2, 4, 8]); + checkBinary(ops, op, { allowImm: false, destMem: false }); + }, + exec(st, ops, ctx) { + const w = operandWidth(ops); + const v = readOperand(st, ops[1], w, ctx); + const took = testFn(st.flags); + ctx.conditionalMove = { took }; + // The real instruction reads its source either way and writes its + // destination either way, which is exactly why it never mispredicts. + // When the condition fails, a 32-bit destination still has its upper + // half cleared, because the write happens regardless. + writeOperand(st, ops[0], w, took ? v : regValue(st, ops[0].name), ctx); + }, + }; + OPS['set' + cond] = { + cls: { write: true, conditional: true }, + gloss: `store 1 when ${CONDITION_TEXT[cond]}, otherwise store 0`, + cond, + arity: 1, + check(ops, op) { + const [d] = ops; + if (isMem(d)) d.size = d.size || 1; + if (!isRM(d) || d.size !== 1) fail(`${op} writes one byte: name an 8-bit register such as al or r9b`); + }, + exec(st, ops, ctx) { + writeOperand(st, ops[0], 1, testFn(st.flags) ? 1n : 0n, ctx); + }, + }; +} + +/* ------------------------------------------------------------------ */ +/* Branch predictors */ +/* ------------------------------------------------------------------ */ + +/** + * Toy predictors. Real hardware is far cleverer; a two-bit saturating + * counter per branch is enough to show why a coin-flip branch costs more + * than a loop branch. 'gshare' adds the one idea that makes real predictors + * good: it indexes its counters by the recent history of outcomes as well as + * by the branch, so a branch that alternates becomes predictable. + */ +export const PREDICTOR_KINDS = ['always-taken', 'never-taken', 'backward-taken', 'one-bit', 'two-bit', 'gshare']; + +export function makePredictor(kind = 'two-bit', options = {}) { + const table = new Map(); + const historyBits = options.historyBits ?? 4; + let history = 0; + const key = (pc) => (kind === 'gshare' ? `${pc}:${history}` : pc); + return { + kind, + predict(pc, backward = false) { + if (kind === 'always-taken') return true; + if (kind === 'never-taken') return false; + if (kind === 'backward-taken') return backward; + const s = table.get(key(pc)); + if (s === undefined) return kind === 'gshare' ? backward : true; + return kind === 'one-bit' ? s === 1 : s >= 2; + }, + update(pc, taken) { + if (kind === 'always-taken' || kind === 'never-taken' || kind === 'backward-taken') return; + const k = key(pc); + if (kind === 'one-bit') table.set(k, taken ? 1 : 0); + else { + const s = table.get(k) ?? (taken ? 2 : 1); + table.set(k, taken ? Math.min(3, s + 1) : Math.max(0, s - 1)); + } + if (kind === 'gshare') history = ((history << 1) | (taken ? 1 : 0)) & ((1 << historyBits) - 1); + }, + state(pc) { return table.get(key(pc)); }, + history() { return history; }, + }; +} + +/* ------------------------------------------------------------------ */ +/* Runner */ +/* ------------------------------------------------------------------ */ + +function snapshot(state) { + return { + regs: { ...state.regs }, + flags: { ...state.flags }, + mem: state.mem, + pc: state.pc, + halted: state.halted, + depth: state.depth, + out: state.out, + exitCode: state.exitCode, + }; +} + +/** Lay the program out at byte addresses, or explain why it cannot be. */ +function safeLayout(program, base) { + program.layouts = program.layouts || {}; + if (!(base in program.layouts)) { + try { program.layouts[base] = layoutProgram(program, base); } catch (err) { program.layouts[base] = { ok: false, error: err.message }; } + } + return program.layouts[base]; +} + +/** + * Run a spec and record everything a view could want. + * + * spec = { + * code: assembly text (required) + * registers: { rdi: 0x1000, rsi: 17, ... } starting register values + * flags: { CF: true } starting flags + * memory: [ { name, label, addr, text } ] byte regions + * maxSteps: cap on executed instructions (default 20000) + * predictor: 'two-bit' | 'one-bit' | 'always-taken' | 'never-taken' + * | 'backward-taken' | 'gshare' + * returns: register holding the answer, default 'rax' + * codeBase: address of the first instruction, default 0x401000 + * stackTop, stackSize: where the stack sits + * stdin: text the read system call hands back + * } + */ +export function run(spec) { + const program = spec.program || assemble(spec.code); + const maxSteps = spec.maxSteps ?? 20000; + const predictor = makePredictor(spec.predictor || 'two-bit', spec.predictorOptions); + const codeBase = spec.codeBase ?? 0x401000; + const layout = safeLayout(program, codeBase); + + const stackTop = spec.stackTop ?? 0x7ffff000; + const stackBytes = spec.stackSize ?? 512; + const regionSpecs = [ + ...(spec.memory || []), + // Every machine gets a stack, so push, pop, and call need no setup. + { name: 'stack', label: 'stack', addr: stackTop - stackBytes, size: stackBytes + 8, kind: 'stack' }, + ]; + + const state = { + regs: Object.fromEntries(ALL_SLOTS.map((r) => [r, 0n])), + flags: { ZF: false, SF: false, CF: false, OF: false, PF: false, ...(spec.flags || {}) }, + mem: makeMemory(regionSpecs), + pc: 0, + halted: false, + depth: 0, + dirty: new Set(), + out: '', + exitCode: null, + stdin: spec.stdin || '', + inPos: 0, + rip: null, + }; + state.regs.rsp = BigInt(stackTop); + for (const [name, value] of Object.entries(spec.registers || {})) { + if (!REGS[name.toLowerCase()]) throw new AsmError(`the spec names a register that does not exist: ${name}`); + regWrite(state, name, BigInt.asUintN(128, BigInt(value))); + } + + const addrOf = (i) => (layout.ok ? layout.addr[i] : i); + const nextAddrOf = (i) => (layout.ok ? layout.addr[i] + layout.len[i] : i + 1); + + const states = [snapshot(state)]; + const steps = []; + const hits = new Array(program.instrs.length).fill(0); + const stats = { + instructions: 0, + reads: 0, + writes: 0, + comparisons: 0, + branches: 0, + taken: 0, + mispredicts: 0, + syscalls: 0, + byOp: {}, + branchSites: {}, + truncated: false, + error: null, + fault: null, + }; + + while (!state.halted) { + if (stats.instructions >= maxSteps) { stats.truncated = true; break; } + if (state.pc < 0 || state.pc >= program.instrs.length) break; + + const instr = program.instrs[state.pc]; + const before = states[states.length - 1]; + const ctx = { + labels: program.labels, pc: state.pc, instr, accesses: [], effective: null, jumpTo: null, branch: null, + halt: false, compared: null, addressOnly: false, vectorCompare: null, alu: null, syscall: null, + returnAddress: BigInt(nextAddrOf(state.pc)), + resolveReturn(addr) { + let target = layout.ok ? layout.byAddr.get(Number(addr)) : Number(addr); + if (layout.ok && Number(addr) === layout.end) target = program.instrs.length; + if (target === undefined || target < 0 || target >= program.instrs.length) { + throw new AsmError(`ret popped ${formatHex(addr)} and tried to continue there, and no instruction starts at that address. Something pushed without popping, or popped without pushing, and the return address was not on top of the stack.`, 'bad-return'); + } + return target; + }, + }; + state.rip = layout.ok ? nextAddrOf(state.pc) : null; + + state.dirty = new Set(); + try { + OPS[instr.op].exec(state, instr.operands, ctx); + } catch (err) { + const stackBottom = stackTop - stackBytes; + if (err.kind === 'page-fault' && err.addr < stackBottom && err.addr >= stackBottom - 4096) { + err.message += '. That is just below the stack: it has run out of room, which is a stack overflow'; + err.kind = 'stack-overflow'; + } + stats.error = `line ${instr.lineNo + 1}: ${err.message}`; + stats.fault = { kind: err.kind || 'error', message: err.message, line: instr.lineNo, instr: instr.index, addr: err.addr ?? null, step: steps.length }; + break; + } + + hits[instr.index]++; + stats.instructions++; + stats.byOp[instr.op] = (stats.byOp[instr.op] || 0) + 1; + for (const a of ctx.accesses) { if (a.kind === 'read') stats.reads++; else stats.writes++; } + if (OPS[instr.op].cls.compare) stats.comparisons++; + if (ctx.syscall) stats.syscalls++; + + let prediction = null; + if (ctx.branch && ctx.branch.kind === 'conditional') { + stats.branches++; + if (ctx.branch.taken) stats.taken++; + const backward = program.labels[ctx.branch.target] <= instr.index; + const predicted = predictor.predict(instr.index, backward); + const correct = predicted === ctx.branch.taken; + if (!correct) stats.mispredicts++; + predictor.update(instr.index, ctx.branch.taken); + prediction = { predicted, correct, counter: predictor.state(instr.index), backward }; + const site = stats.branchSites[instr.index] || { line: instr.lineNo, text: instr.text, seen: 0, taken: 0, mispredicts: 0 }; + site.seen++; + if (ctx.branch.taken) site.taken++; + if (!correct) site.mispredicts++; + stats.branchSites[instr.index] = site; + } + + const nextPc = ctx.halt ? state.pc : (ctx.jumpTo !== null ? ctx.jumpTo : state.pc + 1); + if (ctx.halt) state.halted = true; + state.pc = nextPc; + const after = snapshot(state); + + steps.push({ + index: steps.length, + instr, + pcBefore: before.pc, + pcAfter: after.pc, + address: addrOf(instr.index), + diff: diffStates(before, after), + accesses: ctx.accesses, + effective: ctx.effective, + addressOnly: ctx.addressOnly, + compared: ctx.compared, + vectorCompare: ctx.vectorCompare, + conditionalMove: ctx.conditionalMove ?? null, + alu: ctx.alu, + syscall: ctx.syscall, + branch: ctx.branch, + prediction, + halted: state.halted, + }); + states.push(after); + } + + stats.result = states[states.length - 1].regs[spec.returns || 'rax']; + stats.output = state.out; + stats.exitCode = state.exitCode; + return { spec, program, layout, states, steps, stats, hits, count: steps.length }; +} + +/** Which registers, flags, and bytes changed between two snapshots. */ +export function diffStates(a, b) { + const regs = []; + for (const r of ALL_SLOTS) { + if (a.regs[r] !== b.regs[r]) regs.push({ name: r, before: a.regs[r], after: b.regs[r], vector: r.startsWith('xmm') }); + } + const flags = []; + for (const f of FLAG_NAMES) { + if (a.flags[f] !== b.flags[f]) flags.push({ name: f, before: a.flags[f], after: b.flags[f] }); + } + const mem = []; + if (a.mem !== b.mem) { + for (let i = 0; i < b.mem.length; i++) { + if (a.mem[i] === b.mem[i]) continue; + const ra = a.mem[i].bytes; + const rb = b.mem[i].bytes; + for (let k = 0; k < rb.length; k++) { + if (ra[k] !== rb[k]) mem.push({ addr: b.mem[i].addr + k, before: ra[k], after: rb[k] }); + } + } + } + const out = a.out !== b.out ? b.out.slice(a.out.length) : ''; + return { regs, flags, mem, out }; +} + +/* ------------------------------------------------------------------ */ +/* Reading real compiler output */ +/* ------------------------------------------------------------------ */ + +/** + * Turn the output of `objdump -d -M intel` into assembly this library can + * run. Real output carries byte addresses, encoded bytes, and numeric jump + * targets; this replaces the targets with labels and drops the rest. + * + * fromObjdump(text) -> { code, replaced, targets, dropped, original } + * + * `replaced` names any mnemonic outside this library's subset, or any line + * the assembler refuses. Those become `nop` with the original preserved as a + * comment, so a listing still runs and the student can see exactly what was + * ignored. `original` maps each instruction back to the bytes objdump + * printed, which the encoding view can set beside its own. + */ +export function fromObjdump(text, options = {}) { + const opts = { unknown: 'nop', ...options }; + const raw = String(text).replace(/\r/g, '').split('\n'); + const parsed = []; + const targets = new Set(); + const replaced = []; + const dropped = []; + + const LINE = /^\s*([0-9a-f]+):\s+((?:[0-9a-f]{2}\s)+)\s*(.*)$/i; + const HEADER = /^[0-9a-f]+\s+<([^>]+)>:\s*$/i; + + for (const line of raw) { + const head = HEADER.exec(line.trim()); + if (head) { parsed.push({ kind: 'header', name: head[1] }); continue; } + const m = LINE.exec(line + ' '); + if (!m) { if (line.trim().length) dropped.push(line.trim()); continue; } + const body = m[3] + .replace(/<[^>]*>/g, '') + .replace(/\s+#.*$/, '') + .replace(/\b(BYTE|WORD|DWORD|QWORD|XMMWORD) PTR\b/gi, (w) => w.toLowerCase()) + .replace(/\b(ds|cs|es|ss):/gi, '') + .trim(); + if (!body.length) { + // a continuation line carrying the rest of a long instruction's bytes + const prev = parsed[parsed.length - 1]; + if (prev && prev.kind === 'instr') prev.bytes += ' ' + m[2].trim(); + continue; + } + const mn = (/^([a-z][a-z0-9]*)/i.exec(body) || [null, ''])[1].toLowerCase(); + const rest = body.slice(mn.length).trim(); + const jumpTarget = /^(j[a-z]+|call|loop)$/.test(mn) ? (/^([0-9a-f]+)$/i.exec(rest) || [])[1] : null; + if (jumpTarget) targets.add(jumpTarget.toLowerCase().replace(/^0+(?=.)/, '')); + parsed.push({ kind: 'instr', addr: m[1].toLowerCase().replace(/^0+(?=.)/, ''), mn, rest, jumpTarget: jumpTarget?.toLowerCase().replace(/^0+(?=.)/, ''), original: body, bytes: m[2].trim() }); + } + + const label = (addr) => `.L${addr}`; + const out = []; + const original = []; + for (const item of parsed) { + if (item.kind === 'header') { out.push(`; ${item.name}`); continue; } + if (targets.has(item.addr)) out.push(`${label(item.addr)}:`); + let line; + let ok = false; + if (item.jumpTarget && OPS[item.mn]) { + line = ` ${item.mn.padEnd(7)} ${label(item.jumpTarget)}`; + ok = true; + } else if (OPS[item.mn]) { + line = ` ${item.mn.padEnd(7)} ${item.rest}`.replace(/\s+$/, ''); + try { validate(item.mn, splitOperands(item.rest).map(parseOperand)); ok = true; } catch { ok = false; } + } + if (!ok) { + replaced.push(item.mn); + if (opts.unknown === 'drop') continue; + line = ` nop ; ${item.original}`; + } + out.push(line); + original.push({ addr: item.addr, bytes: item.bytes, text: item.original, kept: ok }); + } + + return { + code: out.join('\n') + '\n', + replaced: [...new Set(replaced)], + targets: [...targets].map(label), + dropped, + original, + }; +} + +/* ------------------------------------------------------------------ */ +/* Formatting helpers used by the views */ +/* ------------------------------------------------------------------ */ + +export function formatHex(v, bytes = 8) { + const value = BigInt(v) & MASK[bytes]; + return '0x' + value.toString(16).padStart(bytes * 2, '0'); +} + +export function formatSigned(v, bytes = 8) { + let value = BigInt(v) & MASK[bytes]; + if (value & SIGNBIT[bytes]) value -= MASK[bytes] + 1n; + return value.toString(10); +} + +export function formatBinary(v, bytes = 1, group = 4) { + const s = (BigInt(v) & MASK[bytes]).toString(2).padStart(bytes * 8, '0'); + return group ? s.replace(new RegExp(`(.{${group}})(?=.)`, 'g'), '$1 ') : s; +} + +/** Printable ASCII for a byte, or a dot. */ +export function byteChar(b) { + return b >= 32 && b < 127 ? String.fromCharCode(b) : '.'; +} + +export function memoryRegions(trace, stepIndex) { + const s = trace.states[Math.max(0, Math.min(stepIndex, trace.states.length - 1))]; + return s.mem; +} diff --git a/src/engines/asm86/machine.js b/src/engines/asm86/machine.js new file mode 100644 index 0000000..54a7cbe --- /dev/null +++ b/src/engines/asm86/machine.js @@ -0,0 +1,332 @@ +/** + * asm86/machine.js + * + * One entry point for everything below the instruction set. + * + * interp.js says what a program does. This file says what it costs, by + * replaying the recorded trace through a model machine: a cache hierarchy, + * optionally virtual memory, a classic five-stage pipeline, and an + * out-of-order core. Every model reads the same trace, so they all agree + * about results and can be compared about time, which is the lesson. + * + * A spec picks its machine with a `machine` field: + * + * machine: 'teaching' tiny caches, so misses happen on small data + * machine: 'realistic' sizes and latencies shaped like a desktop core + * machine: 'ideal' every access hits, every branch is free + * machine: { preset: 'teaching', caches: { prefetch: 'next-line' } } + * machine: { preset: 'teaching', caches: { warm: true } } start with the caches + * as a previous run of the same program left them + * + * Views call hardwareFor(trace) and ask for the part they draw. Each part is + * computed on first request and kept, so a page of widgets over one trace + * does the work once. + */ + +import { Hierarchy } from './cache.js'; +import { AddressSpace } from './paging.js'; +import { simulatePipeline } from './pipeline.js'; +import { simulateOoO } from './ooo.js'; + +/* ------------------------------------------------------------------ */ +/* Presets */ +/* ------------------------------------------------------------------ */ + +export const PRESETS = { + /** + * Small enough that a 17-letter sequence can show every kind of miss. + * L1d: 8 sets x 2 ways x 16-byte lines = 256 bytes. + */ + teaching: { + name: 'teaching', + description: 'caches small enough to watch fill and overflow on the course’s data', + caches: { + l1i: { size: 256, line: 16, ways: 2, latency: 1, policy: 'lru' }, + l1d: { size: 256, line: 16, ways: 2, latency: 4, policy: 'lru' }, + l2: { size: 1024, line: 16, ways: 4, latency: 12, policy: 'lru' }, + l3: null, + memory: { latency: 60 }, + prefetch: 'none', + }, + paging: { enabled: false, pageSize: 4096, tlb: { entries: 4 } }, + pipeline: { forwarding: true, resolveIn: 'EX', memoryStalls: true, kernelCycles: 100 }, + core: { width: 4, rob: 32, rs: 16, mispredictPenalty: 8, fusion: true, eliminate: true, storeBuffer: 8, storeForward: 5, kernelCycles: 100 }, + clockGHz: 1, + }, + /** Shaped like a desktop core of the late 2010s. */ + realistic: { + name: 'realistic', + description: 'cache sizes and latencies shaped like a desktop core', + caches: { + l1i: { size: 32768, line: 64, ways: 8, latency: 1, policy: 'lru' }, + l1d: { size: 32768, line: 64, ways: 8, latency: 4, policy: 'lru' }, + l2: { size: 262144, line: 64, ways: 4, latency: 12, policy: 'lru' }, + l3: { size: 8388608, line: 64, ways: 16, latency: 42, policy: 'lru' }, + memory: { latency: 200 }, + prefetch: 'next-line', + }, + paging: { enabled: false, pageSize: 4096, tlb: { entries: 64 } }, + pipeline: { forwarding: true, resolveIn: 'EX', memoryStalls: true, kernelCycles: 150 }, + core: { width: 4, rob: 224, rs: 97, mispredictPenalty: 16, fusion: true, eliminate: true, storeBuffer: 56, storeForward: 5, kernelCycles: 150 }, + clockGHz: 4, + }, + /** + * Every access hits in one cycle, and the out-of-order core pays nothing + * for a wrong guess: the ceiling the code itself allows. (The five-stage + * pipeline still flushes its two wrong-path fetches.) + */ + ideal: { + name: 'ideal', + description: 'every memory access in 1 cycle and no fetch stalls, to isolate dependencies and ports', + caches: { + l1i: { size: 1 << 20, line: 64, ways: 16, latency: 1 }, + l1d: { size: 1 << 20, line: 64, ways: 16, latency: 1 }, + l2: null, l3: null, memory: { latency: 1 }, prefetch: 'none', + }, + paging: { enabled: false, pageSize: 4096, tlb: { entries: 1024 } }, + pipeline: { forwarding: true, resolveIn: 'EX', memoryStalls: false, kernelCycles: 1 }, + core: { width: 4, rob: 224, rs: 97, mispredictPenalty: 0, fusion: true, eliminate: true, storeBuffer: 56, storeForward: 1, kernelCycles: 1 }, + clockGHz: 1, + }, +}; + +function merge(a, b) { + if (b === undefined) return a; + if (b === null || typeof b !== 'object' || Array.isArray(b)) return b; + const out = { ...(a || {}) }; + for (const [k, v] of Object.entries(b)) out[k] = merge(a?.[k], v); + return out; +} + +/** Turn whatever a spec says into a complete machine description. */ +export function resolveMachine(m) { + if (!m) return PRESETS.teaching; + if (typeof m === 'string') { + if (!PRESETS[m]) throw new Error(`no machine preset named "${m}"; choose ${Object.keys(PRESETS).join(', ')}`); + return PRESETS[m]; + } + const base = PRESETS[m.preset || 'teaching']; + if (!base) throw new Error(`no machine preset named "${m.preset}"; choose ${Object.keys(PRESETS).join(', ')}`); + const out = merge(base, { ...m, preset: undefined }); + delete out.preset; + out.name = m.name || [base.name, ...describeChanges(base, out)].join(', '); + if (!describeChanges(base, out).length) out.name = base.name; + return out; +} + +/** A few words for how a machine differs from its preset, for view captions. */ +function describeChanges(base, m) { + const out = []; + const c = m.caches; const b = base.caches; + if (c.warm && !b.warm) out.push('warm caches'); + if (c.prefetch !== b.prefetch) out.push(c.prefetch === 'none' ? 'no prefetcher' : 'next-line prefetcher'); + for (const lvl of ['l1i', 'l1d', 'l2', 'l3']) { + if (JSON.stringify(c[lvl]) === JSON.stringify(b[lvl])) continue; + if (!c[lvl]) out.push(`no ${lvl.toUpperCase()}`); + else if (c[lvl].size !== b[lvl]?.size) out.push(`${c[lvl].size >= 1024 ? `${c[lvl].size / 1024} KiB` : `${c[lvl].size} B`} ${lvl.replace('l1', 'L1').toUpperCase().replace('L1I', 'L1i').replace('L1D', 'L1d')}`); + else out.push(`${lvl.toUpperCase()} changed`); + } + if (m.paging?.enabled && !base.paging?.enabled) out.push(`paging with a ${m.paging.tlb?.entries ?? 4}-entry TLB`); + if (m.pipeline?.forwarding === false && base.pipeline.forwarding) out.push('no forwarding'); + if (m.pipeline?.resolveIn && m.pipeline.resolveIn !== base.pipeline.resolveIn) out.push(`branches settled in ${m.pipeline.resolveIn}`); + for (const k of ['width', 'rob', 'rs', 'mispredictPenalty']) if (m.core?.[k] !== base.core[k]) out.push(`${k === 'rob' ? 'ROB' : k === 'rs' ? 'scheduler' : k === 'width' ? 'width' : 'mispredict penalty'} ${m.core[k]}`); + if (m.core?.fusion === false && base.core.fusion) out.push('no fusion'); + if (m.core?.eliminate === false && base.core.eliminate) out.push('no move elimination'); + return out; +} + +/* ------------------------------------------------------------------ */ +/* The memory pass */ +/* ------------------------------------------------------------------ */ + +const CHECKPOINT_EVERY = 128; + +/** Everything the memory system did for one executed instruction. */ +function processStep(h, space, trace, k) { + const step = trace.steps[k]; + const layout = trace.layout; + const rec = { fetch: null, data: [], translations: [], walks: [] }; + const cost = { fetch: null, loads: [], stores: [] }; + + const translate = (vaddr, side) => { + if (!space) return { paddr: vaddr, extra: 0, tr: null }; + const tr = space.translate(vaddr, side); + rec.translations.push(tr); + let extra = 0; + if (tr.walk) { + // The page walker reads one entry per level, each through the data caches. + for (const w of tr.walk) { + const r = h.access(w.entryAddr, 8, 'read', 'data'); + r.walkLevel = w.level; + w.access = r; + extra += r.latency; + rec.walks.push(r); + } + } + return { paddr: tr.paddr, extra, tr }; + }; + + if (layout?.ok) { + const vaddr = layout.addr[step.instr.index]; + const t = translate(vaddr, 'i'); + const r = h.access(t.paddr, layout.len[step.instr.index], 'read', 'inst'); + r.vaddr = vaddr; r.translation = t.tr; + rec.fetch = r; + cost.fetch = { latency: r.latency + t.extra, level: r.level }; + } + for (const a of step.accesses) { + // With paging on, an access that crosses a page boundary is two + // translations and two physical accesses: the pages need not be adjacent. + const pieces = []; + if (space) { + let at = a.addr; let left = a.size; + while (left > 0) { + const room = 4096 - (at % 4096); + const n = Math.min(left, room); + pieces.push([at, n]); at += n; left -= n; + } + } else pieces.push([a.addr, a.size]); + let latency = 0; let level = 'L1d'; let walk = 0; let split = pieces.length > 1; + for (const [addr, size] of pieces) { + const t = translate(addr, 'd'); + const r = h.access(t.paddr, size, a.kind, 'data'); + r.vaddr = addr; r.translation = t.tr; r.kernel = Boolean(a.kernel); r.stack = a.stack || null; + if (pieces.length > 1) r.pageSplit = true; + rec.data.push(r); + if (r.latency + t.extra > latency) { latency = r.latency + t.extra; level = r.level; } + walk += t.extra; split = split || r.split; + } + (a.kind === 'read' ? cost.loads : cost.stores).push({ latency, level, walk, split }); + } + return { rec, cost }; +} + +function simulateMemory(trace, cfg) { + const h = new Hierarchy(cfg.caches); + const space = cfg.paging?.enabled ? new AddressSpace(cfg.paging) : null; + if (cfg.caches.warm) { + // Run the whole program once and throw the results away, so the second + // run starts with whatever the first left in the caches and TLB. This is + // the difference between the first call of a function and the thousandth. + for (let k = 0; k < trace.steps.length; k++) processStep(h, space, trace, k); + for (const c of h.all) for (const key of Object.keys(c.stats)) c.stats[key] = 0; + h.memStats = { reads: 0, writes: 0 }; + if (space) for (const key of Object.keys(space.stats)) space.stats[key] = 0; + } + const steps = []; + const stepCost = []; + const checkpoints = []; + for (let k = 0; k < trace.steps.length; k++) { + if (k % CHECKPOINT_EVERY === 0) checkpoints.push({ t: k, h: h.clone(), space: space ? space.clone() : null }); + const { rec, cost } = processStep(h, space, trace, k); + steps.push(rec); + stepCost.push(cost); + } + const final = { h, space }; + + /** The caches and TLB as they stood after t steps. */ + const memo = new Map(); + function stateAt(t) { + const tt = Math.max(0, Math.min(t, trace.steps.length)); + if (tt === trace.steps.length) return final; + if (memo.has(tt)) return memo.get(tt); + const result = replayTo(tt); + memo.set(tt, result); + if (memo.size > 8) memo.delete(memo.keys().next().value); + return result; + } + function replayTo(tt) { + let cp = checkpoints[0]; + for (const c of checkpoints) if (c.t <= tt) cp = c; + const hh = cp.h.clone(); + const ss = cp.space ? cp.space.clone() : null; + for (let k = cp.t; k < tt; k++) processStep(hh, ss, trace, k); + return { h: hh, space: ss }; + } + + // Running totals per step, for counters that move as the widget steps. + const running = []; + const acc = { l1dHits: 0, l1dMisses: 0, l1iMisses: 0, l2Misses: 0, l3Misses: 0, memory: 0, tlbMisses: 0, memCycles: 0 }; + running.push({ ...acc }); + steps.forEach((rec) => { + for (const r of [...rec.data, ...(rec.fetch ? [rec.fetch] : []), ...rec.walks]) { + for (const line of r.lines) { + for (const e of line.trail) { + if (e.level === 'L1d') { if (e.hit) acc.l1dHits++; else acc.l1dMisses++; } + if (e.level === 'L1i' && !e.hit) acc.l1iMisses++; + if (e.level === 'L2' && !e.hit) acc.l2Misses++; + if (e.level === 'L3' && !e.hit) acc.l3Misses++; + if (e.level === 'memory') acc.memory++; + } + } + } + for (const tr of rec.translations) if (!tr.tlbHit) acc.tlbMisses++; + for (const r of rec.data) acc.memCycles += r.latency; + running.push({ ...acc }); + }); + + return { steps, stepCost, stateAt, final, running, hierarchy: h, space }; +} + +/* ------------------------------------------------------------------ */ +/* The hardware object views share */ +/* ------------------------------------------------------------------ */ + +const cache = new WeakMap(); + +/** + * hardwareFor(trace, machine?) -> { + * config, + * memory() caches, TLB, per-step records, stateAt(t) + * pipeline() the five-stage model + * core() the out-of-order model + * perf() totals in the style of `perf stat` + * } + */ +/** @param {any} trace @param {any} [machine] the machine preset or spec; absent: the trace's own */ +export function hardwareFor(trace, machine) { + const cfg = resolveMachine(machine ?? trace.spec?.machine); + const key = JSON.stringify(cfg); + let byCfg = cache.get(trace); + if (!byCfg) { byCfg = new Map(); cache.set(trace, byCfg); } + if (byCfg.has(key)) return byCfg.get(key); + + const l1 = { data: cfg.caches.l1d.latency, inst: cfg.caches.l1i?.latency ?? 1 }; + let mem = null; let pipe = null; let core = null; + const hw = { + config: cfg, + trace, + l1, + memory() { return mem || (mem = simulateMemory(trace, cfg)); }, + pipeline() { return pipe || (pipe = simulatePipeline(trace, cfg.pipeline, hw.memory(), l1)); }, + core() { return core || (core = simulateOoO(trace, cfg.core, hw.memory(), l1)); }, + perf() { + const m = hw.memory(); const p = hw.pipeline(); const c = hw.core(); + const h = m.final.h; + const s = trace.stats; + const l1d = h.l1d.stats; const l1i = h.l1i?.stats; + return { + instructions: s.instructions, + uops: c.uopCount, + cyclesInOrder: p.cycles, + cyclesOoO: c.cycles, + ipc: c.ipc, + cpiInOrder: p.cpi, + branches: s.branches, + mispredicts: s.mispredicts, + l1dLoads: l1d.accesses, + l1dMisses: l1d.misses, + l1iMisses: l1i ? l1i.misses : 0, + l2Misses: h.shared[0]?.stats.misses ?? 0, + l3Misses: h.shared[1]?.stats.misses ?? 0, + memoryReads: h.memStats.reads, + tlbMisses: m.space ? m.space.stats.tlbMisses : null, + seconds: c.cycles / (cfg.clockGHz * 1e9), + bounds: c.bounds, + stalls: c.stallCycles, + }; + }, + }; + byCfg.set(key, hw); + return hw; +} diff --git a/src/engines/asm86/ooo.js b/src/engines/asm86/ooo.js new file mode 100644 index 0000000..2fe2803 --- /dev/null +++ b/src/engines/asm86/ooo.js @@ -0,0 +1,370 @@ +/** + * asm86/ooo.js + * + * An out-of-order core, the kind in every laptop and server since the + * mid-1990s. + * + * The five-stage pipeline runs instructions in the order written and stalls + * the whole line when one of them waits. An out-of-order core does not. + * It works like this: + * + * 1. The front end fetches and decodes several instructions a cycle, + * splitting each into uops. + * 2. Rename gives every result a fresh physical register. Two uops that + * both write rax no longer interfere, so the only waits left are the + * real ones: a uop that needs a value waits for whoever produces it. + * 3. Each uop takes a slot in the reorder buffer (ROB), in program order, + * and a place in the scheduler. + * 4. Every cycle, the scheduler sends any uop whose inputs are ready to a + * free execution port that can run it, oldest first. This is the + * out-of-order part: a uop can run long before an older one that is + * still waiting for memory. + * 5. Uops retire from the ROB strictly in program order, once finished. + * Retirement is where results become official, which is why the + * program still sees everything happen in order. + * + * The consequence students need: a loop's speed is set by the longest chain + * of dependent uops running through it, or by the busiest port, or by the + * front end, whichever is worst, and not by how many instructions it has. + * The `bounds` this model reports are those three limits. + * + * Trace-driven, like the pipeline model: only the path the program took is + * simulated. A mispredicted branch blocks the front end until the branch + * executes, plus a refill penalty, which is what the wrong-path work costs. + */ + +import { decompose, canFuse, portsFor, UOP_KINDS } from './uops.js'; + +/** + * config = { + * width: 4, uops renamed and retired per cycle + * rob: 64, reorder buffer entries + * rs: 32, scheduler entries + * mispredictPenalty: 12, cycles to refill the front end after a wrong guess + * fusion: true, cmp/test + jcc become one uop + * eliminate: true, zero idioms and register moves handled at rename + * storeBuffer: 16, store-buffer entries: stores in flight, and those + * retired but not yet written to the cache + * storeForward: 5, latency of a load served by the store buffer + * kernelCycles: 100 what a system call costs + * } + */ +export function simulateOoO(trace, config, mem, l1) { + const C = { + width: 4, rob: 64, rs: 32, mispredictPenalty: 12, fusion: true, eliminate: true, + storeBuffer: 16, storeForward: 5, kernelCycles: 100, ...config, + }; + + /* ---------- 1. Decode every executed instruction into uops ---------- */ + const uops = []; + const insts = []; + trace.steps.forEach((step, k) => { + const instr = step.instr; + const prevInst = insts[k - 1]; + const inst = { index: k, step, instr, uops: [], fusedInto: null }; + insts.push(inst); + const reads = step.accesses.filter((a) => a.kind === 'read'); + const writes = step.accesses.filter((a) => a.kind === 'write'); + const cost = mem?.stepCost[k]; + + // Macro-fusion: the compare before this branch already became a uop; + // turn that uop into the branch rather than adding another. + if (C.fusion && prevInst && prevInst.uops.length && canFuse(prevInst.instr, instr)) { + const host = uops[prevInst.uops[prevInst.uops.length - 1]]; + if (host && (host.kind === 'alu') && !host.fused) { + host.kind = 'branch'; + host.ports = portsFor('branch'); + host.fused = true; + host.note = `${prevInst.instr.op} and ${instr.op} fused into one micro-op: compare and branch together`; + host.branch = step.branch; host.mispredicted = Boolean(step.prediction && !step.prediction.correct); + host.taken = Boolean(step.branch?.taken); + host.srcs = [...new Set([...host.srcs])]; + inst.fusedInto = host.id; + inst.uops.push(host.id); + host.alsoInst = k; + // the jump's own bytes still have to be fetched before the pair can go + if (cost?.fetch && (!prevInst.fetch || cost.fetch.latency > prevInst.fetch.latency)) prevInst.fetch = cost.fetch; + return; + } + } + + const parts = decompose(instr, { eliminate: C.eliminate }); + let r = 0; let w = 0; + const temps = {}; + for (const p of parts) { + const u = { + id: uops.length, inst: k, kind: p.kind, note: p.note, + srcs: p.srcs, dsts: p.dsts, latency: p.latency, + ports: p.kind === 'eliminated' || p.kind === 'nop' || p.kind === 'system' ? [] : portsFor(p.kind === 'locked' ? 'load' : p.kind), + unpipelined: Boolean(p.unpipelined), + producers: [], memProducer: null, memProducers: [], + alloc: null, issue: null, port: null, complete: null, retire: null, + taken: p.taken ?? false, + }; + if (p.kind === 'load' || p.kind === 'locked') { + const acc = reads[r]; const c = cost?.loads[r]; r++; + u.access = acc || null; + u.latency = (p.kind === 'locked' ? UOP_KINDS.locked.latency : 0) + (c ? c.latency : l1.data); + u.level = c?.level ?? 'L1d'; + if (c?.walk) u.walk = c.walk; + } + if (p.kind === 'store-addr') { u.access = writes[w] || null; u.storeLatency = cost?.stores[w]?.latency ?? l1.data; w++; } + if (p.kind === 'locked') { u.storeAccess = writes[w] || null; u.storeLatency = cost?.stores[w]?.latency ?? l1.data; w++; } + if (p.kind === 'branch') { + u.branch = step.branch; + u.mispredicted = Boolean(step.prediction && !step.prediction.correct); + u.taken = Boolean(step.branch?.taken); + } + if (p.kind === 'system') u.latency = C.kernelCycles; + // temps named inside one instruction resolve to uops of the same instruction + u.srcs = u.srcs.map((s) => (s.startsWith('t') && /^t\d$/.test(s) ? temps[s] ?? null : s)).filter((s) => s !== null); + for (const d of p.dsts) if (/^t\d$/.test(d)) temps[d] = { temp: u.id }; + uops.push(u); + inst.uops.push(u.id); + } + if (cost?.fetch) inst.fetch = cost.fetch; + }); + + /* ---------- 2. Rename: find each uop's producers ---------- */ + const lastWriter = new Map(); + const stores = []; // store-data uops, with their store-addr access + let lastLocked = null; // a locked instruction holds back every younger load + for (const u of uops) { + const producers = new Set(); + for (const s of u.srcs) { + if (typeof s === 'object') { producers.add(s.temp); continue; } + const p = lastWriter.get(s); + if (p !== undefined) producers.add(p); + } + u.producers = [...producers].filter((p) => p !== u.id); + u.renamedSrcs = u.srcs.map((s) => (typeof s === 'object' ? { name: `t`, from: s.temp } : { name: s, from: lastWriter.get(s) ?? null })); + // Memory. A load that reads bytes a recent store wrote cannot use the + // cache: the value is still in the store buffer. If the youngest such + // store covers every byte the load wants, the store buffer forwards it. + // If not, the load must wait for every overlapping store to finish and + // then pays a store-forwarding stall on top of the cache. + if ((u.kind === 'load' || u.kind === 'locked') && u.access) { + const a = u.access; + const overlapping = []; + for (let k = stores.length - 1, seen = 0; k >= 0 && seen < C.storeBuffer; k--, seen++) { + const s = stores[k]; + if (s.addr < a.addr + a.size && a.addr < s.addr + s.size) overlapping.push(s); + } + if (overlapping.length) { + const young = overlapping[0]; + u.memProducer = young.dataUop; + if (young.addr <= a.addr && young.addr + young.size >= a.addr + a.size) { + u.memProducers = [young.dataUop]; + u.forwarded = true; + u.latency = C.storeForward; + u.level = 'store buffer'; + } else { + u.memProducers = overlapping.map((x) => x.dataUop); + u.forwardFailed = true; + u.latency += C.storeForward + 6; + u.note = `${u.note}; the store buffer holds only part of these bytes, and the load waits for the stores and pays a forwarding stall`; + } + } + } + if ((u.kind === 'load' || u.kind === 'locked') && lastLocked !== null && lastLocked !== u.id) u.memProducers.push(lastLocked); + u.memProducers = [...new Set(u.memProducers)]; + for (const d of u.dsts) if (!/^t\d$/.test(d)) lastWriter.set(d, u.id); + if (u.kind === 'store-data') { + const sa = uops[u.id - 1]; + if (sa?.access) stores.push({ addr: sa.access.addr, size: sa.access.size, dataUop: u.id }); + u.storeLatency = sa?.storeLatency ?? l1.data; + } + if (u.kind === 'locked') { + if (u.storeAccess) stores.push({ addr: u.storeAccess.addr, size: u.storeAccess.size, dataUop: u.id }); + lastLocked = u.id; + } + } + + /* ---------- 3. Cycle by cycle ---------- */ + const N = uops.length; + const portFree = {}; + const portUse = {}; + for (const p of ['p0', 'p1', 'p5', 'p6', 'p2', 'p3', 'p4', 'p7']) { portFree[p] = 0; portUse[p] = 0; } + let rs = []; + let waiting = []; // eliminated uops whose source is still being computed + let dividerFree = 0; // the divider is one unit behind port 0, busy until its division finishes + let sbCount = 0; // store-buffer entries in use + const sbCommits = []; // when each retired store will have written the cache, in order + let lastCommit = 0; + let nextAlloc = 0; + let robHead = 0; + let frontendReady = 0; + let blockedBy = null; + let frontReason = null; // why frontendReady lies in the future + let cycle = 0; + const stallCycles = { mispredict: 0, fetch: 0, robFull: 0, rsFull: 0, sbFull: 0, serialize: 0 }; + const allocStall = []; // per cycle: why allocation stopped, if it did + const fetchPaid = new Set(); + const limit = N * 200 + 10000; + + const done = (p) => uops[p].complete !== null && uops[p].complete <= cycle; + const ready = (u) => u.producers.every(done) && u.memProducers.every(done); + const isStore = (u) => u.kind === 'store-data' || u.kind === 'locked'; + + while (robHead < N && cycle < limit) { + // stores that have finished writing the cache give back their entries + while (sbCommits.length && sbCommits[0] <= cycle) { sbCommits.shift(); sbCount--; } + + // retire, oldest first, in order + for (let n = 0; n < C.width && robHead < nextAlloc; n++) { + const u = uops[robHead]; + if (u.complete === null || u.complete > cycle) break; + u.retire = cycle; + robHead++; + if (isStore(u)) { + // After retiring, a store still has to write the cache, one at a + // time, in order; one that misses holds up every store behind it. + const cost = u.storeLatency > l1.data ? u.storeLatency : 1; + lastCommit = Math.max(cycle, lastCommit) + cost; + u.commit = lastCommit; + sbCommits.push(lastCommit); + } + } + + // renamed aliases become ready the moment their source does + if (waiting.length) waiting = waiting.filter((u) => { if (ready(u)) { u.complete = cycle; return false; } return true; }); + + // issue: oldest ready uops to free ports that can run them + if (rs.length) { + const still = []; + for (const u of rs) { + if (u.alloc >= cycle || !ready(u)) { still.push(u); continue; } + if (u.kind === 'locked' && robHead < u.id) { still.push(u); continue; } // waits for everything older + const free = u.ports.filter((p) => portFree[p] <= cycle); + if (!free.length || (u.unpipelined && dividerFree > cycle)) { u.portWait = (u.portWait || 0) + 1; still.push(u); continue; } + const p = free.reduce((best, q) => (portUse[q] < portUse[best] ? q : best), free[0]); + u.issue = cycle; u.port = p; u.complete = cycle + u.latency; + portFree[p] = cycle + 1; + if (u.unpipelined) dividerFree = cycle + u.latency; + portUse[p]++; + if (u.mispredicted && blockedBy === u.id) { frontendReady = u.complete + C.mispredictPenalty; frontReason = 'mispredict'; blockedBy = null; } + } + rs = still; + } + + // allocate: rename up to `width` uops into the ROB and scheduler + let why = null; + if (blockedBy !== null) why = 'mispredict'; + else if (cycle < frontendReady) why = 'frontend'; + else { + for (let n = 0; n < C.width && nextAlloc < N; n++) { + const u = uops[nextAlloc]; + const inst = insts[u.inst]; + if (nextAlloc - robHead >= C.rob) { why = 'robFull'; break; } + if (u.ports.length && rs.length >= C.rs) { why = 'rsFull'; break; } + if (inst.fetch && !fetchPaid.has(u.inst) && inst.uops[0] === u.id) { + fetchPaid.add(u.inst); + const extra = Math.max(0, inst.fetch.latency - l1.inst); + if (extra > 0) { frontendReady = cycle + extra; frontReason = 'fetch'; why = 'fetch'; break; } + } + if (u.kind === 'system' && robHead < nextAlloc) { why = 'serialize'; break; } + if (isStore(u) && sbCount >= C.storeBuffer) { why = 'sbFull'; break; } + if (isStore(u)) sbCount++; + u.alloc = cycle; + nextAlloc++; + if (u.kind === 'system') { + u.issue = cycle; u.complete = cycle + u.latency; frontendReady = u.complete; frontReason = 'serialize'; why = 'serialize'; break; + } + if (!u.ports.length) { + // Eliminated at rename: no port, no latency, but an alias cannot be + // ready before the value it points at. + if (ready(u)) u.complete = cycle; else waiting.push(u); + } else rs.push(u); + if (u.kind === 'branch' && u.mispredicted) { blockedBy = u.id; break; } + if (u.kind === 'branch' && u.taken) break; // a taken branch ends this cycle's fetch group + } + } + if (why) { + const key = why === 'frontend' ? frontReason : why; + allocStall[cycle] = key; + if (key in stallCycles) stallCycles[key]++; + } + cycle++; + } + + /* ---------- 4. Summaries ---------- */ + for (const inst of insts) { + const us = inst.uops.map((id) => uops[id]); + if (!us.length) continue; + inst.alloc = Math.min(...us.map((u) => u.alloc)); + inst.issue = Math.min(...us.map((u) => u.issue ?? u.alloc)); + inst.complete = Math.max(...us.map((u) => u.complete)); + inst.retire = Math.max(...us.map((u) => u.retire)); + } + + // The three limits: dependencies, ports, front end. + const earliest = new Array(N).fill(0); + let chain = 0; let chainEnd = -1; + for (const u of uops) { + let start = 0; + for (const p of u.producers) start = Math.max(start, earliest[p]); + for (const p of u.memProducers) start = Math.max(start, earliest[p]); + earliest[u.id] = start + u.latency; + if (earliest[u.id] > chain) { chain = earliest[u.id]; chainEnd = u.id; } + } + const criticalPath = []; + for (let id = chainEnd; id >= 0;) { + criticalPath.unshift(id); + const u = uops[id]; + const preds = [...u.producers, ...u.memProducers]; + if (!preds.length) break; + id = preds.reduce((b, p) => (earliest[p] > earliest[b] ? p : b), preds[0]); + } + const sets = new Map(); + for (const u of uops) { if (!u.ports.length) continue; const key = u.ports.join(','); sets.set(key, u.ports); } + let portBound = 0; let portBoundSet = null; + for (const [key, set] of sets) { + let work = 0; + for (const u of uops) if (u.ports.length && u.ports.every((p) => set.includes(p))) work += 1; + const b = work / set.length; + if (b > portBound) { portBound = b; portBoundSet = key; } + } + const frontendBound = N / C.width; + // The divider is one unit behind port 0 and takes a whole division at a time. + const dividerBound = uops.filter((u) => u.unpipelined).reduce((a, u) => a + u.latency, 0); + const cycles = cycle; + const bound = [ + { name: 'dependency chain', cycles: chain, text: 'the longest run of micro-ops each needing the one before' }, + { name: 'execution ports', cycles: Math.ceil(portBound), text: `micro-ops that can run only on ${portBoundSet ? portBoundSet.split(',').join(', ') : 'some port'}` }, + { name: 'front end', cycles: Math.ceil(frontendBound), text: `${N} micro-ops through a ${C.width}-wide rename stage` }, + ...(dividerBound ? [{ name: 'divider', cycles: dividerBound, text: 'divisions one after another on the single divider' }] : []), + ].sort((a, b) => b.cycles - a.cycles); + + return { + config: C, + uops, + insts, + cycles, + instructions: insts.length, + uopCount: N, + ipc: cycles ? insts.length / cycles : 0, + upc: cycles ? N / cycles : 0, + portUse, + stallCycles, + allocStall, + bounds: bound, + criticalPath, + truncated: cycle >= limit, + }; +} + +/** What the core held at one cycle: ROB, scheduler, ports, and the rename table. */ +export function coreAt(ooo, cycle) { + const rob = []; const rs = []; const executing = []; const rat = {}; + for (const u of ooo.uops) { + if (u.alloc === null || u.alloc > cycle) continue; + if (u.retire === null || u.retire > cycle) rob.push(u); + if (u.ports.length && (u.issue === null || u.issue > cycle)) rs.push(u); + if (u.issue !== null && u.issue <= cycle && u.complete > cycle) executing.push(u); + for (const d of u.dsts) if (!/^t\d$/.test(d)) rat[d] = u; + } + const issuedNow = ooo.uops.filter((u) => u.issue === cycle); + const retiredNow = ooo.uops.filter((u) => u.retire === cycle); + const allocNow = ooo.uops.filter((u) => u.alloc === cycle); + return { cycle, rob, rs, executing, rat, issuedNow, retiredNow, allocNow, stall: ooo.allocStall[cycle] || null }; +} diff --git a/src/engines/asm86/paging.js b/src/engines/asm86/paging.js new file mode 100644 index 0000000..81e1181 --- /dev/null +++ b/src/engines/asm86/paging.js @@ -0,0 +1,130 @@ +/** + * asm86/paging.js + * + * Virtual memory: how an address a program uses becomes an address in the + * physical memory chips. + * + * Every address in every listing is virtual. The processor translates each + * one before touching memory, using tables the operating system keeps: in + * x86-64, four levels of them. The 48 meaningful bits of an address split + * into five fields: + * + * | PML4 index | PDPT index | PD index | PT index | offset in page | + * | 9 bits | 9 bits | 9 bits | 9 bits | 12 bits | + * + * Each index picks one of 512 eight-byte entries in one table page, and the + * entry says where the next table is. The last one names the physical page, + * and the offset carries across unchanged. + * + * Four memory reads for every memory read would be ruinous, so the result is + * cached in the translation lookaside buffer, the TLB. A TLB hit costs + * nothing extra. A miss costs a page walk: four dependent loads. + * + * A page no table maps is a page fault. The interpreter already refuses to + * touch memory outside every region; this is the mechanism that makes an + * operating system do the same. + */ + +export const LEVEL_NAMES = ['PML4', 'PDPT', 'PD', 'PT']; + +/** + * config = { + * pageSize: 4096, + * tlb: { entries: 4 }, fully associative, least recently used + * firstFrame: 0x2a0 where the made-up physical frames start + * } + */ +export class AddressSpace { + constructor(config = {}) { + this.pageSize = config.pageSize ?? 4096; + this.offsetBits = Math.log2(this.pageSize); + if (this.offsetBits !== 12) throw new Error('only 4 KiB pages are modeled'); + this.tlbEntries = config.tlb?.entries ?? 4; + this.nextFrame = config.firstFrame ?? 0x2a0; + this.frames = []; // every allocated frame, with what it holds + this.root = this.allocFrame('page table (PML4)'); + this.tables = new Map(); // frame -> Map(index -> frame) + this.tables.set(this.root, new Map()); + this.tlb = { i: [], d: [] }; + this.now = 0; + this.stats = { translations: 0, tlbHits: 0, tlbMisses: 0, walks: 0, pagesMapped: 0 }; + } + + allocFrame(what) { + const f = this.nextFrame++; + this.frames.push({ frame: f, what }); + return f; + } + + /** The five fields of a virtual address. */ + split(vaddr) { + const v = BigInt(vaddr); + const idx = (shift) => Number((v >> BigInt(shift)) & 0x1ffn); + return { + pml4: idx(39), pdpt: idx(30), pd: idx(21), pt: idx(12), + offset: Number(v & 0xfffn), + vpn: Number(v >> 12n), + }; + } + + translate(vaddr, side = 'd') { + this.now++; + this.stats.translations++; + const f = this.split(vaddr); + const tlb = this.tlb[side]; + const hitAt = tlb.findIndex((e) => e.vpn === f.vpn); + let frame; let walk = null; let evicted = null; + if (hitAt >= 0) { + this.stats.tlbHits++; + tlb[hitAt].lastUse = this.now; + frame = tlb[hitAt].frame; + } else { + this.stats.tlbMisses++; + this.stats.walks++; + walk = []; + let table = this.root; + const indices = [f.pml4, f.pdpt, f.pd, f.pt]; + for (let level = 0; level < 4; level++) { + const entries = this.tables.get(table); + const index = indices[level]; + let next = entries.get(index); + let created = false; + if (next === undefined) { + next = this.allocFrame(level < 3 ? `page table (${LEVEL_NAMES[level + 1]})` : `data page for virtual page 0x${f.vpn.toString(16)}`); + entries.set(index, next); + if (level < 3) this.tables.set(next, new Map()); + else this.stats.pagesMapped++; + created = true; + } + walk.push({ level: LEVEL_NAMES[level], table, index, entryAddr: table * this.pageSize + index * 8, next, created }); + table = next; + } + frame = table; + if (tlb.length >= this.tlbEntries) { + const lru = tlb.reduce((best, e, k) => (e.lastUse < tlb[best].lastUse ? k : best), 0); + evicted = tlb[lru]; + tlb.splice(lru, 1); + } + tlb.push({ vpn: f.vpn, frame, lastUse: this.now }); + } + return { + vaddr, ...f, side, + tlbHit: hitAt >= 0, + walk, evicted, + frame, + paddr: frame * this.pageSize + f.offset, + }; + } + + tlbSnapshot(side = 'd') { return this.tlb[side].map((e) => ({ ...e })); } + + clone() { + const a = Object.create(AddressSpace.prototype); + Object.assign(a, this); + a.tables = new Map([...this.tables].map(([k, v]) => [k, new Map(v)])); + a.tlb = { i: this.tlb.i.map((e) => ({ ...e })), d: this.tlb.d.map((e) => ({ ...e })) }; + a.frames = this.frames.slice(); + a.stats = { ...this.stats }; + return a; + } +} diff --git a/src/engines/asm86/pipeline.js b/src/engines/asm86/pipeline.js new file mode 100644 index 0000000..c4af4b0 --- /dev/null +++ b/src/engines/asm86/pipeline.js @@ -0,0 +1,169 @@ +/** + * asm86/pipeline.js + * + * The classic five-stage pipeline, the machine every architecture course + * draws first: + * + * IF fetch the instruction's bytes + * ID decode it, and read its source registers + * EX do the arithmetic, or work out the address + * MEM read or write memory, if the instruction touches it + * WB write the result back to its register + * + * Five instructions are in flight at once, one per stage, so a new one can + * finish every cycle even though each takes five. The pipeline is only as + * good as its ability to keep that up, and three things stop it: + * + * data hazards an instruction needs a value the one ahead of it has + * not produced yet. Forwarding passes a result straight + * from EX to the next EX, which fixes most of these; a + * load's value is not ready until after MEM, so an + * instruction that uses it immediately still waits a + * cycle (a load-use stall). + * control hazards a branch is not settled until EX. Instructions fetched + * behind it on a wrong guess are thrown away. + * memory a cache miss holds MEM, or IF, for as long as it takes. + * + * This model is trace-driven: it replays the path the program really took, + * and adds up when each instruction reached each stage. + */ + +import { OPS } from './interp.js'; +import { effects, decompose, depName } from './uops.js'; + +export const STAGES = ['IF', 'ID', 'EX', 'MEM', 'WB']; + +/** How many cycles an instruction spends in EX: the slowest of its execution uops. */ +function exLatency(instr) { + let lat = 1; + for (const u of decompose(instr, { eliminate: false })) { + if (['load', 'store-addr', 'store-data', 'nop', 'eliminated', 'system'].includes(u.kind)) continue; + if (u.kind === 'locked') { lat = Math.max(lat, 2); continue; } + lat = Math.max(lat, u.latency); + } + return lat; +} + +/** + * config = { + * forwarding: true, pass results from EX and MEM straight to EX + * resolveIn: 'EX' | 'ID', where a conditional branch is settled + * memoryStalls: true, a cache miss holds MEM (and IF) until it is served + * kernelCycles: 40 what a system call costs the pipeline + * } + * mem is the result of the memory model, for per-step latencies. + */ +export function simulatePipeline(trace, config, mem, l1) { + const cfg = { forwarding: true, resolveIn: 'EX', memoryStalls: true, kernelCycles: 40, ...config }; + const rows = []; + const lastWriter = new Map(); + let nextFetchMin = 0; + let prev = null; + const totals = { dataStalls: 0, loadUse: 0, memoryStalls: 0, fetchStalls: 0, flushed: 0, structural: 0, multiCycle: 0 }; + + trace.steps.forEach((step, k) => { + const instr = step.instr; + const fx = effects(instr); + const cost = mem?.stepCost[k]; + const loads = step.accesses.filter((a) => a.kind === 'read'); + const touchesMemory = step.accesses.length > 0; + const lat = step.syscall ? cfg.kernelCycles : exLatency(instr); + // Which results only exist after MEM: whatever a memory-reading + // instruction writes, except the stack pointer, which EX works out. + const readsMemory = loads.length > 0; + const loaded = new Set(readsMemory ? [...fx.writes].filter((w) => w !== 'rsp') : []); + const missExtra = cfg.memoryStalls && cost ? Math.max(0, ...cost.loads.map((l) => l.latency - l1.data), ...cost.stores.map((s) => s.latency - l1.data), 0) : 0; + const fetchExtra = cfg.memoryStalls && cost?.fetch ? Math.max(0, cost.fetch.latency - l1.inst) : 0; + const D = { IF: 1 + fetchExtra, EX: lat, MEM: 1 + (touchesMemory ? missExtra : 0) }; + + const E = {}; + const stalls = []; + E.IF = Math.max(nextFetchMin, prev ? prev.E.ID : 0); + E.ID = Math.max(E.IF + D.IF, prev ? prev.E.EX : 0); + if (fetchExtra) { stalls.push({ stage: 'IF', cycles: fetchExtra, reason: 'fetch', text: `instruction fetch missed the L1i cache and waited ${fetchExtra} extra cycle${fetchExtra > 1 ? 's' : ''} for ${cost.fetch.level}` }); totals.fetchStalls += fetchExtra; } + + // When can EX start? After ID, after the instruction ahead leaves EX, + // and once every source value is available. + let ready = E.ID + 1; let hazard = null; + // A branch settled in ID needs its flags a stage earlier than EX does. + const early = cfg.resolveIn === 'ID' && step.branch?.kind === 'conditional' ? 1 : 0; + for (const r of fx.reads) { + const w = lastWriter.get(r); + if (!w) continue; + const { row: p, loaded: fromMem } = w; + let at; + if (cfg.forwarding) at = (fromMem ? p.E.MEM + p.D.MEM : p.E.EX + p.D.EX) + early; + else at = p.E.WB + 1; // written in WB's first half, read in ID's second half + if (at > ready) { ready = at; hazard = { reg: r, from: p.index, load: fromMem }; } + } + const structuralEX = prev ? prev.E.MEM : 0; + E.EX = Math.max(E.ID + 1, structuralEX, ready); + const idWait = E.EX - (E.ID + 1); + if (idWait > 0) { + if (hazard && ready >= structuralEX) { + const load = hazard.load && cfg.forwarding; + stalls.push({ stage: 'ID', cycles: idWait, reason: load ? 'load-use' : 'data', reg: hazard.reg, from: hazard.from, text: `waited ${idWait} cycle${idWait > 1 ? 's' : ''} for ${depName(hazard.reg)} from step ${hazard.from}${load ? ', a load whose value exists only after MEM' : cfg.forwarding ? '' : ', which reaches the register file only in WB'}` }); + totals.dataStalls += idWait; if (load) totals.loadUse += idWait; + } else { + stalls.push({ stage: 'ID', cycles: idWait, reason: 'structural', text: `waited ${idWait} cycle${idWait > 1 ? 's' : ''} for the instruction ahead to leave EX` }); + totals.structural += idWait; + } + } + E.MEM = Math.max(E.EX + D.EX, prev ? prev.E.WB : 0); + if (D.EX > 1) { stalls.push({ stage: 'EX', cycles: D.EX - 1, reason: 'multi-cycle', text: step.syscall ? `the kernel runs for ${D.EX} cycles, and nothing behind it may start` : `${instr.op} occupies EX for ${D.EX} cycles` }); totals.multiCycle += D.EX - 1; } + E.WB = Math.max(E.MEM + D.MEM, prev ? prev.E.WB + 1 : 0); + if (D.MEM > 1) { stalls.push({ stage: 'MEM', cycles: D.MEM - 1, reason: 'memory', text: `the access missed L1d and waited ${D.MEM - 1} extra cycle${D.MEM > 2 ? 's' : ''}` }); totals.memoryStalls += D.MEM - 1; } + // durations as actually spent, stalls included + const spent = { IF: E.ID - E.IF, ID: E.EX - E.ID, EX: E.MEM - E.EX, MEM: E.WB - E.MEM, WB: 1 }; + + const row = { index: k, step, E, D: spent, work: D, stalls, loaded, squashed: 0, wrongPath: [] }; + + // Control: a mispredicted branch sends fetch the wrong way until it is settled. + nextFetchMin = E.IF + 1; + if (step.prediction && !step.prediction.correct) { + const settle = cfg.resolveIn === 'ID' ? E.EX : E.EX + D.EX; + // Fetch moves on once the branch has left IF, but while the branch + // itself waits in ID, the one wrong instruction fetched behind it waits + // in IF too. The wrong path therefore occupies the cycles from the + // branch's last cycle in ID until it settles. + const from = Math.max(E.ID, E.EX - 1); + const resume = Math.max(settle, from); + row.squashed = Math.max(0, resume - from); + row.wrongFrom = from; + totals.flushed += row.squashed; + // name the instructions fetched down the path it guessed + const program = trace.program; + let pc = step.prediction.predicted ? program.labels[step.branch.target] : instr.index + 1; + for (let s = 0; s < row.squashed && pc < program.instrs.length; s++) { row.wrongPath.push(program.instrs[pc]); pc++; } + nextFetchMin = resume; + } + if (step.syscall) nextFetchMin = Math.max(nextFetchMin, E.WB); // nothing is fetched until the kernel returns + + for (const w of fx.writes) lastWriter.set(w, { row, loaded: loaded.has(w) }); + rows.push(row); + prev = row; + }); + + const cycles = rows.length ? rows[rows.length - 1].E.WB + 1 : 0; + return { + config: cfg, + rows, + cycles, + instructions: rows.length, + cpi: rows.length ? cycles / rows.length : 0, + ideal: rows.length ? rows.length + 4 : 0, + totals, + }; +} + +/** Which instruction occupies each stage at a given cycle. */ +export function stageOccupancy(pipe, cycle) { + const occ = {}; + for (const r of pipe.rows) { + for (const s of STAGES) { + const start = r.E[s]; const end = start + r.D[s]; + if (cycle >= start && cycle < end) occ[s] = r; + } + } + return occ; +} diff --git a/src/engines/asm86/sandbox.js b/src/engines/asm86/sandbox.js new file mode 100644 index 0000000..883b677 --- /dev/null +++ b/src/engines/asm86/sandbox.js @@ -0,0 +1,420 @@ +/** + * asm86/sandbox.js + * + * A workbench: write assembly, set up registers and memory, choose a model + * machine and the views to draw, and run. Everything the lesson widgets can + * show is available here, on a student's own code. + * + * sandbox({ code, setup, machine, views, examples, useHash }) -> node + * + * The setup box takes one instruction per line, in plain words: + * + * rdi = 0x1000 any register, any width + * dl = 'A' + * memory 0x1000 "GATTACA" text + * memory 0x2000 bytes 1 2 3 255 bytes + * memory 0x3000 qwords 3 11 14 eight-byte little-endian values + * memory 0x4000 zero 64 a run of zeros + * stdin "what the read call returns" + * flags CF flags that start set + * + * "Copy link" puts the whole sandbox state in the page address, so a + * student can send exactly what they are looking at. "Spec" prints the + * dictionary a lesson would pass to asm86.widget, so an instructor can turn + * an experiment into a page widget by pasting it. + */ + +import { el, add, clear, panel } from './dom.js'; +import { encodeState, decodeState } from '../shared/share.js'; +import { mountWidget, viewNames } from './views.js'; +import { PRESETS } from './machine.js'; +import { PREDICTOR_KINDS, assemble } from './interp.js'; +import { HW_SPECS } from './specs-hw.js'; +import { ALL_SPECS } from './specs.js'; + +/* ------------------------------------------------------------------ */ +/* View sets */ +/* ------------------------------------------------------------------ */ + +export const VIEW_SETS = { + 'first steps': ['code', 'registers', 'flags', 'explain'], + 'bits and flags': ['code', 'registers', ['bits', { span: 2 }], 'flags', 'explain'], + 'memory and addresses': ['code', 'registers', 'memory', 'address', 'explain'], + 'machine code': [['code', { addresses: true, bytes: true }], 'encoding', 'explain'], + procedures: ['code', 'stack', 'registers', 'explain'], + 'system calls': ['code', 'console', 'registers', 'explain'], + caches: ['code', ['counters', { only: ['reads', 'l1d-misses', 'l2-misses', 'memory', 'cycles'] }], 'cache', 'hierarchy'], + 'virtual memory': ['code', ['counters', { only: ['reads', 'tlb-misses', 'cycles'] }], 'paging', 'hierarchy'], + pipeline: ['code', 'pipeline', ['counters', { only: ['instructions', 'mispredicts', 'cycles-in-order'] }]], + 'out of order': ['code', ['counters', { only: ['instructions', 'uops', 'cycles'] }], 'uops', 'timeline', 'bottleneck'], + 'the core, cycle by cycle': ['code', 'core', 'timeline'], + 'branch prediction': ['code', ['counters', { only: ['instructions', 'branches', 'mispredicts', 'cycles'] }], 'branches', 'predictors'], + 'whole machine': ['code', 'registers', 'machine', 'explain', ['counters', { only: ['instructions', 'uops', 'cycles', 'l1d-misses', 'mispredicts'] }], 'perf'], +}; + +/* ------------------------------------------------------------------ */ +/* The setup language */ +/* ------------------------------------------------------------------ */ + +function num(tok) { + const t = tok.trim(); + if (/^'.'$/.test(t)) return BigInt(t.charCodeAt(1)); + if (/^-?(0x[0-9a-f_]+|0b[01_]+|\d[\d_]*)$/i.test(t)) { + const neg = t.startsWith('-'); + const v = BigInt(t.replace(/^-/, '').replace(/_/g, '')); + return neg ? -v : v; + } + throw new Error(`"${t}" is not a number`); +} + +function unquote(s) { + const m = /^"((?:[^"\\]|\\.)*)"$/.exec(s.trim()); + if (!m) throw new Error(`expected text in double quotes, found ${s.trim()}`); + return m[1].replace(/\\n/g, '\n').replace(/\\t/g, '\t').replace(/\\"/g, '"').replace(/\\\\/g, '\\'); +} + +/** Turn the setup box into registers, memory, flags, and stdin. */ +export function parseSetup(text) { + const out = { registers: {}, memory: [], flags: {}, stdin: '' }; + String(text || '').split('\n').forEach((raw, i) => { + const line = raw.replace(/(^|\s)(#|;|\/\/).*$/, '').trim(); + if (!line) return; + try { + let m; + if ((m = /^([a-z][a-z0-9]*)\s*=\s*(.+)$/i.exec(line))) { + out.registers[m[1].toLowerCase()] = num(m[2]); + } else if ((m = /^memory\s+(\S+)\s+(.*)$/i.exec(line))) { + const addr = Number(num(m[1])); + const rest = m[2].trim(); + const name = `m${out.memory.length}_${addr.toString(16)}`; + let r; + if (rest.startsWith('"')) out.memory.push({ name, label: `0x${addr.toString(16)}`, addr, text: unquote(rest) }); + else if ((r = /^bytes\s+(.*)$/i.exec(rest))) out.memory.push({ name, label: `0x${addr.toString(16)}`, addr, bytes: r[1].trim().split(/[\s,]+/).map((b) => Number(BigInt.asUintN(8, num(b)))) }); + else if ((r = /^qwords\s+(.*)$/i.exec(rest))) { + const bytes = []; + for (const q of r[1].trim().split(/[\s,]+/)) { let v = BigInt.asUintN(64, num(q)); for (let k = 0; k < 8; k++) { bytes.push(Number(v & 0xffn)); v >>= 8n; } } + out.memory.push({ name, label: `0x${addr.toString(16)}`, addr, bytes }); + } else if ((r = /^zero\s+(\S+)$/i.exec(rest))) out.memory.push({ name, label: `0x${addr.toString(16)}`, addr, size: Number(num(r[1])) }); + else throw new Error('after the address, write "text", bytes ..., qwords ..., or zero n'); + } else if ((m = /^stdin\s+(.*)$/i.exec(line))) { + out.stdin = unquote(m[1]); + } else if ((m = /^flags\s+(.*)$/i.exec(line))) { + for (const f of m[1].split(/[\s,]+/)) { + if (!['ZF', 'SF', 'CF', 'OF', 'PF'].includes(f.toUpperCase())) throw new Error(`no flag named ${f}`); + out.flags[f.toUpperCase()] = true; + } + } else throw new Error('not a setup line; see the examples in the box\u2019s placeholder'); + } catch (e) { + throw new Error(`setup line ${i + 1}: ${e.message}`); + } + }); + return out; +} + +/** The reverse: a spec's registers and memory as setup text. */ +export function setupFromSpec(spec) { + const lines = []; + for (const [r, v] of Object.entries(spec.registers || {})) { + const b = BigInt(v); + lines.push(`${r} = ${b >= 0n && b < 256n && b >= 32n && b < 127n && /^(dl|al|bl|cl)$/.test(r) ? `'${String.fromCharCode(Number(b))}'` : b > 255n ? `0x${b.toString(16)}` : b.toString()}`); + } + for (const m of spec.memory || []) { + if (m.text != null) lines.push(`memory 0x${Number(m.addr).toString(16)} ${JSON.stringify(String(m.text))}`); + else if (m.bytes) { + const bytes = Array.from(m.bytes); + if (bytes.length > 64 && bytes.every((b) => b === 0)) lines.push(`memory 0x${Number(m.addr).toString(16)} zero ${bytes.length}`); + else if (bytes.length % 8 === 0 && bytes.length >= 16 && bytes.length <= 1024) { + const q = []; + for (let k = 0; k < bytes.length; k += 8) { let v = 0n; for (let j = 7; j >= 0; j--) v = (v << 8n) | BigInt(bytes[k + j]); q.push(v.toString()); } + lines.push(`memory 0x${Number(m.addr).toString(16)} qwords ${q.join(' ')}`); + } else lines.push(`memory 0x${Number(m.addr).toString(16)} bytes ${bytes.join(' ')}`); + } else lines.push(`memory 0x${Number(m.addr).toString(16)} zero ${m.size || 0}`); + } + if (spec.stdin) lines.push(`stdin ${JSON.stringify(spec.stdin)}`); + for (const [f, on] of Object.entries(spec.flags || {})) if (on) lines.push(`flags ${f}`); + return lines.join('\n'); +} + +/* ------------------------------------------------------------------ */ +/* Sharing */ +/* ------------------------------------------------------------------ */ + +/** A spec as JavaScript source a lesson can paste. */ +export function specSource(spec) { + const toJs = (v, indent = ' ') => { + if (typeof v === 'bigint') return v > 255n ? `0x${v.toString(16)}` : v.toString(); + if (typeof v === 'string') return v.includes('\n') ? `\`${v.replace(/`/g, '\\`').replace(/\$\{/g, '\\${')}\`` : JSON.stringify(v); + if (Array.isArray(v)) { + if (v.every((x) => typeof x === 'number') && v.length > 16) return `[${v.join(', ')}]`; + return `[${v.map((x) => toJs(x, indent + ' ')).join(', ')}]`; + } + if (v && typeof v === 'object') { + const entries = Object.entries(v).filter(([, x]) => x !== undefined); + return `{ ${entries.map(([k, x]) => `${/^[a-z_$][\w$]*$/i.test(k) ? k : JSON.stringify(k)}: ${k === 'addr' && typeof x === 'number' ? `0x${x.toString(16)}` : toJs(x, indent + ' ')}`).join(', ')} }`; + } + return String(v); + }; + const lines = Object.entries(spec).filter(([, v]) => v !== undefined && !(typeof v === 'object' && v && !Array.isArray(v) && !Object.keys(v).length) && !(Array.isArray(v) && !v.length && v !== spec.views)) + .map(([k, v]) => ` ${k}: ${toJs(v)},`); + return `{\n${lines.join('\n')}\n}`; +} + +/* ------------------------------------------------------------------ */ +/* The sandbox */ +/* ------------------------------------------------------------------ */ + +const DEFAULT_CODE = `; rdi = sequence rsi = n dl = the letter rax = how many + xor eax, eax + xor ecx, ecx +.scan: + cmp rcx, rsi + jae .done + movzx r8d, byte ptr [rdi + rcx*1] + cmp r8b, dl + jne .next + add rax, 1 +.next: + add rcx, 1 + jmp .scan +.done: + ret +`; + +const DEFAULT_SETUP = `rdi = 0x1000 +rsi = 17 +dl = 'A' +memory 0x1000 "GATTACAGGCATTAGCA"`; + +const SETUP_HELP = `rdi = 0x1000 +dl = 'A' +memory 0x1000 "GATTACA" +memory 0x2000 bytes 1 2 3 +memory 0x3000 qwords 3 11 14 +memory 0x4000 zero 64 +stdin "typed input" +flags CF`; + +/** + * options = { + * code, setup, starting program and setup text + * machine: 'teaching', preset name + * viewSet: 'first steps', a key of VIEW_SETS + * examples: { name: spec }, the Examples menu; defaults to every course spec + * useHash: true, read and write the page address for sharing + * remember: true, keep the last program in this browser + * } + */ +export function sandbox(options = {}) { + const opts = { machine: 'teaching', viewSet: 'first steps', useHash: true, remember: true, ...options }; + const examples = opts.examples || { ...HW_SPECS, ...ALL_SPECS }; + const root = el('div', { class: 'a86 a86-sandbox' }); + + let state = { + code: opts.code ?? DEFAULT_CODE, + setup: opts.setup ?? DEFAULT_SETUP, + preset: typeof opts.machine === 'string' ? opts.machine : opts.machine?.preset || 'teaching', + warm: false, prefetch: 'preset', paging: false, forwarding: true, predictor: 'two-bit', + viewSet: opts.viewSet, + views: null, // a spec's own view list, when loaded from an example + title: '', + }; + const hash = opts.useHash && typeof location !== 'undefined' ? /#sandbox=([\w-]+)/.exec(location.hash) : null; + if (hash) { try { state = { ...state, ...decodeState(hash[1]) }; } catch { /* a broken link falls back to the default */ } } + else if (opts.remember) { + try { const saved = localStorage.getItem('asm86-sandbox'); if (saved) state = { ...state, ...JSON.parse(saved) }; } catch { /* storage may be unavailable */ } + } + + /* ---------- controls ---------- */ + const exampleSel = el('select', { class: 'a86-sb-select', 'aria-label': 'load an example' }, [ + el('option', { value: '', text: 'Load an example\u2026' }), + ...Object.entries(examples).map(([k, s]) => el('option', { value: k, text: s.title || k })), + ]); + const presetSel = el('select', { class: 'a86-sb-select', 'aria-label': 'machine' }, Object.keys(PRESETS).map((k) => el('option', { value: k, text: `${k} machine` }))); + const predictorSel = el('select', { class: 'a86-sb-select', 'aria-label': 'branch predictor' }, PREDICTOR_KINDS.map((k) => el('option', { value: k, text: `${k} predictor` }))); + const viewSel = el('select', { class: 'a86-sb-select', 'aria-label': 'views' }, [ + ...Object.keys(VIEW_SETS).map((k) => el('option', { value: k, text: `show: ${k}` })), + el('option', { value: '__example', text: 'show: the example\u2019s own views' }), + ]); + const check = (label, key, title) => { + const box = el('input', { type: 'checkbox' }); + box.checked = Boolean(state[key]); + box.addEventListener('change', () => { state[key] = box.checked; runNow(); }); + return { box, node: el('label', { class: 'a86-sb-check', title }, [box, ` ${label}`]) }; + }; + const warmBox = check('warm caches', 'warm', 'start with the caches as a previous run left them'); + const pagingBox = check('paging', 'paging', 'translate addresses through page tables and a TLB'); + const fwdBox = check('forwarding', 'forwarding', 'the five-stage pipeline passes results straight from EX to EX'); + const prefetchSel = el('select', { class: 'a86-sb-select', 'aria-label': 'prefetcher' }, [ + el('option', { value: 'preset', text: 'preset prefetcher' }), el('option', { value: 'none', text: 'no prefetcher' }), el('option', { value: 'next-line', text: 'next-line prefetcher' }), + ]); + + const editor = el('textarea', { class: 'a86-sb-code', spellcheck: 'false', autocapitalize: 'off', autocomplete: 'off', 'aria-label': 'assembly', rows: '18' }); + const setupBox = el('textarea', { class: 'a86-sb-setup', spellcheck: 'false', placeholder: SETUP_HELP, 'aria-label': 'registers and memory', rows: '6' }); + const runBtn = el('button', { class: 'a86-btn a86-btn-primary', type: 'button', text: 'Assemble and run', title: 'Ctrl+Enter' }); + const linkBtn = el('button', { class: 'a86-btn', type: 'button', text: 'Copy link' }); + const specBtn = el('button', { class: 'a86-btn', type: 'button', text: 'Spec' }); + const resetBtn = el('button', { class: 'a86-btn', type: 'button', text: 'Clear' }); + const status = el('span', { class: 'a86-sb-status', role: 'status' }); + const specOut = el('pre', { class: 'a86-sb-spec', hidden: true }); + const out = el('div', { class: 'a86-sb-out' }); + + for (const ta of [editor, setupBox]) { + ta.addEventListener('keydown', (e) => { + if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) { e.preventDefault(); runNow(); return; } + if (e.key === 'Tab' && !e.shiftKey && ta === editor) { + e.preventDefault(); + const { selectionStart: a, selectionEnd: b, value } = ta; + ta.value = `${value.slice(0, a)} ${value.slice(b)}`; + ta.selectionStart = ta.selectionEnd = a + 8; + } + }); + } + + add(root, el('div', { class: 'a86-sb-grid' }, [ + el('div', { class: 'a86-sb-left' }, [ + el('div', { class: 'a86-sb-row' }, [exampleSel]), + el('label', { class: 'a86-sb-label', text: 'Program' }), + editor, + el('label', { class: 'a86-sb-label', text: 'Registers, memory, and input' }), + setupBox, + ]), + el('div', { class: 'a86-sb-right' }, [ + el('label', { class: 'a86-sb-label', text: 'Machine' }), + el('div', { class: 'a86-sb-row' }, [presetSel, prefetchSel]), + el('div', { class: 'a86-sb-row' }, [warmBox.node, pagingBox.node, fwdBox.node]), + el('div', { class: 'a86-sb-row' }, [predictorSel]), + el('label', { class: 'a86-sb-label', text: 'Views' }), + el('div', { class: 'a86-sb-row' }, [viewSel]), + el('div', { class: 'a86-sb-row a86-sb-actions' }, [runBtn, linkBtn, specBtn, resetBtn]), + status, + el('p', { class: 'a86-sb-help', text: 'Ctrl+Enter runs the program. The arrow keys step once the widget has focus. A spec can name any view the course documents; the menus here are shortcuts.' }), + ]), + ])); + add(root, specOut); + add(root, out); + + /* ---------- state <-> controls ---------- */ + function toControls() { + editor.value = state.code; + setupBox.value = state.setup; + presetSel.value = state.preset; + predictorSel.value = state.predictor; + viewSel.value = state.views && state.viewSet === '__example' ? '__example' : state.viewSet; + prefetchSel.value = state.prefetch; + warmBox.box.checked = state.warm; pagingBox.box.checked = state.paging; fwdBox.box.checked = state.forwarding; + } + function fromControls() { + state.code = editor.value; + state.setup = setupBox.value; + state.preset = presetSel.value; + state.predictor = predictorSel.value; + state.viewSet = viewSel.value; + state.prefetch = prefetchSel.value; + } + + function machine() { + const m = { preset: state.preset }; + if (state.warm) m.caches = { ...(m.caches || {}), warm: true }; + if (state.prefetch !== 'preset') m.caches = { ...(m.caches || {}), prefetch: state.prefetch }; + if (state.paging) m.paging = { enabled: true }; + if (!state.forwarding) m.pipeline = { forwarding: false }; + return Object.keys(m).length === 1 ? state.preset : m; + } + + function buildSpec() { + const setup = parseSetup(state.setup); + const views = state.viewSet === '__example' && state.views ? state.views : VIEW_SETS[state.viewSet] || VIEW_SETS['first steps']; + return { + title: state.title || undefined, + code: state.code, + registers: Object.fromEntries(Object.entries(setup.registers).map(([k, v]) => [k, v])), + memory: setup.memory, + flags: Object.keys(setup.flags).length ? setup.flags : undefined, + stdin: setup.stdin || undefined, + predictor: state.predictor, + machine: machine(), + layout: 'two-column', + views, + }; + } + + let widgetCtl = null; + function runNow() { + fromControls(); + specOut.hidden = true; + let spec; + try { spec = buildSpec(); } catch (e) { status.textContent = e.message; status.className = 'a86-sb-status is-error'; return; } + try { assemble(spec.code); } catch (e) { status.textContent = e.message; status.className = 'a86-sb-status is-error'; } + clear(out); + const host = el('div'); + add(out, host); + const t0 = typeof performance !== 'undefined' ? performance.now() : 0; + widgetCtl = mountWidget(host, spec); + if (widgetCtl.trace) { + const ms = typeof performance !== 'undefined' ? Math.round(performance.now() - t0) : 0; + const s = widgetCtl.trace.stats; + status.className = `a86-sb-status${s.error ? ' is-error' : ''}`; + status.textContent = s.error + ? `Ran ${widgetCtl.trace.count} instructions, then: ${s.error}` + : `Ran ${widgetCtl.trace.count} instructions${s.truncated ? ' (stopped at the step limit)' : ''}; rax = ${s.result}. ${ms ? `(${ms} ms)` : ''}`; + widgetCtl.goto(0); + } + persist(false); + } + + function persist(toHash) { + if (opts.remember) { try { localStorage.setItem('asm86-sandbox', JSON.stringify(state)); } catch { /* fine without */ } } + if (toHash && opts.useHash && typeof history !== 'undefined') { + history.replaceState(null, '', `#sandbox=${encodeState(state)}`); + } + } + + runBtn.addEventListener('click', runNow); + for (const s of [presetSel, predictorSel, viewSel, prefetchSel]) s.addEventListener('change', runNow); + exampleSel.addEventListener('change', () => { + const spec = examples[exampleSel.value]; + exampleSel.value = ''; + if (!spec) return; + const m = spec.machine; + state = { + ...state, + code: spec.code.replace(/^\n/, ''), + setup: setupFromSpec(spec), + title: spec.title || '', + preset: typeof m === 'string' ? m : m?.preset || 'teaching', + warm: Boolean(m?.caches?.warm), + prefetch: m?.caches?.prefetch || 'preset', + paging: Boolean(m?.paging?.enabled), + forwarding: m?.pipeline?.forwarding !== false, + predictor: spec.predictor || 'two-bit', + views: spec.views, + viewSet: '__example', + }; + toControls(); + runNow(); + }); + linkBtn.addEventListener('click', async () => { + fromControls(); + persist(true); + try { await navigator.clipboard.writeText(location.href); status.textContent = 'Link copied: it holds the program, setup, machine, and views.'; } catch { status.textContent = 'The address bar now holds a link to exactly this.'; } + status.className = 'a86-sb-status'; + }); + specBtn.addEventListener('click', () => { + fromControls(); + try { + const spec = buildSpec(); + specOut.textContent = `// paste into an {ojs} cell: show(${'{'}...${'}'})\n${specSource(spec)}`; + specOut.hidden = !specOut.hidden; + } catch (e) { status.textContent = e.message; } + }); + resetBtn.addEventListener('click', () => { + state = { ...state, code: DEFAULT_CODE, setup: DEFAULT_SETUP, title: '', views: null, viewSet: 'first steps' }; + toControls(); runNow(); + }); + + toControls(); + runNow(); + return root; +} + +export { viewNames }; diff --git a/src/engines/asm86/specs-hw.js b/src/engines/asm86/specs-hw.js new file mode 100644 index 0000000..3d76dca --- /dev/null +++ b/src/engines/asm86/specs-hw.js @@ -0,0 +1,670 @@ +/** + * asm86/specs-hw.js + * + * Widgets for the machine underneath the instruction set, in the order a + * course meets them: bits, machine code, the stack, the operating system, + * memory, the pipeline, the out-of-order core, and the whole chip. + * + * Every spec is a plain dictionary, as in specs.js. The `machine` field says + * which model machine times it; see machine.js for the presets. + */ + +import { qwords } from './specs.js'; + +const SEQ17 = 'GATTACAGGCATTAGCA'; + +/* ------------------------------------------------------------------ */ +/* Level 1: bits */ +/* ------------------------------------------------------------------ */ + +export const BITS_ADD = `; one byte at a time, so every bit fits on screen + mov al, 100 + add al, 27 ; 127, the largest signed byte + add al, 1 ; 128: too big for a signed byte, so OF + add al, 128 ; 256 wraps to 0: a carry out, so CF and ZF + mov bl, 5 + sub bl, 7 ; 5 - 7 borrows: CF, and -2 read as signed + cmp bl, 254 ; 254 is -2 unsigned: equal, so ZF + ret +`; + +export const specBitsAdd = { + title: 'The adder, one column at a time', + blurb: 'Each result bit is the sum of two bits and a carry. Watch the carries ripple left, then read each flag off the picture.', + code: BITS_ADD, + layout: 'two-column', + views: [ + 'code', + ['registers', { watch: ['rax', 'rbx'], radix: 'both' }], + ['bits', { span: 2 }], + 'flags', + 'explain', + ], +}; + +export const SHIFTS = ` mov al, 0b10110001 + shl al, 1 ; the top bit falls into CF + shr al, 2 ; zeros come in at the top + mov bl, -64 + sar bl, 3 ; copies of the sign bit come in: -64 / 8 + mov cl, 0b10000001 + rol cl, 1 ; the top bit wraps round to the bottom + ror cl, 2 + ret +`; + +export const specShifts = { + title: 'Shifts and rotates', + blurb: 'Every bit moves at once. What falls off the end lands in CF; what comes in depends on the instruction.', + code: SHIFTS, + layout: 'two-column', + views: ['code', ['registers', { watch: ['rax', 'rbx', 'rcx'], radix: 'both' }], ['bits', { span: 2 }], 'flags'], +}; + +export const SIGNED_UNSIGNED = `; the same eight bits, compared two ways + mov al, 200 ; 200 unsigned, -56 signed + cmp al, 100 + seta bl ; unsigned: is 200 above 100? yes + setg cl ; signed: is -56 greater than 100? no + ret +`; + +export const specSignedUnsigned = { + title: 'Above or greater?', + blurb: 'One subtraction sets all four flags. Unsigned conditions read CF and ZF; signed ones read SF, OF, and ZF. The bits never say which reading you meant.', + code: SIGNED_UNSIGNED, + layout: 'two-column', + views: ['code', ['registers', { watch: ['rax', 'rbx', 'rcx'], radix: 'both' }], ['bits', { span: 2 }], 'flags', 'explain'], +}; + +/* ------------------------------------------------------------------ */ +/* Level 2: machine code */ +/* ------------------------------------------------------------------ */ + +export const ENCODING_TOUR = `; from one byte to ten + nop + xor eax, eax + add rax, 1 + add eax, 0x1000 + mov r8, 5 + mov rax, 0x1122334455667788 + movzx r8d, byte ptr [rdi + rcx*1] + mov rdx, qword ptr [rsp] + lea r11, [rbp + r13*4 - 8] + mov byte ptr [r14 + rcx*1], r9b + movdqu xmm8, xmmword ptr [rdi] + cmp r8b, 'A' + jne .done + popcnt r10, r8 +.done: + ret +`; + +export const specEncoding = { + title: 'Machine code, byte by byte', + blurb: 'The listing shows each instruction’s address and bytes, as objdump prints them. Step through and the panel below takes one instruction apart.', + code: ENCODING_TOUR, + registers: { rdi: 0x1000, r14: 0x2000, rcx: 2 }, + memory: [ + { name: 'T', label: 'T', addr: 0x1000, text: 'GATTACAGGCATTAGC' }, + { name: 'out', label: 'out', addr: 0x2000, size: 8 }, + ], + layout: 'one-column', + views: [['code', { addresses: true, bytes: true }], 'encoding', 'explain'], +}; + +/* ------------------------------------------------------------------ */ +/* Level 3: procedures and the stack */ +/* ------------------------------------------------------------------ */ + +export const STACK_FRAME = `; rdi = n rax = 1*1 + 2*2 + ... + n*n +main: + push rbx ; rbx is the caller's: save it + xor ebx, ebx ; total <- 0 + mov rcx, rdi +.loop: + test rcx, rcx + je .done + mov rdi, rcx + push rcx ; square may overwrite rcx + call square + pop rcx + add rbx, rax + dec rcx + jmp .loop +.done: + mov rax, rbx + pop rbx ; hand the caller its rbx back + ret + +square: + push rbp ; prologue: open a frame + mov rbp, rsp + sub rsp, 16 ; room for a local + mov qword ptr [rbp - 8], rdi + mov rax, qword ptr [rbp - 8] + imul rax, rdi + leave ; epilogue: rsp <- rbp, pop rbp + ret +`; + +export const specStackFrame = { + title: 'Calls, frames, and the stack', + blurb: 'call pushes the address to come back to; the prologue saves rbp and makes room for locals; leave and ret undo both. The stack view labels every slot with what put it there.', + code: STACK_FRAME, + registers: { rdi: 3, rbx: 0x1234 }, + layout: 'two-column', + views: [ + 'code', + ['stack', { slots: 9 }], + ['registers', { watch: ['rsp', 'rbp', 'rax', 'rbx', 'rcx', 'rdi'], radix: 'hex' }], + 'explain', + ], +}; + +export const FACTORIAL = `; rdi = n rax = n! +fact: + cmp rdi, 1 + jbe .base + push rdi ; keep n for after the call + dec rdi + call fact ; rax <- (n - 1)! + pop rdi + imul rax, rdi ; rax <- n * (n - 1)! + ret +.base: + mov eax, 1 + ret +`; + +export const specRecursion = { + title: 'Recursion is just the stack', + blurb: 'Each call pushes one return address and one saved n. At the deepest point the stack holds every unfinished call, and the answers come back in reverse order.', + code: FACTORIAL, + registers: { rdi: 5 }, + layout: 'two-column', + views: ['code', ['stack', { slots: 12 }], ['registers', { watch: ['rdi', 'rax', 'rsp'], radix: 'dec' }], 'explain'], +}; + +export const STACK_BUG = `; the helper pushes and forgets to pop + mov rdi, 7 + call helper + ret +helper: + push rdi ; saved, and never restored + add rdi, 1 + ret ; pops 7 and tries to go there +`; + +export const specStackBug = { + title: 'One push too many', + blurb: 'ret trusts the top of the stack completely. Leave anything on top of the return address and ret jumps to it.', + code: STACK_BUG, + layout: 'two-column', + views: ['code', ['stack', { slots: 5 }], ['registers', { watch: ['rsp', 'rdi'], radix: 'hex' }], 'explain'], +}; + +/* ------------------------------------------------------------------ */ +/* Level 4: the operating system */ +/* ------------------------------------------------------------------ */ + +export const HELLO = `; write(1, message, 14), then exit(0) + mov eax, 1 ; system call 1 is write + mov edi, 1 ; file descriptor 1 is the terminal + mov esi, 0x1000 ; the address of the bytes + mov edx, 14 ; how many bytes + syscall + mov eax, 60 ; system call 60 is exit + xor edi, edi ; exit code 0 + syscall +`; + +export const specHello = { + title: 'Hello, world, without a library', + blurb: 'A program cannot touch the screen. It puts a request in registers and executes syscall, and the kernel does the rest.', + code: HELLO, + memory: [{ name: 'msg', label: 'message', addr: 0x1000, text: 'Hello, world!\n' }], + layout: 'two-column', + views: ['code', 'console', ['registers', { watch: ['rax', 'rdi', 'rsi', 'rdx', 'rcx', 'r11'], radix: 'hex' }], ['memory', {}], 'explain'], +}; + +export const SHOUT = `; read what was typed, capitalize it, print it back + xor eax, eax ; system call 0 is read + xor edi, edi ; file descriptor 0 is the keyboard + mov esi, 0x1000 + mov edx, 32 + syscall ; rax <- how many bytes arrived + mov r12, rax + xor ecx, ecx +.each: + cmp rcx, r12 + jae .print + movzx r8d, byte ptr [rsi + rcx*1] + lea r9d, [r8 - 97] ; how far past 'a' + cmp r9d, 25 + ja .next ; not a to z: one unsigned test covers both ends + sub r8d, 32 ; 'a' - 'A' = 32 + mov byte ptr [rsi + rcx*1], r8b +.next: + add rcx, 1 + jmp .each +.print: + mov eax, 1 + mov edi, 1 + mov rdx, r12 + syscall + mov eax, 60 + xor edi, edi + syscall +`; + +export const specShout = { + title: 'Reading, changing, writing', + blurb: 'A read and a write bracket ordinary work. Notice the range check in between: one unsigned comparison rejects both too small and too large.', + code: SHOUT, + stdin: 'gattaca, ok?\n', + memory: [{ name: 'buf', label: 'buf', addr: 0x1000, size: 16 }], + layout: 'two-column', + views: ['code', 'console', ['memory', {}], ['registers', { watch: ['rax', 'rcx', 'r8', 'r9', 'r12'], radix: 'dec' }], 'explain'], +}; + +export const NULL_POINTER = `; a pointer that points nowhere + mov rdi, 0x1000 + movzx eax, byte ptr [rdi] ; fine: something lives at 0x1000 + xor edi, edi ; a null pointer + movzx eax, byte ptr [rdi] ; nothing lives at address 0 + ret +`; + +export const specNullPointer = { + title: 'A segmentation fault, from underneath', + blurb: 'The load at address 0 finds no page mapping it, and the processor raises a page fault. The kernel finds no page it could supply and ends the program.', + code: NULL_POINTER, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: 'GATTACA' }], + layout: 'two-column', + views: ['code', ['registers', { watch: ['rdi', 'rax'], radix: 'hex' }], ['explain', { span: 2 }]], +}; + +/* ------------------------------------------------------------------ */ +/* Level 5: memory */ +/* ------------------------------------------------------------------ */ + +const LONG = (SEQ17.repeat(6)).slice(0, 96); + +export const COUNT_SCAN = `; rdi = sequence rsi = n dl = the letter rax = how many + xor eax, eax + xor ecx, ecx +.scan: + cmp rcx, rsi + jae .done + movzx r8d, byte ptr [rdi + rcx*1] + cmp r8b, dl + jne .next + add rax, 1 +.next: + add rcx, 1 + jmp .scan +.done: + ret +`; + +export const specCacheScan = { + title: 'A scan, seen by the cache', + blurb: 'The first letter of each 16-byte line misses and brings the whole line in; the next 15 hit. A sequential scan is fast for exactly that reason.', + code: COUNT_SCAN, + registers: { rdi: 0x1000, rsi: 96, rdx: 65 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: LONG }], + machine: 'teaching', + start: 20, + layout: 'two-column', + views: [ + 'code', + ['counters', { only: ['reads', 'l1d-misses', 'l2-misses', 'memory'] }], + ['cache', { level: 'L1d' }], + 'hierarchy', + ], +}; + +export const specPrefetch = { + ...specCacheScan, + title: 'The same scan, with a prefetcher', + blurb: 'The hardware fetches the next line early whenever a line misses, guessing it will be wanted. A sequential scan is the easiest pattern there is to guess.', + machine: { preset: 'teaching', caches: { prefetch: 'next-line' } }, +}; + +export const STRIDE = (gap) => `; rdi = table rsi = rounds rax = sum of three values ${gap} bytes apart + xor eax, eax +.round: + test rsi, rsi + je .done + add rax, qword ptr [rdi] + add rax, qword ptr [rdi + ${gap}] + add rax, qword ptr [rdi + ${gap * 2}] + dec rsi + jmp .round +.done: + ret +`; + +export const specCacheConflict = { + title: 'Three lines, one set', + blurb: 'Every one of these addresses lands in set 0, because they are 128 bytes apart, and a set holds only two lines. Each load evicts the line the next round needs: a conflict miss, with most of the cache empty.', + code: STRIDE(128), + registers: { rdi: 0x1000, rsi: 5 }, + memory: [{ name: 'table', label: 'table', addr: 0x1000, bytes: qwords(Array.from({ length: 36 }, (_, k) => k)) }], + machine: 'teaching', + start: 12, + layout: 'two-column', + views: ['code', ['counters', { only: ['reads', 'l1d-misses', 'l2-misses'] }], ['cache', { level: 'L1d' }], 'hierarchy'], +}; + +export const specCacheNoConflict = { + ...specCacheConflict, + title: 'Three lines, three sets', + blurb: 'The same program with the values 16 bytes apart. Each lands in its own set, the first round misses, and every round after that hits.', + code: STRIDE(16), +}; + +function gridCode(order) { + const [outer, inner] = order === 'rows' ? ['r8', 'r9'] : ['r9', 'r8']; + return `; rdi = a 32 x 32 grid of bytes, stored row after row +; rax = the sum of every byte, visiting ${order === 'rows' ? 'row by row' : 'column by column'} + xor eax, eax + xor ${outer}d, ${outer}d +.outer: + cmp ${outer}, 32 + jae .done + xor ${inner}d, ${inner}d +.inner: + cmp ${inner}, 32 + jae .nextouter + mov r10, r8 ; r8 is the row + shl r10, 5 ; row * 32 + add r10, r9 ; + column + movzx r11d, byte ptr [rdi + r10*1] + add rax, r11 + add ${inner}, 1 + jmp .inner +.nextouter: + add ${outer}, 1 + jmp .outer +.done: + ret +`; +} + +const GRID = Array.from({ length: 1024 }, (_, k) => ((k >> 5) + (k & 31)) % 7); + +export const specRowOrder = { + title: 'Row by row', + blurb: 'Consecutive iterations read consecutive bytes, and each 16-byte line serves 16 loads. The prefetcher sees a steady stream and fetches each line before it is asked for.', + code: gridCode('rows'), + registers: { rdi: 0x2000 }, + memory: [{ name: 'grid', label: 'grid', addr: 0x2000, bytes: GRID }], + // Every real core prefetches a sequential stream; without it, the misses + // of a row-order scan fall too far apart to overlap, which is a lesson + // about memory-level parallelism rather than about locality. + machine: { preset: 'teaching', caches: { prefetch: 'next-line', l2: { size: 512 } } }, + maxSteps: 12000, + layout: 'two-column', + views: ['code', ['counters', { only: ['instructions', 'reads', 'l1d-misses', 'cycles'] }], 'hierarchy', ['cache', { level: 'L1d' }]], +}; + +export const specColumnOrder = { + ...specRowOrder, + title: 'Column by column', + blurb: 'The same sum over the same bytes, visiting them 32 bytes apart. Each line is evicted long before the loop comes back for its next byte, and nearly every load misses. The next-line prefetcher cannot help with a jump of 32 bytes.', + code: gridCode('columns'), +}; + +export const PAGES = `; rdi = a 24 KiB array rax = one byte from each of six pages, twice over + xor eax, eax + mov r8d, 2 ; two passes +.pass: + xor ecx, ecx +.page: + cmp rcx, 6 + jae .nextpass + mov r9, rcx + shl r9, 12 ; page number * 4096 + movzx r10d, byte ptr [rdi + r9*1] + add rax, r10 + add rcx, 1 + jmp .page +.nextpass: + dec r8 + jne .pass + ret +`; + +export const specPaging = { + title: 'Six pages, four TLB entries', + blurb: 'Every load names a virtual address, which the TLB must translate. The TLB holds four translations and this loop needs six. Each load evicts one the loop will want again, and every access pays for a four-level page walk.', + code: PAGES, + registers: { rdi: 0x10000 }, + memory: [{ name: 'array', label: 'array', addr: 0x10000, bytes: Array.from({ length: 6 * 4096 }, (_, k) => (k % 4096 === 0 ? k / 4096 + 1 : 0)) }], + machine: { preset: 'teaching', paging: { enabled: true, tlb: { entries: 4 } } }, + start: 12, + layout: 'two-column', + views: ['code', ['counters', { only: ['reads', 'tlb-misses', 'l1d-misses', 'cycles'] }], ['paging', { span: 2 }], 'hierarchy'], +}; + +/* ------------------------------------------------------------------ */ +/* Level 6: the pipeline */ +/* ------------------------------------------------------------------ */ + +export const HAZARDS = ` add rax, 1 ; independent work flows straight through + add rbx, 1 + add rax, rbx ; needs rbx from the line above: forwarded + movzx ecx, byte ptr [rdi] ; a load + add rcx, 1 ; needs it at once: a load-use stall + imul rdx, rcx ; three cycles in EX + add rdx, 1 ; waits for the multiply + mov esi, 5 ; independent again + ret +`; + +export const specPipelineHazards = { + title: 'Five instructions in flight', + blurb: 'Each column is a cycle, and instructions overlap down the diagonal. A row that repeats a stage in amber is waiting for a value the row above has not finished making.', + code: HAZARDS, + registers: { rdi: 0x1000 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: 'GATTACA' }], + machine: 'ideal', + start: 5, + layout: 'one-column', + views: ['code', 'pipeline', 'explain'], +}; + +export const specPipelineNoForwarding = { + ...specPipelineHazards, + title: 'The same code, without forwarding', + blurb: 'This pipeline has no path from one EX to the next, and a value is usable only after it has gone all the way to WB. Nearly every dependent instruction now waits.', + machine: { preset: 'ideal', pipeline: { forwarding: false } }, +}; + +export const specPipelineBranches = { + title: 'What a wrong guess costs a pipeline', + blurb: 'The branch settles only in EX, and by then two more instructions have been fetched on the guess. A wrong guess throws them away: the red rows.', + code: COUNT_SCAN, + registers: { rdi: 0x1000, rsi: 17, rdx: 65 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: SEQ17 }], + machine: 'ideal', + start: 19, + layout: 'one-column', + views: ['code', 'pipeline', ['counters', { only: ['instructions', 'branches', 'mispredicts', 'cycles-in-order'] }]], +}; + +/* ------------------------------------------------------------------ */ +/* Level 7: out of order */ +/* ------------------------------------------------------------------ */ + +const VALUES = qwords(Array.from({ length: 64 }, (_, k) => 1 + (k % 3))); + +export const ONE_CHAIN = `; rdi = 64 values rax = their product, with one running product + mov eax, 1 + xor ecx, ecx +.each: + imul rax, qword ptr [rdi + rcx*8] + imul rax, qword ptr [rdi + rcx*8 + 8] + imul rax, qword ptr [rdi + rcx*8 + 16] + imul rax, qword ptr [rdi + rcx*8 + 24] + add rcx, 4 + cmp rcx, 64 + jb .each + ret +`; + +export const FOUR_CHAINS = `; rdi = 64 values rax = their product, with four running products + mov eax, 1 + mov r8d, 1 + mov r9d, 1 + mov r10d, 1 + xor ecx, ecx +.each: + imul rax, qword ptr [rdi + rcx*8] + imul r8, qword ptr [rdi + rcx*8 + 8] + imul r9, qword ptr [rdi + rcx*8 + 16] + imul r10, qword ptr [rdi + rcx*8 + 24] + add rcx, 4 + cmp rcx, 64 + jb .each + imul rax, r8 ; combine the four at the end + imul r9, r10 + imul rax, r9 + ret +`; + +const warmRealistic = { preset: 'realistic', caches: { warm: true } }; + +export const specOneChain = { + title: 'One chain of multiplications', + blurb: 'Each imul needs the one before it, and a multiplication takes 3 cycles. This loop can go no faster than 3 cycles per value, however many execution units the core has.', + code: ONE_CHAIN, + registers: { rdi: 0x1000 }, + memory: [{ name: 'values', label: 'values', addr: 0x1000, bytes: VALUES }], + machine: warmRealistic, + start: 30, + layout: 'two-column', + views: ['code', ['counters', { only: ['instructions', 'uops', 'cycles'] }], 'timeline', 'bottleneck'], +}; + +export const specFourChains = { + ...specOneChain, + title: 'Four chains of multiplications', + blurb: 'The same multiplications, split across four running products that never wait for each other. The core overlaps them, and the single multiplier port becomes the limit instead.', + code: FOUR_CHAINS, +}; + +export const UOP_TOUR = ` xor eax, eax ; zero idiom: done at rename + mov rbx, rax ; move elimination: done at rename + add rax, qword ptr [rdi] ; a load, then an add + add qword ptr [rdi], rax ; load, add, store address, store data + mov rcx, qword ptr [rdi] ; answered by the store buffer + cmp rax, rbx + jne .skip ; fused with the cmp into one uop + nop +.skip: + imul rdx, rcx ; three cycles, port 1 only + lea r8, [rax + rbx*2 + 5] ; three-part lea: the slow form + div rcx ; tens of cycles, and blocks the divider + ret +`; + +export const specUops = { + title: 'Instructions become micro-ops', + blurb: 'The table shows each micro-op of the last instruction, the ports that can run it, and what it waited for. Some instructions become several micro-ops, some none at all, and a compare and the jump after it can become one.', + code: UOP_TOUR, + registers: { rdi: 0x1000, rdx: 0 }, + memory: [{ name: 'x', label: 'x', addr: 0x1000, bytes: qwords([7, 0]) }], + machine: warmRealistic, + start: 4, + layout: 'one-column', + views: ['code', 'uops', 'timeline'], +}; + +export const specCore = { + ...specFourChains, + title: 'Inside the core, cycle by cycle', + blurb: 'The reorder buffer holds everything in flight, oldest first, and the ports run whatever in it is ready. The rename table says which in-flight result each register name means right now.', + start: 20, + layout: 'one-column', + views: ['code', 'core', 'timeline'], +}; + +/* ------------------------------------------------------------------ */ +/* Level 8: the whole machine */ +/* ------------------------------------------------------------------ */ + +export const specWholeMachine = { + title: 'Everything at once', + blurb: 'The whole core lights up one instruction at a time, showing where its bytes were fetched from, which ports its micro-ops ran on, how far its load traveled, and whether the branch predictor was right.', + code: COUNT_SCAN, + registers: { rdi: 0x1000, rsi: 96, rdx: 65 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: LONG }], + machine: { preset: 'teaching', paging: { enabled: true } }, + start: 6, + layout: 'two-column', + views: ['code', ['registers', { watch: ['rax', 'rcx', 'r8'], roles: { rax: 'total', rcx: 'i', r8: 'the letter' }, radix: 'dec' }], 'machine', 'explain', ['counters', { only: ['instructions', 'uops', 'cycles', 'l1d-misses', 'tlb-misses', 'mispredicts'] }]], +}; + +export const specPerfBranchy = { + title: 'perf stat: the branchy count', + code: COUNT_SCAN, + registers: { rdi: 0x1000, rsi: 102, rdx: 65 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: SEQ17.repeat(6) }], + machine: warmRealistic, + layout: 'one-column', + views: ['perf', 'bottleneck'], +}; + +export const specPerfBranchless = { + ...specPerfBranchy, + title: 'perf stat: the branchless count', + code: `; rdi = sequence rsi = n dl = the letter rax = how many + xor eax, eax + xor ecx, ecx +.scan: + cmp rcx, rsi + jae .done + movzx r8d, byte ptr [rdi + rcx*1] + cmp r8b, dl + sete r9b + movzx r9d, r9b + add rax, r9 + add rcx, 1 + jmp .scan +.done: + ret +`, +}; + +export const HW_SPECS = { + specBitsAdd, + specShifts, + specSignedUnsigned, + specEncoding, + specStackFrame, + specRecursion, + specStackBug, + specHello, + specShout, + specNullPointer, + specCacheScan, + specPrefetch, + specCacheConflict, + specCacheNoConflict, + specRowOrder, + specColumnOrder, + specPaging, + specPipelineHazards, + specPipelineNoForwarding, + specPipelineBranches, + specOneChain, + specFourChains, + specUops, + specCore, + specWholeMachine, + specPerfBranchy, + specPerfBranchless, +}; diff --git a/src/engines/asm86/specs.js b/src/engines/asm86/specs.js new file mode 100644 index 0000000..44ddc5d --- /dev/null +++ b/src/engines/asm86/specs.js @@ -0,0 +1,844 @@ +/** + * asm86/specs.js + * + * The assembly kernels this course reads, plus a widget configuration for + * each. Every spec is a plain dictionary, so a lesson can copy one, change + * the sequence or the pattern, and get a working widget. + */ + +/* ------------------------------------------------------------------ */ +/* Kernels */ +/* ------------------------------------------------------------------ */ + +/** Lesson 1's count_base, at byte level. Answer lands in rax. */ +export const COUNT_BASE = `; rdi = address of the sequence rsi = n, its length +; edx = the base being counted rax = the answer + xor eax, eax ; total <- 0 + xor ecx, ecx ; i <- 0 +.scan: + cmp rcx, rsi ; i against n + jae .done ; leave once i reaches n + movzx r8d, byte ptr [rdi + rcx*1] ; load T[i] + cmp r8b, dl ; T[i] against b + jne .next + add rax, 1 ; total <- total + 1 +.next: + add rcx, 1 ; i <- i + 1 + jmp .scan +.done: + ret +`; + +/** Lesson 3's naive_search, counting matches rather than listing them. */ +export const NAIVE_SEARCH = `; rdi = text rsi = n rdx = pattern rcx = m rax = matches + xor eax, eax ; matches <- 0 + xor r8, r8 ; i <- 0 + mov r9, rsi + sub r9, rcx + add r9, 1 ; r9 <- n - m + 1 +.outer: + cmp r8, r9 ; i against the last legal start + jae .done + xor r10, r10 ; j <- 0 + lea r11, [rdi + r8*1] ; address of this window +.inner: + cmp r10, rcx ; j against m + jae .hit ; every letter agreed + movzx r12d, byte ptr [r11 + r10*1] ; T[i + j] + movzx r13d, byte ptr [rdx + r10*1] ; P[j] + cmp r12b, r13b + jne .next ; one disagreement settles it + add r10, 1 ; j <- j + 1 + jmp .inner +.hit: + add rax, 1 +.next: + add r8, 1 ; i <- i + 1 + jmp .outer +.done: + ret +`; + +/** The same search with the flag written out, matching the Mojo line for line. */ +export const NAIVE_SEARCH_FLAG = `; rdi = text rsi = n rdx = pattern rcx = m +; r14 stands in for the Mojo variable "matched" + xor eax, eax + xor r8, r8 + mov r9, rsi + sub r9, rcx + add r9, 1 +.outer: + cmp r8, r9 + jae .done + mov r14, 1 ; matched <- True + xor r10, r10 ; j <- 0 +.inner: + cmp r10, rcx + jae .verdict + lea r11, [rdi + r8*1] + movzx r12d, byte ptr [r11 + r10*1] + movzx r13d, byte ptr [rdx + r10*1] + cmp r12b, r13b + je .agree + xor r14, r14 ; matched <- False + jmp .verdict ; break +.agree: + add r10, 1 + jmp .inner +.verdict: + cmp r14, 1 + jne .next + add rax, 1 +.next: + add r8, 1 + jmp .outer +.done: + ret +`; + +/** One instruction at a time, for the reference page's worked examples. */ +export const SINGLE_STEPS = ` mov rax, 7 ; a constant into a register + mov rbx, rax ; one register into another + add rax, 3 ; arithmetic in place + sub rax, 4 + movzx ecx, byte ptr [rdi] ; one byte of memory, zero-filled + lea rdx, [rdi + 4] ; an address, with no memory read + cmp rax, 6 ; set the flags, keep no answer + sete r8b ; turn a flag into a number + ret +`; + +/** Doc 6: the same count with the branch removed by `sete`. */ +export const COUNT_BRANCHLESS = `; rdi = sequence rsi = n edx = the base rax = the answer + xor eax, eax + xor ecx, ecx +.scan: + cmp rcx, rsi + jae .done ; the only branch left + movzx r8d, byte ptr [rdi + rcx*1] + cmp r8b, dl ; sets ZF, jumps nowhere + sete r9b ; 1 when equal, 0 when not + movzx r9d, r9b + add rax, r9 ; add 1 or add 0 + add rcx, 1 + jmp .scan +.done: + ret +`; + +/** Doc 6: the same idea with a conditional move instead. */ +export const COUNT_CMOV = `; rdi = sequence rsi = n edx = the base rax = the answer + xor eax, eax + xor ecx, ecx +.scan: + cmp rcx, rsi + jae .done + movzx r8d, byte ptr [rdi + rcx*1] + lea r10, [rax + 1] ; the total it would become + cmp r8b, dl + cmove rax, r10 ; keep it only when they matched + add rcx, 1 + jmp .scan +.done: + ret +`; + +/** Doc 7: sixteen letters compared per instruction. */ +export const COUNT_VECTOR = `; rdi = sequence rsi = n rdx = sixteen copies of the base + xor eax, eax + xor ecx, ecx + movdqu xmm1, xmmword ptr [rdx] ; sixteen copies of b + movzx r13d, byte ptr [rdx] ; and one copy, for the leftovers +.blocks: + lea r9, [rcx + 16] + cmp r9, rsi + ja .tail ; fewer than sixteen letters left + movdqu xmm0, xmmword ptr [rdi + rcx*1] + pcmpeqb xmm0, xmm1 ; sixteen comparisons, one instruction + pmovmskb r8d, xmm0 ; one bit per lane + popcnt r8d, r8d ; how many bits are set + add rax, r8 + add rcx, 16 + jmp .blocks +.tail: + cmp rcx, rsi + jae .done + movzx r8d, byte ptr [rdi + rcx*1] ; the leftovers, one at a time + cmp r8b, r13b + jne .next + add rax, 1 +.next: + add rcx, 1 + jmp .tail +.done: + ret +`; + +/** Doc 7: what a real aligner does first, sixteen candidate starts at a time. */ +export const VECTOR_FILTER = `; rdi = text rsi = n rdx = sixteen copies of P[0] +; rax = how many starts survive the filter + xor eax, eax + xor ecx, ecx + movdqu xmm1, xmmword ptr [rdx] +.blocks: + lea r9, [rcx + 16] + cmp r9, rsi + ja .done + movdqu xmm0, xmmword ptr [rdi + rcx*1] + pcmpeqb xmm0, xmm1 + pmovmskb r8d, xmm0 ; a bit for every candidate start +.bits: + test r8d, r8d + je .nextblock ; no candidates left in this block + tzcnt r10d, r8d ; position of the lowest set bit + lea r11, [rcx + r10*1] ; the candidate start itself + add rax, 1 + mov r12d, r8d + sub r12d, 1 + and r8d, r12d ; clear that bit and look again + jmp .bits +.nextblock: + add rcx, 16 + jmp .blocks +.done: + ret +`; + +/* ------------------------------------------------------------------ */ +/* Shapes */ +/* ------------------------------------------------------------------ */ + +/** + * The code shapes cataloged in the x86-64 reference, as runnable kernels. + * Any lesson can drop one into a widget, or swap its body for its own. + * Every one of these is checked in test-patterns.mjs. + */ +export const PATTERNS = { + /** A counted loop with an accumulator: how many G or C. */ + accumulate: `; rdi = sequence rsi = n rax = how many G or C + xor eax, eax + xor ecx, ecx +.each: + cmp rcx, rsi + jae .done + movzx r8d, byte ptr [rdi + rcx*1] + cmp r8b, 'G' + je .keep + cmp r8b, 'C' + jne .next +.keep: + add rax, 1 +.next: + add rcx, 1 + jmp .each +.done: + ret +`, + + /** An early exit with an invented answer: the first position holding dl. */ + earlyExit: `; rdi = sequence rsi = n dl = the letter rax = position or -1 + mov rax, -1 + xor ecx, ecx +.each: + cmp rcx, rsi + jae .done + movzx r8d, byte ptr [rdi + rcx*1] + cmp r8b, dl + jne .next + mov rax, rcx + jmp .done +.next: + add rcx, 1 + jmp .each +.done: + ret +`, + + /** Data as an index: complement every letter through a 256-byte table. */ + tableLookup: `; rdi = sequence rsi = n rdx = 256-byte table r14 = where to write + xor ecx, ecx +.each: + cmp rcx, rsi + jae .done + movzx r8d, byte ptr [rdi + rcx*1] ; the letter + movzx r9d, byte ptr [rdx + r8*1] ; the table entry for it + mov byte ptr [r14 + rcx*1], r9b + add rcx, 1 + jmp .each +.done: + ret +`, + + /** Halving a range: binary search over eight-byte values. */ + binarySearch: `; rdi = sorted values rsi = how many rdx = the value wanted +; rax = its index, or -1 + mov rax, -1 + xor r8, r8 ; lo + mov r9, rsi ; hi, one past the end +.narrow: + cmp r8, r9 + jae .done + lea r10, [r8 + r9] + shr r10, 1 ; mid, halfway between + mov r11, qword ptr [rdi + r10*8] + cmp r11, rdx + je .found + jb .upper + mov r9, r10 ; hi <- mid + jmp .narrow +.upper: + lea r8, [r10 + 1] ; lo <- mid + 1 + jmp .narrow +.found: + mov rax, r10 +.done: + ret +`, + + /** Two pointers: how many values two sorted lists share. */ + twoPointers: `; rdi = list A rsi = |A| rdx = list B rcx = |B| +; rax = how many values appear in both + xor eax, eax + xor r8, r8 + xor r9, r9 +.step: + cmp r8, rsi + jae .done + cmp r9, rcx + jae .done + mov r10, qword ptr [rdi + r8*8] + mov r11, qword ptr [rdx + r9*8] + cmp r10, r11 + je .both + jb .advanceA + add r9, 1 ; the second list is behind + jmp .step +.advanceA: + add r8, 1 ; the first list is behind + jmp .step +.both: + add rax, 1 + add r8, 1 + add r9, 1 + jmp .step +.done: + ret +`, + + /** Packing: k letters into one number, two bits each. */ + packBits: `; rdi = sequence rsi = k rdx = table of two-bit codes +; rax = the k letters packed into one number + xor eax, eax + xor ecx, ecx +.each: + cmp rcx, rsi + jae .done + movzx r8d, byte ptr [rdi + rcx*1] + movzx r9d, byte ptr [rdx + r8*1] ; 0, 1, 2, or 3 + shl rax, 2 ; make room for two more bits + or rax, r9 ; drop the code into them + add rcx, 1 + jmp .each +.done: + ret +`, + + /** Walking two sequences at once: how many positions differ. */ + twoSequences: `; rdi and rsi are two sequences of the same length rdx = n +; rax = how many positions differ + xor eax, eax + xor ecx, ecx +.each: + cmp rcx, rdx + jae .done + movzx r8d, byte ptr [rdi + rcx*1] + movzx r9d, byte ptr [rsi + rcx*1] + cmp r8b, r9b + je .next + add rax, 1 +.next: + add rcx, 1 + jmp .each +.done: + ret +`, + + /** A nested loop: how many ordered pairs with i < j. */ + nestedLoop: `; rsi = n rax = how many pairs with i < j + xor eax, eax + xor r8, r8 +.outer: + cmp r8, rsi + jae .done + lea r9, [r8 + 1] ; reset every time round +.inner: + cmp r9, rsi + jae .nextouter + add rax, 1 + add r9, 1 + jmp .inner +.nextouter: + add r8, 1 + jmp .outer +.done: + ret +`, + + /** Branchless choice: the larger of two values. */ + branchlessChoice: `; rdi and rsi hold two values rax = the larger + mov rax, rdi + cmp rsi, rax + cmova rax, rsi + ret +`, + + /** One procedure, called twice. */ + calledTwice: `; rdi = sequence rsi = n rax = G count plus C count + xor r14, r14 + mov dl, 'G' + call .tally + mov dl, 'C' + call .tally + mov rax, r14 + ret + +.tally: + xor ecx, ecx +.each: + cmp rcx, rsi + jae .back + movzx r8d, byte ptr [rdi + rcx*1] + cmp r8b, dl + jne .next + add r14, 1 +.next: + add rcx, 1 + jmp .each +.back: + ret +`, +}; + +/** A 256-byte table mapping a letter to a byte, for the lookup shapes. */ +export function byteTable(map, dflt = 0) { + const bytes = new Array(256).fill(dflt); + for (const [k, v] of Object.entries(map)) { + bytes[k.charCodeAt(0)] = typeof v === 'string' ? v.charCodeAt(0) : v; + } + return bytes; +} + +/** Little-endian eight-byte values, for the shapes that index with *8. */ +export function qwords(numbers) { + const out = []; + for (const n of numbers) { + let v = BigInt(n); + for (let k = 0; k < 8; k++) { out.push(Number(v & 0xffn)); v >>= 8n; } + } + return out; +} + +export const COMPLEMENT_TABLE = byteTable({ A: 'T', C: 'G', G: 'C', T: 'A' }, 'N'.charCodeAt(0)); +export const TWO_BIT_TABLE = byteTable({ A: 0, C: 1, G: 2, T: 3 }); + +/* ------------------------------------------------------------------ */ +/* Widget specs */ +/* ------------------------------------------------------------------ */ + +const GATTACA = 'GATTACA'; +const SEQ17 = 'GATTACAGGCATTAGCA'; +const SEQ32 = (SEQ17 + SEQ17).slice(0, 32); + +/** Doc 5, first widget: one instruction at a time, nothing else moving. */ +export const specFirstInstructions = { + title: 'Eight instructions, one at a time', + blurb: 'Read the next line, say what it will change, then step.', + code: SINGLE_STEPS, + registers: { rdi: 0x1000 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: GATTACA }], + layout: 'two-column', + views: [ + 'code', + ['registers', { watch: ['rax', 'rbx', 'rcx', 'rdx', 'r8'], radix: 'both' }], + 'explain', + 'flags', + ['dataflow', { span: 2 }], + ], +}; + +/** Doc 5: rax at four widths, for the zero-extension rule. */ +export const specRegisterWidths = { + title: 'One register, four names', + blurb: 'Watch what a four-byte write does to the upper half, and what a one-byte write does not do.', + code: ` mov rax, 0x1122334455667788 + mov eax, 0x99 ; four bytes: the top half is cleared + mov rbx, 0x1122334455667788 + mov bl, 0x99 ; one byte: everything above is left alone + ret +`, + layout: 'two-column', + views: [ + 'code', + ['registers', { watch: ['rax', 'rbx'], widths: ['rax', 'rbx'], radix: 'hex' }], + ['explain', { span: 2 }], + ], +}; + +/** Doc 5, the main event: count_base in assembly, beside lesson 1's table. */ +export const specCountBase = { + title: 'count_base, as the machine runs it', + blurb: 'The same scan as lesson 1. The trace table fills in exactly like the one you filled in by hand.', + code: COUNT_BASE, + registers: { rdi: 0x1000, rsi: GATTACA.length, rdx: 65 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: GATTACA }], + layout: 'two-column', + views: [ + 'code', + ['registers', { + watch: ['rax', 'rcx', 'r8', 'rdi', 'rsi', 'rdx'], + roles: { rax: 'total', rcx: 'i', r8: 'the letter just loaded', rdi: 'address of T', rsi: 'n', rdx: 'b' }, + radix: 'dec', + }], + ['memory', { title: 'The sequence in memory' }], + 'flags', + 'explain', + ['trace', { + title: 'One row per position, as in lesson 1', + rowsAt: '.next', + columns: [ + { label: 'i', reg: 'rcx' }, + { label: 'T[i]', byteAt: { base: 'rdi', index: 'rcx' } }, + { label: 'is it b?', compare: [{ byteAt: { base: 'rdi', index: 'rcx' } }, { reg: 'dl' }] }, + { label: 'total', reg: 'rax' }, + ], + }], + ['address', { span: 2 }], + ['counters', { only: ['instructions', 'comparisons', 'reads', 'branches'] }], + ], +}; + +/** Doc 5: the address adder, on its own. */ +export const specAddressing = { + title: 'Where [rdi + rcx*1] points', + blurb: 'Change nothing but rcx and the address moves along the strip.', + code: ` movzx eax, byte ptr [rdi + rcx*1] + add rcx, 1 + movzx eax, byte ptr [rdi + rcx*1] + add rcx, 1 + movzx eax, byte ptr [rdi + rcx*1] + lea rdx, [rdi + rcx*1] + ret +`, + registers: { rdi: 0x1000 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: GATTACA }], + layout: 'one-column', + views: ['code', 'address', ['memory', {}], ['registers', { watch: ['rax', 'rcx', 'rdx', 'rdi'], radix: 'hex' }], 'datapath'], +}; + +/** Doc 6: the nested search, with the window shaded and matches marked. */ +export const specNaiveSearch = { + title: 'naive_search, one comparison at a time', + blurb: 'The shaded run is the window at the current start. A pale cell marks a start already recorded as a match.', + code: NAIVE_SEARCH, + registers: { rdi: 0x1000, rsi: GATTACA.length, rdx: 0x2000, rcx: 3 }, + memory: [ + { name: 'T', label: 'T', addr: 0x1000, text: GATTACA }, + { name: 'P', label: 'P', addr: 0x2000, text: 'ATT' }, + ], + layout: 'two-column', + views: [ + 'code', + ['registers', { + watch: ['rax', 'r8', 'r10', 'r12', 'r13', 'r9'], + roles: { rax: 'matches', r8: 'i, the start', r10: 'j, inside the pattern', r12: 'T[i+j]', r13: 'P[j]', r9: 'n - m + 1' }, + radix: 'dec', + }], + ['memory', { + title: 'Text and pattern', + window: { start: 'r8', length: 'rcx', region: 'T' }, + marks: [{ label: '.hit', position: 'r8', region: 'T', className: 'match' }], + }], + 'explain', + 'flags', + ['trace', { + title: 'One row per comparison, as in lesson 3', + rowsAt: { matches: 'cmp r12b, r13b' }, + rowValues: 'after', + columns: [ + { label: 'i', reg: 'r8' }, + { label: 'j', reg: 'r10' }, + { label: 'T[i+j]', byteAt: { base: 'r11', index: 'r10' } }, + { label: 'P[j]', byteAt: { base: 'rdx', index: 'r10' } }, + { label: 'equal?', yesNo: 'ZF' }, + ], + }], + ['counters', { only: ['instructions', 'comparisons', 'reads', 'branches', 'mispredicts'] }], + 'branches', + ], +}; + +/** Doc 6: the same scan over enough sequence for the predictor to matter. */ +export const specBranchCost = { + title: 'What the branch predictor makes of DNA', + blurb: 'The loop branch is nearly always taken, so it is easy to guess. The letter comparison is close to a coin flip on a four-letter alphabet, so it is not.', + code: COUNT_BASE, + registers: { rdi: 0x1000, rsi: SEQ17.repeat(6).length, rdx: 65 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: SEQ17.repeat(6) }], + predictor: 'two-bit', + layout: 'two-column', + start: 0, + views: [ + 'code', + ['counters', { only: ['instructions', 'comparisons', 'branches', 'taken', 'mispredicts'] }], + ['branches', { span: 2 }], + ['registers', { watch: ['rax', 'rcx'], roles: { rax: 'total', rcx: 'i' }, radix: 'dec' }], + 'explain', + ], +}; + +/** Doc 5: the machine diagram with a load, an add, and a branch to compare. */ +export const specDatapath = { + title: 'Which parts of the machine each instruction wakes up', + blurb: 'Four instructions, four different routes through the same hardware.', + code: ` movzx eax, byte ptr [rdi + rcx*1] ; address adder, then data memory + add rax, 1 ; arithmetic unit only + cmp rax, 4 ; arithmetic unit, answer discarded + jne .away ; branch unit, reading the flags + lea rdx, [rdi + rcx*1] ; address adder, no memory at all +.away: + ret +`, + registers: { rdi: 0x1000, rcx: 1 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: GATTACA }], + layout: 'one-column', + views: ['code', 'datapath', 'dataflow', 'explain'], +}; + +/* -------- doc 6: what prediction costs -------------------------- */ + +/** The branchy scan and the branchless scan, over the same sequence. */ +export const specBranchless = { + title: 'The same count with the branch taken out', + blurb: 'sete turns the comparison into a number, so the processor has nothing left to guess inside the loop.', + code: COUNT_BRANCHLESS, + registers: { rdi: 0x1000, rsi: SEQ17.repeat(6).length, rdx: 65 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: SEQ17.repeat(6) }], + layout: 'two-column', + views: [ + 'code', + ['counters', { only: ['instructions', 'comparisons', 'branches', 'mispredicts'] }], + ['branches', { span: 2 }], + ['registers', { watch: ['rax', 'rcx', 'r8', 'r9'], roles: { rax: 'total', rcx: 'i', r8: 'the letter', r9: '1 or 0' }, radix: 'dec' }], + 'explain', + ], +}; + +/** A conditional move doing the same job. */ +export const specConditionalMove = { + title: 'Choosing a value without jumping', + blurb: 'cmove reads both possibilities and keeps one. No branch means no guess, and no guess means no mispredict.', + code: COUNT_CMOV, + registers: { rdi: 0x1000, rsi: SEQ17.length, rdx: 65 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: SEQ17 }], + layout: 'two-column', + views: [ + 'code', + ['registers', { watch: ['rax', 'r10', 'rcx', 'r8'], roles: { rax: 'total', r10: 'the total it would become', rcx: 'i', r8: 'the letter' }, radix: 'dec' }], + 'explain', + 'flags', + ['memory', {}], + ['counters', { only: ['instructions', 'branches', 'mispredicts'] }], + ], +}; + +/** Three predictors on the branchy scan, for comparison against the two above. */ +export const specPredictors = { + title: 'Three predictors, one scan', + blurb: 'Run the branchy scan under each predictor in turn. The loop branch is easy for all three; the letter comparison is hard for all three.', + code: COUNT_BASE, + registers: { rdi: 0x1000, rsi: SEQ17.repeat(6).length, rdx: 65 }, + memory: [{ name: 'T', label: 'T', addr: 0x1000, text: SEQ17.repeat(6) }], + layout: 'two-column', + views: [ + ['predictors', { kinds: ['always-taken', 'never-taken', 'one-bit', 'two-bit'] }], + 'code', + ['branches', {}], + ], +}; + +/* -------- doc 7: sixteen at a time ------------------------------ */ + +/** Sixteen letters compared in one instruction. */ +export const specVectorCount = { + title: 'Counting sixteen letters at once', + blurb: 'One pcmpeqb compares sixteen pairs of bytes. Watch the lanes fill in, then watch the mask collect one bit from each.', + code: COUNT_VECTOR, + registers: { rdi: 0x1000, rsi: SEQ17.length, rdx: 0x2000 }, + memory: [ + { name: 'T', label: 'T', addr: 0x1000, text: SEQ17 }, + { name: 'B', label: 'b \u00d7 16', addr: 0x2000, text: 'A'.repeat(16) }, + ], + layout: 'two-column', + views: [ + 'code', + ['registers', { watch: ['rax', 'rcx', 'r8', 'r13'], roles: { rax: 'total', rcx: 'i', r8: 'the mask, then the count', r13: 'b, for the leftovers' }, radix: 'dec' }], + ['vectors', { + span: 2, + watch: ['xmm0', 'xmm1'], + roles: { xmm0: 'sixteen letters, then sixteen verdicts', xmm1: 'sixteen copies of b' }, + maskFrom: 'r8d', + }], + ['memory', {}], + 'explain', + ['counters', { only: ['instructions', 'comparisons', 'reads', 'branches'] }], + ], +}; + +/** The first-byte filter that real read aligners run before any confirmation. */ +export const specVectorFilter = { + title: 'Sixteen candidate starts at a time', + blurb: 'Compare the pattern\u2019s first letter against sixteen positions at once, then walk the set bits. Every start the filter rejects is a start the slow check never sees.', + code: VECTOR_FILTER, + registers: { rdi: 0x1000, rsi: 32, rdx: 0x2000 }, + memory: [ + { name: 'T', label: 'T', addr: 0x1000, text: SEQ32 }, + { name: 'B', label: 'P[0] \u00d7 16', addr: 0x2000, text: 'A'.repeat(16) }, + ], + layout: 'two-column', + views: [ + 'code', + ['vectors', { span: 2, watch: ['xmm0', 'xmm1'], roles: { xmm0: 'letters, then verdicts', xmm1: 'sixteen copies of P[0]' }, maskFrom: 'r8d' }], + ['registers', { watch: ['rax', 'r8', 'r10', 'r11', 'rcx'], roles: { rax: 'candidates found', r8: 'the mask, bits cleared as used', r10: 'lane of the lowest set bit', r11: 'the candidate start', rcx: 'block start' }, radix: 'dec' }], + ['memory', { marks: [{ label: '.bits', position: 'r11', region: 'T', className: 'match' }] }], + 'explain', + ['counters', { only: ['instructions', 'comparisons', 'reads', 'branches', 'mispredicts'] }], + ], +}; + +/* -------- the shapes, as widgets --------------------------------- */ + +/** Binary search, for any lesson about indexes. */ +export const specBinarySearch = { + title: 'Halving a range', + blurb: 'The midpoint comes from an lea and a shr. Watch lo and hi close in, and notice the scale of *8 because these values are eight bytes wide.', + code: PATTERNS.binarySearch, + registers: { rdi: 0x1000, rsi: 10, rdx: 57 }, + memory: [{ name: 'values', label: 'values', addr: 0x1000, bytes: qwords([3, 11, 14, 22, 40, 57, 61, 88, 90, 104]) }], + layout: 'two-column', + views: [ + 'code', + ['registers', { watch: ['rax', 'r8', 'r9', 'r10', 'r11', 'rdx'], roles: { rax: 'the answer', r8: 'lo', r9: 'hi', r10: 'mid', r11: 'the value at mid', rdx: 'the value wanted' }, radix: 'dec' }], + 'explain', + ['address', {}], + ['trace', { + title: 'One row per halving', + rowsAt: '.narrow', + columns: [ + { label: 'lo', reg: 'r8' }, + { label: 'hi', reg: 'r9' }, + { label: 'mid', reg: 'r10' }, + { label: 'value at mid', reg: 'r11' }, + ], + }], + ['counters', { only: ['instructions', 'comparisons', 'reads', 'branches'] }], + ], +}; + +/** A table lookup writing its answer to memory. */ +export const specTableLookup = { + title: 'Data as an index', + blurb: 'The letter is the index. One load replaces a chain of comparisons, and the result goes straight to memory.', + code: PATTERNS.tableLookup, + registers: { rdi: 0x1000, rsi: 17, rdx: 0x2000, r14: 0x3000 }, + memory: [ + { name: 'T', label: 'T', addr: 0x1000, text: SEQ17 }, + { name: 'table', label: 'table', addr: 0x2000, bytes: COMPLEMENT_TABLE }, + { name: 'out', label: 'out', addr: 0x3000, size: 17 }, + ], + layout: 'two-column', + views: [ + 'code', + ['registers', { watch: ['rcx', 'r8', 'r9'], roles: { rcx: 'i', r8: 'the letter', r9: 'its complement' }, radix: 'dec' }], + ['memory', { title: 'Sequence in, complement out', regions: ['T', 'out'] }], + ['address', {}], + 'explain', + ], +}; + +/** Two pointers over two sorted lists. */ +export const specTwoPointers = { + title: 'Two pointers', + blurb: 'Each index moves only when the comparison says it is behind. No counted loop can do this.', + code: PATTERNS.twoPointers, + registers: { rdi: 0x1000, rsi: 6, rdx: 0x2000, rcx: 7 }, + memory: [ + { name: 'A', label: 'A', addr: 0x1000, bytes: qwords([1, 4, 6, 10, 13, 16]) }, + { name: 'B', label: 'B', addr: 0x2000, bytes: qwords([2, 4, 5, 10, 16, 20, 31]) }, + ], + layout: 'two-column', + views: [ + 'code', + ['registers', { watch: ['rax', 'r8', 'r9', 'r10', 'r11'], roles: { rax: 'shared values', r8: 'index into A', r9: 'index into B', r10: 'A value', r11: 'B value' }, radix: 'dec' }], + 'explain', + ['trace', { + title: 'One row per comparison', + rowsAt: { matches: 'cmp r10, r11' }, + rowValues: 'after', + columns: [ + { label: 'i', reg: 'r8' }, + { label: 'j', reg: 'r9' }, + { label: 'A[i]', reg: 'r10' }, + { label: 'B[j]', reg: 'r11' }, + { label: 'equal?', yesNo: 'ZF' }, + ], + }], + ['counters', { only: ['instructions', 'comparisons', 'reads'] }], + ], +}; + +/** Packing letters two bits at a time. */ +export const specPackBits = { + title: 'Packing letters into one number', + blurb: 'Shift left by two to make room, then OR the next code in. Thirty-two letters fit in one register.', + code: PATTERNS.packBits, + registers: { rdi: 0x1000, rsi: 7, rdx: 0x2000 }, + memory: [ + { name: 'T', label: 'T', addr: 0x1000, text: 'GATTACA' }, + { name: 'codes', label: 'codes', addr: 0x2000, bytes: TWO_BIT_TABLE }, + ], + layout: 'two-column', + views: [ + 'code', + ['registers', { watch: ['rax', 'rcx', 'r8', 'r9'], roles: { rax: 'the packed code', rcx: 'i', r8: 'the letter', r9: 'its two-bit code' }, radix: 'both' }], + ['memory', { regions: ['T'] }], + 'explain', + ['trace', { + title: 'The code growing two bits at a time', + rowsAt: '.each', + columns: [ + { label: 'i', reg: 'rcx' }, + { label: 'T[i]', byteAt: { base: 'rdi', index: 'rcx' } }, + { label: 'packed so far', hex: 'rax' }, + ], + }], + ], +}; + +export const ALL_SPECS = { + specFirstInstructions, + specRegisterWidths, + specCountBase, + specAddressing, + specNaiveSearch, + specBranchCost, + specDatapath, + specBranchless, + specConditionalMove, + specPredictors, + specVectorCount, + specVectorFilter, + specBinarySearch, + specTableLookup, + specTwoPointers, + specPackBits, +}; diff --git a/src/engines/asm86/uops.js b/src/engines/asm86/uops.js new file mode 100644 index 0000000..137d75d --- /dev/null +++ b/src/engines/asm86/uops.js @@ -0,0 +1,401 @@ +/** + * asm86/uops.js + * + * What an instruction asks of the hardware underneath it: which values it + * reads and writes, and which micro-operations a modern x86 core splits it + * into. + * + * An x86 instruction is not what the execution units see. The front end of a + * modern core translates each one into micro-ops (uops): simple, fixed-format + * operations, each needing one kind of execution unit. `add rax, [rdi]` is + * two of them, a load and an add; `add [rdi], rax` is four. The out-of-order + * core schedules uops, not instructions, which is why an instruction count + * says so little about time. + * + * The port layout and latencies below are shaped like Intel's Skylake, the + * most documented x86 core, and rounded to the numbers a student can hold in + * their head. They are a teaching model, not a cycle-accurate description of + * any product: the shape (four ALUs, two load ports, one divider, a + * three-cycle multiplier) is the lesson, not the exact figures. + */ + +import { OPS, REGS, CONDITION_FLAGS } from './interp.js'; + +/* ------------------------------------------------------------------ */ +/* Ports */ +/* ------------------------------------------------------------------ */ + +/** Each execution port and the kinds of uop it can accept. */ +export const PORTS = { + p0: { label: 'port 0', units: ['alu', 'shift', 'branch', 'vec', 'div', 'vec-mask', 'vec-move', 'fp', 'fp-div', 'fp-sqrt', 'fp-cvt', 'fp-cmp'] }, + p1: { label: 'port 1', units: ['alu', 'mul', 'bitcount', 'vec', 'lea', 'lea-slow', 'fp', 'fp-cvt'] }, + p5: { label: 'port 5', units: ['alu', 'vec', 'lea', 'vec-move'] }, + p6: { label: 'port 6', units: ['alu', 'shift', 'branch'] }, + p2: { label: 'port 2', units: ['load', 'store-addr'] }, + p3: { label: 'port 3', units: ['load', 'store-addr'] }, + p4: { label: 'port 4', units: ['store-data'] }, + p7: { label: 'port 7', units: ['store-addr'] }, +}; +export const PORT_ORDER = ['p0', 'p1', 'p5', 'p6', 'p2', 'p3', 'p4', 'p7']; + +/** Uop kind -> latency in cycles, and what the kind means in words. */ +export const UOP_KINDS = { + alu: { latency: 1, text: 'integer arithmetic or logic' }, + shift: { latency: 1, text: 'shift, rotate, setcc, or cmov' }, + branch: { latency: 1, text: 'check a branch against its prediction' }, + mul: { latency: 3, text: 'integer multiply, pipelined: one can start every cycle' }, + div: { latency: 26, text: 'integer divide, not pipelined: the divider is busy until it finishes' }, + bitcount: { latency: 3, text: 'popcnt, tzcnt, lzcnt, bsf, bsr' }, + lea: { latency: 1, text: 'address arithmetic with one or two parts' }, + 'lea-slow': { latency: 3, text: 'address arithmetic with base, index, and displacement all present' }, + vec: { latency: 1, text: 'sixteen byte lanes at once' }, + 'vec-mask': { latency: 2, text: 'gather one bit from each lane' }, + 'vec-move': { latency: 2, text: 'move between vector and ordinary registers' }, + fp: { latency: 4, text: 'add, subtract or multiply doubles, pipelined: one can start every cycle on each of two ports' }, + 'fp-div': { latency: 14, text: 'divide doubles, not pipelined: the divider is busy until it finishes' }, + 'fp-sqrt': { latency: 18, text: 'square root of a double, not pipelined: the divider is busy until it finishes' }, + 'fp-cvt': { latency: 5, text: 'convert between an integer and a double' }, + 'fp-cmp': { latency: 3, text: 'compare doubles into the flags' }, + load: { latency: 4, text: 'read memory; the time depends on where the line is found' }, + 'store-addr': { latency: 1, text: 'compute a store address and put it in the store buffer' }, + 'store-data': { latency: 1, text: 'put the value being stored into the store buffer' }, + nop: { latency: 0, text: 'takes a slot in the reorder buffer and nothing else' }, + eliminated: { latency: 0, text: 'handled at rename: no execution unit needed' }, + locked: { latency: 18, text: 'a locked read-modify-write: waits for everything before it' }, + system: { latency: 0, text: 'enter the kernel; everything before must finish first' }, +}; + +export function portsFor(kind) { + return PORT_ORDER.filter((p) => PORTS[p].units.includes(kind)); +} + +/* ------------------------------------------------------------------ */ +/* What each instruction reads and writes */ +/* ------------------------------------------------------------------ */ + +const base = (name) => REGS[name]?.base ?? name; + +/** + * Flags are renamed in three groups, close to how recent Intel cores track + * them: CF on its own, OF on its own, and ZF with SF and PF (the adjust + * flag this subset does not model). An instruction that writes only + * some groups, such as inc, which keeps CF, therefore creates no false + * dependency on the others, and a reader waits only for the groups it tests. + */ +export const FLAG_GROUPS = ['cf', 'of', 'zs']; +const GROUP_OF = { CF: 'cf', OF: 'of', ZF: 'zs', SF: 'zs', PF: 'zs' }; +export const FLAG_GROUP_NAME = { cf: 'CF', of: 'OF', zs: 'ZF/SF/PF' }; +export const isFlagGroup = (n) => n === 'cf' || n === 'of' || n === 'zs'; +/** A dependency name as a reader would write it. */ +export const depName = (n) => FLAG_GROUP_NAME[n] || n; +const addrRegs = (o) => [o.base, o.index].filter((r) => r && r !== 'rip').map(base); + +/** + * The architectural values an instruction reads and writes, by name: + * 64-bit register names, 'xmm0'..'xmm15', and the flag groups 'cf', 'of', + * and 'zs'. Memory is handled + * separately, from the trace's recorded accesses. + * + * Two subtleties matter to a pipeline: + * a write to an 8- or 16-bit register also *reads* the full register, + * because the untouched bits have to be merged back in; + * xor or sub of a register with itself reads nothing: the answer is zero + * whatever the register held, and hardware knows it. + */ +export function effects(instr) { + const { op, operands: ops } = instr; + const cls = OPS[op].cls; + const reads = new Set(); + const writes = new Set(); + const mem = ops.find((o) => o.kind === 'mem'); + if (mem) for (const r of addrRegs(mem)) reads.add(r); + + const readOp = (o) => { if (o?.kind === 'reg') reads.add(base(o.name)); }; + const writeOp = (o) => { + if (o?.kind !== 'reg') return; + writes.add(base(o.name)); + const info = REGS[o.name]; + if (info.size < 4) reads.add(base(o.name)); // partial write: merge with the old value + }; + + const zeroIdiom = ['xor', 'sub', 'pxor', 'psubb'].includes(op) && ops.length === 2 + && ops[0].kind === 'reg' && ops[1].kind === 'reg' && ops[0].name === ops[1].name; + + if (zeroIdiom) { writeOp(ops[0]); if (cls.flags) FLAG_GROUPS.forEach((g) => writes.add(g)); return { reads, writes, zeroIdiom: true }; } + + switch (true) { + case (op === 'movsd' || op === 'vmovsd') && ops.length === 2: + // a register-to-register movsd keeps the destination's upper half, so it reads the destination too; a load clears it + readOp(ops[1]); if (op === 'movsd' && ops[0].kind === 'reg' && ops[1].kind === 'reg') readOp(ops[0]); writeOp(ops[0]); + break; + case ['mov', 'movabs', 'movzx', 'movsx', 'movsxd', 'lea', 'movdqu', 'movdqa', 'movd', 'movq', 'pmovmskb'].includes(op): + readOp(ops[1]); writeOp(ops[0]); + break; + case ['cmp', 'test', 'bt', 'ucomisd', 'vucomisd', 'comisd', 'vcomisd'].includes(op): + readOp(ops[0]); readOp(ops[1]); + break; + case ['push'].includes(op): readOp(ops[0]); break; + case ['pop'].includes(op): writeOp(ops[0]); break; + case op === 'xchg': readOp(ops[0]); readOp(ops[1]); writeOp(ops[0]); writeOp(ops[1]); break; + case op === 'leave': reads.add('rbp'); writes.add('rbp'); writes.add('rsp'); break; + case ['mul', 'div', 'idiv'].includes(op) || (op === 'imul' && ops.length === 1): { + const w = ops[0].kind === 'reg' ? REGS[ops[0].name].size : ops[0].size; + readOp(ops[0]); reads.add('rax'); writes.add('rax'); + if (op !== 'mul' && op !== 'imul') { if (w > 1) reads.add('rdx'); } + if (w > 1) writes.add('rdx'); + break; + } + case op === 'imul' && ops.length === 3: readOp(ops[1]); writeOp(ops[0]); break; + case op === 'cqo' || op === 'cdq': reads.add('rax'); writes.add('rdx'); break; + case op === 'cdqe': reads.add('rax'); writes.add('rax'); break; + case ['popcnt', 'tzcnt', 'lzcnt'].includes(op): readOp(ops[1]); writeOp(ops[0]); break; + case ['bsf', 'bsr'].includes(op): readOp(ops[1]); readOp(ops[0]); writeOp(ops[0]); break; // unchanged on zero + case op.startsWith('cmov'): readOp(ops[0]); readOp(ops[1]); writeOp(ops[0]); break; + case op === 'shld' || op === 'shrd': readOp(ops[0]); readOp(ops[1]); readOp(ops[2]); writeOp(ops[0]); break; // the destination is a source too + case ops.length === 3 && op !== 'imul': readOp(ops[1]); readOp(ops[2]); writeOp(ops[0]); break; // the BMI and AVX three-operand shapes: two sources, one destination + case op === 'mulx': reads.add('rdx'); readOp(ops[2]); writeOp(ops[0]); writeOp(ops[1]); break; + case op === 'xadd': readOp(ops[0]); readOp(ops[1]); writeOp(ops[0]); writeOp(ops[1]); break; + case op === 'cmpxchg': readOp(ops[0]); readOp(ops[1]); reads.add('rax'); writeOp(ops[0]); writes.add('rax'); break; + case op.startsWith('set'): writeOp(ops[0]); break; + case op === 'syscall': + ['rax', 'rdi', 'rsi', 'rdx'].forEach((r) => reads.add(r)); + ['rax', 'rcx', 'r11'].forEach((r) => writes.add(r)); + break; + case ['jmp', 'call', 'ret', 'nop', 'stc', 'clc', 'cmc'].includes(op) || /^j/.test(op): break; + default: + // two-operand arithmetic, and the one-operand inc, dec, neg, not, and shifts + readOp(ops[0]); if (ops[1]) readOp(ops[1]); writeOp(ops[0]); + } + + // flags, by group + if (cls.conditional) for (const f of CONDITION_FLAGS[OPS[op].cond]) reads.add(GROUP_OF[f]); + if (['adc', 'sbb', 'cmc'].includes(op)) reads.add('cf'); + if (op === 'syscall') FLAG_GROUPS.forEach((g) => reads.add(g)); // r11 receives the whole of RFLAGS + if (cls.flags) { + const fw = new Set(); + if (op === 'inc' || op === 'dec') { fw.add('of'); fw.add('zs'); } // CF is left alone + else if (op === 'bt' || op === 'stc' || op === 'clc' || op === 'cmc') fw.add('cf'); + else if (op === 'rol' || op === 'ror') { fw.add('cf'); fw.add('of'); } // rotates leave ZF and SF alone + else if (op === 'shld' || op === 'shrd') { + FLAG_GROUPS.forEach((g) => fw.add(g)); + const w = ops[0].kind === 'reg' ? REGS[ops[0].name].size : ops[0].size; + if (ops[2]?.kind === 'reg') for (const g of fw) reads.add(g); // a count of zero would leave them as they were + else if (((ops[2]?.value ?? 1n) & (w === 8 ? 63n : 31n)) === 0n) fw.clear(); // a count of zero writes nothing + } + else FLAG_GROUPS.forEach((g) => fw.add(g)); + if (cls.shift) { + const w = ops[0].kind === 'reg' ? REGS[ops[0].name].size : ops[0].size; + const mask = w === 8 ? 63n : 31n; + if (ops[1]?.kind === 'reg') for (const g of fw) reads.add(g); // a count of zero would leave them as they were + else if ((((ops[1]?.value ?? 1n) & mask)) === 0n) fw.clear(); // a count of zero writes nothing + } + for (const g of fw) writes.add(g); + } + if (cls.stack) { reads.add('rsp'); writes.add('rsp'); } + if (op === 'call' || op === 'ret') { reads.add('rsp'); writes.add('rsp'); } + return { reads, writes, zeroIdiom: false }; +} + +/** Which flags a flag-reading instruction actually looks at, for display. */ +export function flagsRead(instr) { + const c = OPS[instr.op].cond; + if (c) return CONDITION_FLAGS[c]; + if (['adc', 'sbb', 'cmc'].includes(instr.op)) return ['CF']; + return []; +} + +/* ------------------------------------------------------------------ */ +/* Micro-op decomposition */ +/* ------------------------------------------------------------------ */ + +const FUSIBLE_FIRST = new Set(['cmp', 'test', 'add', 'sub', 'and', 'inc', 'dec']); + +/** Conditions each kind of first instruction can fuse with, as on Skylake. */ +const ARITH_CC = new Set(['e', 'z', 'ne', 'nz', 'b', 'nae', 'c', 'ae', 'nb', 'nc', 'be', 'na', 'a', 'nbe', 'l', 'nge', 'ge', 'nl', 'le', 'ng', 'g', 'nle']); +const INCDEC_CC = new Set(['e', 'z', 'ne', 'nz', 'l', 'nge', 'ge', 'nl', 'le', 'ng', 'g', 'nle']); + +/** + * Can this instruction and the conditional jump after it become one uop? + * test and and fuse with every condition; cmp, add, and sub with every + * condition but the overflow, sign, and parity tests; inc and dec only with + * the tests that ignore CF. + */ +export function canFuse(a, b) { + if (!a || !b || !OPS[b.op].cls.conditional || !/^j/.test(b.op)) return false; + if (!FUSIBLE_FIRST.has(a.op)) return false; + const hasMem = a.operands.some((o) => o.kind === 'mem'); + const hasImm = a.operands.some((o) => o.kind === 'imm'); + if (hasMem && hasImm) return false; // cmp [mem], imm does not fuse + const cc = OPS[b.op].cond; + if (['and', 'test'].includes(a.op)) return true; + if (['inc', 'dec'].includes(a.op)) return INCDEC_CC.has(cc); + return ARITH_CC.has(cc); +} + +function divLatency(width, signed) { + return { 1: 23, 2: 24, 4: 26, 8: signed ? 42 : 36 }[width] ?? 26; +} + +/** + * Split one instruction into uops. Each uop names: + * kind a key of UOP_KINDS + * srcs values it waits for: architectural names, or 't0', 't1' for + * values passed between uops of the same instruction + * dsts values it produces + * latency cycles from starting to finishing (loads are filled in later, + * from the cache model) + * note one line for the uops view + * + * options: { eliminate: true } rename-time tricks on or off + */ +export function decompose(instr, options = {}) { + const opts = { eliminate: true, ...options }; + const { op, operands: ops } = instr; + const cls = OPS[op].cls; + const fx = effects(instr); + const mem = ops.find((o) => o.kind === 'mem'); + const aregs = mem ? addrRegs(mem) : []; + const u = []; + const add = (kind, srcs, dsts, note, extra = {}) => { + u.push({ kind, srcs: [...new Set(srcs)], dsts: [...new Set(dsts)], latency: extra.latency ?? UOP_KINDS[kind].latency, note, ...extra }); + }; + const regDsts = [...fx.writes]; + const flagW = [...fx.writes].filter(isFlagGroup); + const flagR = [...fx.reads].filter(isFlagGroup); + const regSrcs = [...fx.reads].filter((r) => !aregs.includes(r) || ops.some((o) => o.kind === 'reg' && base(o.name) === r)); + + if (fx.zeroIdiom && opts.eliminate) { + add('eliminated', [], regDsts, `${op} of a register with itself is always zero: recognized at rename, no execution unit, no wait on the old value`); + return u; + } + if (op === 'nop') { add('nop', [], [], 'occupies a slot and retires; nothing executes'); return u; } + if (op === 'syscall') { add('system', [...fx.reads], regDsts, 'serializing: waits for every older instruction, then hands control to the kernel'); return u; } + + const loadInto = (t) => add('load', aregs, [t], `load ${mem.size || 8} byte${(mem.size || 8) > 1 ? 's' : ''} from ${mem.source}`); + const store = (srcVal) => { + add('store-addr', aregs, [], `work out ${mem.source} for the store buffer`); + add('store-data', srcVal ? [srcVal] : [], [], srcVal ? 'send the value to the store buffer' : 'send the constant to the store buffer'); + }; + + // movsd / vmovsd: one double between memory and an xmm register, or between registers + if (op === 'movsd' || op === 'vmovsd') { + const [d, s] = ops; + if (ops.length === 3) { add('vec', [base(ops[1].name), base(ops[2].name)], [base(d.name)], 'merge the third operand\'s double into the second\'s upper half'); return u; } + if (d.kind === 'mem') { store(base(s.name)); return u; } + if (s.kind === 'mem') { add('load', aregs, [base(d.name)], `load one double (8 bytes) from ${s.source}; the rest of the register is cleared`); return u; } + add('vec', [base(s.name), base(d.name)], [base(d.name)], 'copy the low double, keeping the destination\'s upper half'); + return u; + } + + // Moves + if (['mov', 'movabs', 'movzx', 'movsx', 'movsxd', 'movdqu', 'movdqa', 'movd', 'movq'].includes(op)) { + const [d, s] = ops; + const vec = cls.vector; + if (d.kind === 'mem') { + if (s.kind === 'reg') store(base(s.name)); + else store(null); + return u; + } + // An 8- or 16-bit destination keeps the rest of its register, so the + // write must wait for the old value and merge the new bits into it. + const narrow = REGS[d.name].size < 4; + if (s.kind === 'mem') { + if (narrow) { + add('load', aregs, ['t0'], `load ${s.size} byte${s.size > 1 ? 's' : ''} from ${s.source}`); + add('alu', ['t0', base(d.name)], [base(d.name)], `merge into the low bits of ${base(d.name)}, keeping the rest`); + return u; + } + add('load', aregs, [base(d.name)], `load ${s.size} byte${s.size > 1 ? 's' : ''} from ${s.source}${op === 'movzx' ? ', zero-extended on the way in' : op.startsWith('movs') ? ', sign-extended on the way in' : ''}`); + return u; + } + if (s.kind === 'imm') { add('alu', narrow ? [base(d.name)] : [], [base(d.name)], narrow ? `write a constant into the low bits of ${base(d.name)}, keeping the rest` : 'write a constant'); return u; } + const cross = vec && (REGS[d.name].vector !== REGS[s.name].vector); + if (cross) { add('vec-move', [base(s.name)], [base(d.name)], 'cross between the vector and integer register files'); return u; } + const wide = REGS[d.name].size >= 4 && op !== 'movsx' && op !== 'movsxd' && !(op === 'movzx' && REGS[s.name].size === 2); + if (opts.eliminate && wide && op !== 'movzx') { + add('eliminated', [base(s.name)], [base(d.name)], 'move elimination: the renamer points both names at one physical register, and nothing executes'); + return u; + } + add(vec ? 'vec' : 'alu', [base(s.name), ...(narrow ? [base(d.name)] : [])], [base(d.name)], op === 'movzx' || op === 'movsx' ? 'widen a register' : narrow ? `copy into the low bits of ${base(d.name)}, keeping the rest` : 'copy a register'); + return u; + } + + if (op === 'lea') { + // On Skylake the slow form is three components, or an index with rbp or r13 as the base. + const parts = [mem.base, mem.index, mem.disp !== 0n ? 'd' : null].filter(Boolean).length; + const slow = parts === 3 || (Boolean(mem.index) && (mem.base === 'rbp' || mem.base === 'r13')); + add(slow ? 'lea-slow' : 'lea', aregs, [base(ops[0].name)], slow ? 'three-part address arithmetic: one port, three cycles' : 'address arithmetic on a spare adder'); + return u; + } + + if (op === 'push') { + if (ops[0].kind === 'mem') add('load', aregs, ['t0'], `load from ${ops[0].source}`); + add('store-addr', [], [], 'the stack engine supplies rsp - 8; no wait on rsp'); + add('store-data', ops[0].kind === 'reg' ? [base(ops[0].name)] : ops[0].kind === 'mem' ? ['t0'] : [], [], 'send the value to the store buffer'); + return u; + } + if (op === 'pop') { + if (ops[0].kind === 'reg') add('load', [], [base(ops[0].name)], 'load from the top of the stack; the stack engine tracks rsp'); + else { add('load', [], ['t0'], 'load from the top of the stack'); store('t0'); } + return u; + } + if (op === 'call') { + add('store-addr', [], [], 'store the return address below rsp'); + add('store-data', [], [], 'the return address itself'); + add('branch', [], [], 'redirect fetch to the target', { taken: true }); + return u; + } + if (op === 'ret') { + add('load', [], ['t0'], 'load the return address'); + add('branch', ['t0'], [], 'check the return stack buffer’s guess against it', { taken: true }); + return u; + } + if (op === 'leave') { + add('alu', ['rbp'], ['rsp'], 'rsp <- rbp'); + add('load', ['rsp'], ['rbp'], 'pop rbp'); + return u; + } + if (op === 'jmp') { add('branch', [], [], 'unconditional: resolved as soon as it is decoded', { taken: true }); return u; } + if (cls.conditional && /^j/.test(op)) { add('branch', flagR, [], `read ${flagR.map(depName).join(' and ')} and check the prediction`); return u; } + + if (op === 'xchg') { + if (mem) { + add('locked', [...aregs, ...regSrcs], regDsts, 'xchg with memory is implicitly locked: it waits for every older store and blocks younger loads'); + return u; + } + add('alu', regSrcs, ['t0'], 'swap, part 1'); add('alu', regSrcs, ['t1'], 'swap, part 2'); add('alu', ['t0', 't1'], regDsts, 'swap, part 3'); + return u; + } + + if (['div', 'idiv'].includes(op)) { + const w = mem ? mem.size : REGS[ops[0].name].size; + if (mem) loadInto('t0'); + add('div', mem ? ['t0', ...regSrcs] : regSrcs, regDsts, `${w * 8}-bit division: about ${divLatency(w, op === 'idiv')} cycles, and the divider accepts nothing else meanwhile`, { latency: divLatency(w, op === 'idiv'), unpipelined: true }); + return u; + } + + // Everything else: an optional load, one execution uop, an optional store. + let kind = 'alu'; + if (cls.multiply) kind = 'mul'; + else if (cls.bitcount) kind = 'bitcount'; + else if (cls.shift || op.startsWith('set') || op.startsWith('cmov')) kind = 'shift'; + else if (op === 'pmovmskb') kind = 'vec-mask'; + else if (cls.fp) kind = cls.fp === 'add' || cls.fp === 'mul' ? 'fp' : `fp-${cls.fp}`; + else if (cls.vector) kind = 'vec'; + + const destMem = mem && ops[0] === mem && cls.write; + const execSrcs = regSrcs.filter((r) => !(mem && aregs.includes(r) && !ops.some((o) => o.kind === 'reg' && base(o.name) === r))); + // setcc writes its byte without reading it; everything else here reads its memory operand + const readsMem = mem && !op.startsWith('set'); + if (readsMem) loadInto('t0'); + const outs = destMem ? ['t1', ...flagW] : regDsts; + const wideMul = (op === 'mul' || (op === 'imul' && ops.length === 1)) && (mem ? mem.size : REGS[ops[0].name].size) > 1; + if (wideMul) { + add('mul', [...(mem ? ['t0'] : []), ...execSrcs], ['rax', ...flagW], 'multiply, low half of the product', { latency: 3 }); + add('mul', [...(mem ? ['t0'] : []), ...execSrcs], ['rdx'], 'the high half of the product, on its own micro-op', { latency: 4 }); + return u; + } + add(kind, [...(readsMem ? ['t0'] : []), ...execSrcs], outs, UOP_KINDS[kind].text, kind === 'fp-div' || kind === 'fp-sqrt' ? { unpipelined: true } : {}); + if (destMem) store('t1'); + return u; +} diff --git a/src/engines/asm86/views.js b/src/engines/asm86/views.js new file mode 100644 index 0000000..a17a488 --- /dev/null +++ b/src/engines/asm86/views.js @@ -0,0 +1,1312 @@ +/** + * asm86/views.js + * + * A library of views over a trace produced by interp.js. Every view is a + * factory taking (trace, options) and returning { node, update(frame) }. + * No view knows anything about a particular program: each one reads the + * recorded instructions and state and draws whatever it finds. + * + * Nothing here depends on a framework. The nodes are plain DOM elements, so + * they drop into Observable JS, a Quarto page, or a bare HTML file. + */ + +import { + REG64, XMM, REGS, OPS, FLAG_NAMES, CONDITION_TEXT, CONDITION_FLAGS, CONDITIONS, SYSCALLS, + regValue, readByte, formatHex, formatSigned, byteChar, lanes, namesOf, + run, assemble, fromObjdump, AsmError, splitComment, +} from './interp.js'; +import { el, sv, add, clear, panel, valueOf } from './dom.js'; +import { HW_VIEWS } from './hwviews.js'; +import { hardwareFor, PRESETS, resolveMachine } from './machine.js'; + +/** A frame is everything a view needs to draw one moment. */ +export function frameAt(trace, t) { + const clamped = Math.max(0, Math.min(t, trace.count)); + const state = trace.states[clamped]; + const last = clamped > 0 ? trace.steps[clamped - 1] : null; + const next = state.pc < trace.program.instrs.length && !state.halted + ? trace.program.instrs[state.pc] + : null; + return { t: clamped, trace, state, last, next }; +} + +/* ------------------------------------------------------------------ */ +/* 1. Code listing */ +/* ------------------------------------------------------------------ */ + +/** + * The program, with a marker on the instruction about to run, a tint on the + * one that just ran, and an execution count per line. + * + * options: { + * heat: true, tint hot lines + * showCounts: true, execution count in the left gutter + * wideAt: 60, take the full row past this many characters + * addresses: false, each instruction's byte address, as objdump prints it + * bytes: false each instruction's machine code + * } + */ +export function codeView(trace, options = {}) { + const opts = { heat: true, showCounts: true, wideAt: 60, addresses: false, bytes: false, ...options }; + const rows = []; + const body = el('div', { class: 'a86-code' }); + const layout = trace.layout?.ok ? trace.layout : null; + const machine = layout && (opts.addresses || opts.bytes); + if (machine) body.classList.add('has-machine'); + + /** + * An instruction and its comment go in separate cells. The instruction + * cell is sized once, in characters, from the longest instruction in the + * program, so comments line up in a column across every row and the + * instruction itself can never be clipped. The comment cell takes the + * slack and wraps when there is not enough of it. + */ + const codeOf = (raw) => { + const { code, comment } = splitComment(raw.replace(/\s+$/, '')); + return { code: code.replace(/\s+$/, ''), comment: comment ? `; ${comment}`.trim() : '' }; + }; + + const parts = trace.program.lines.map((line) => ({ line, ...codeOf(line.raw) })); + const widest = Math.max(8, ...parts.map((p) => p.code.length)); + const widestComment = Math.max(0, ...parts.filter((p) => p.code.length).map((p) => p.comment.length)); + body.style.setProperty('--a86-code-cols', `${widest + 1}ch`); + const widestBytes = layout ? Math.max(2, ...layout.hex.map((h) => h.length)) : 0; + body.style.setProperty('--a86-addr-cols', opts.addresses ? '7ch' : '0'); + body.style.setProperty('--a86-bytes-cols', opts.bytes ? `${widestBytes + 2}ch` : '0'); + + for (const p of parts) { + if (p.line.kind === 'blank' && !p.code.length && !p.comment.length) continue; + const bare = !p.code.length && p.comment.length; // a comment on a line of its own + const row = el('div', { class: `a86-code-row a86-code-${p.line.kind}${bare ? ' is-note' : ''}` }); + const count = el('span', { class: 'a86-code-count' }); + const marker = el('span', { class: 'a86-code-marker' }); + add(row, [count, marker]); + if (machine && !bare) { + const i = p.line.instrIndex; + add(row, el('code', { class: 'a86-code-addr', text: opts.addresses && i != null ? layout.addr[i].toString(16) : '' })); + add(row, el('code', { class: 'a86-code-bytes', text: opts.bytes && i != null ? layout.hex[i] : '' })); + } + if (bare) { + add(row, el('code', { class: 'a86-code-text a86-code-comment', text: p.comment })); + } else { + add(row, el('code', { class: 'a86-code-text', text: p.code })); + if (p.comment.length) add(row, el('span', { class: 'a86-code-comment', text: p.comment })); + } + body.appendChild(row); + rows.push({ row, count, marker, line: p.line }); + } + + const maxHits = Math.max(1, ...trace.hits); + let lastRow = null; + + function update(frame) { + let current = null; + for (const r of rows) { + const idx = r.line.instrIndex; + r.row.classList.remove('is-next', 'is-last'); + if (idx == null) { r.count.textContent = ''; continue; } + if (frame.next && frame.next.index === idx) { r.row.classList.add('is-next'); current = r.row; } + if (frame.last && frame.last.instr.index === idx) r.row.classList.add('is-last'); + r.marker.textContent = frame.next && frame.next.index === idx ? '\u25b8' : ''; + if (opts.showCounts) { + const hits = countHits(trace, frame.t, idx); + r.count.textContent = hits || ''; + if (opts.heat) r.row.style.setProperty('--heat', (hits / maxHits).toFixed(3)); + } + } + // keep the next instruction in view inside a scrolling listing + if (current && current !== lastRow && body.scrollHeight > body.clientHeight + 4) { + const top = current.offsetTop - body.offsetTop; + if (top < body.scrollTop || top > body.scrollTop + body.clientHeight - 24) body.scrollTop = Math.max(0, top - body.clientHeight / 3); + } + lastRow = current; + } + + const node = panel(opts.title ?? 'Program', body, 'a86-code-panel'); + // A listing whose instructions and comments together outrun a half-width + // column reads better across the whole row than with its comments wrapped + // onto second lines. An unannotated listing stays beside the state panels. + const machineWidth = machine ? (opts.addresses ? 7 : 0) + (opts.bytes ? widestBytes + 2 : 0) : 0; + const byCount = widest + widestComment + machineWidth > opts.wideAt; + if (byCount) node.classList.add('is-wide'); + if (opts.maxHeight) body.style.maxHeight = opts.maxHeight; + + // The character count above is a guess made before the widget has a size. + // Once it is on the page, measure: if the listing's natural width will not + // fit the left column of a two-column grid at the size actually rendered, + // give it the whole row. A page narrower than its nominal body width is + // common on laptops, and a clipped instruction is worse than a tall widget. + if (!byCount && typeof ResizeObserver !== 'undefined') { + const fit = () => { + const grid = node.parentElement; + if (!grid || !node.isConnected) return; + const cols = getComputedStyle(grid).gridTemplateColumns.split(' ').filter(Boolean); + if (cols.length < 2) { node.classList.remove('is-wide'); return; } + const first = parseFloat(cols[0]); + body.style.width = 'max-content'; + const need = body.offsetWidth; + body.style.width = ''; + const pad = node.offsetWidth - node.clientWidth + 24; + node.classList.toggle('is-wide', need + pad > first); + }; + // Measuring changes the layout, so do it on the next frame rather than + // inside the observer's own callback. + let pending = 0; + const ro = new ResizeObserver(() => { cancelAnimationFrame(pending); pending = requestAnimationFrame(fit); }); + const attach = () => { if (node.parentElement) ro.observe(node.parentElement); else requestAnimationFrame(attach); }; + requestAnimationFrame(attach); + } + return { node, update }; +} + +const hitCache = new WeakMap(); +function countHits(trace, t, instrIndex) { + let table = hitCache.get(trace); + if (!table) { + table = []; + const running = new Array(trace.program.instrs.length).fill(0); + table.push(running.slice()); + for (const s of trace.steps) { + running[s.instr.index]++; + table.push(running.slice()); + } + hitCache.set(trace, table); + } + return table[Math.min(t, table.length - 1)][instrIndex]; +} + +/* ------------------------------------------------------------------ */ +/* 2. Register file */ +/* ------------------------------------------------------------------ */ + +/** + * One card per register. Registers named in `watch` always show; the rest + * appear only when the program actually touches them, so a widget never + * shows fifteen boxes of zeros. + * + * options: { + * watch: ['rax','rcx'], always visible, in this order + * roles: { rdi: 'sequence', rsi: 'n' }, a label under the name + * widths: ['rax'], show the 64/32/16/8-bit views of these + * radix: 'both' | 'hex' | 'dec' + * } + */ +export function registerView(trace, options = {}) { + const opts = { radix: 'both', watch: [], roles: {}, widths: [], ...options }; + const used = new Set(opts.watch.map((r) => r.toLowerCase())); + for (const ins of trace.program.instrs) { + for (const o of ins.operands) { + if (o.kind === 'reg') used.add(REGS[o.name].base); + if (o.kind === 'mem') { for (const r of [o.base, o.index]) if (r && REGS[r]) used.add(REGS[r].base); } + } + } + for (const r of opts.watch) if (!REGS[r.toLowerCase()]) throw new AsmError(`registers view: no register named ${r}`); + const order = [...new Set(opts.watch.map((r) => REGS[r.toLowerCase()].base)) + .values()].filter((r) => !REGS[r].vector).concat(REG64.filter((r) => used.has(r) && !opts.watch.some((w) => REGS[w.toLowerCase()].base === r))); + + const body = el('div', { class: 'a86-regs' }); + const cards = order.map((name) => { + const value = el('span', { class: 'a86-reg-value' }); + const prev = el('span', { class: 'a86-reg-prev' }); + const role = opts.roles[name] ? el('span', { class: 'a86-reg-role', text: opts.roles[name] }) : null; + const widthStrip = opts.widths.includes(name) ? el('div', { class: 'a86-reg-widths' }) : null; + const card = el('div', { class: 'a86-reg' }, [ + el('span', { class: 'a86-reg-name', text: name }), + role, value, prev, widthStrip, + ]); + body.appendChild(card); + return { name, card, value, prev, widthStrip }; + }); + + function update(frame) { + const changed = new Map((frame.last?.diff.regs || []).map((d) => [d.name, d])); + for (const c of cards) { + const v = frame.state.regs[c.name]; + c.value.textContent = render(v, opts.radix); + const d = changed.get(c.name); + c.card.classList.toggle('is-changed', Boolean(d)); + c.prev.textContent = d ? `was ${render(d.before, opts.radix === 'both' ? 'dec' : opts.radix)}` : ''; + if (c.widthStrip) renderWidths(c.widthStrip, c.name, v); + } + } + + function render(v, radix) { + if (radix === 'hex') return formatHex(v); + if (radix === 'dec') return formatSigned(v); + return `${formatSigned(v)} ${formatHex(v)}`; + } + + function renderWidths(node, base, v) { + clear(node); + const names = namesOf(base); + const sizes = [8, 4, 2, 1, 1]; + names.forEach((n, k) => { + const high = REGS[n].high; + const masked = high ? formatHex((BigInt(v) >> 8n) & 0xffn, 1) : formatHex(v, sizes[k]); + add(node, el('div', { class: 'a86-width-row' }, [ + el('code', { class: 'a86-width-name', text: n }), + el('code', { class: 'a86-width-bits', text: masked }), + el('span', { class: 'a86-width-size', text: high ? 'bits 8\u201315' : `${sizes[k] * 8} bits` }), + ])); + }); + } + + return { node: panel(opts.title ?? 'Registers', body), update }; +} + +/* ------------------------------------------------------------------ */ +/* 3. Flags */ +/* ------------------------------------------------------------------ */ + +const FLAG_MEANING = { + ZF: 'the last result was zero', + SF: 'the last result had its top bit set', + CF: 'the last unsigned operation ran past the end of the register', + OF: 'the last signed operation gave the wrong sign', + PF: 'the low byte of the last result had an even number of one bits; after a floating-point compare, one side was NaN', +}; + +export function flagsView(trace, options = {}) { + const opts = { ...options }; + const body = el('div', { class: 'a86-flags' }); + const lamps = FLAG_NAMES.map((name) => { + const dot = el('span', { class: 'a86-flag-dot' }); + const val = el('span', { class: 'a86-flag-value' }); + const lamp = el('div', { class: 'a86-flag', title: FLAG_MEANING[name] }, [ + el('code', { class: 'a86-flag-name', text: name }), dot, val, + ]); + body.appendChild(lamp); + return { name, lamp, dot, val }; + }); + const note = el('p', { class: 'a86-flag-note' }); + const reading = el('p', { class: 'a86-flag-note a86-flag-reading' }); + + /** The flags a conditional instruction consults, from its condition code. */ + const readsOf = (instr) => (instr && OPS[instr.op].cond ? CONDITION_FLAGS[OPS[instr.op].cond] : instr && ['adc', 'sbb', 'cmc'].includes(instr.op) ? ['CF'] : []); + + function update(frame) { + const changed = new Set((frame.last?.diff.flags || []).map((d) => d.name)); + const readNow = new Set(readsOf(frame.last?.instr)); + const readNext = new Set(readsOf(frame.next)); + for (const l of lamps) { + const on = frame.state.flags[l.name]; + l.lamp.classList.toggle('is-on', on); + l.lamp.classList.toggle('is-changed', changed.has(l.name)); + l.lamp.classList.toggle('is-consulted', readNow.has(l.name)); + l.lamp.classList.toggle('is-wanted', readNext.has(l.name) && !readNow.has(l.name)); + l.val.textContent = on ? '1' : '0'; + } + const c = frame.last?.compared; + note.textContent = c + ? `${frame.last.instr.op} compared ${formatSigned(c.a, c.width)} against ${formatSigned(c.b, c.width)}` + : (frame.last && OPS[frame.last.instr.op].cls.flags ? 'the last instruction set the flags as a side effect' : ''); + const last = frame.last; + if (last && readNow.size) { + const what = last.branch?.kind === 'conditional' ? (last.branch.taken ? 'jumped' : 'fell through') + : last.conditionalMove ? (last.conditionalMove.took ? 'copied' : 'kept the old value') + : last.instr.op.startsWith('set') ? `stored ${CONDITIONS[OPS[last.instr.op].cond](frame.state.flags) ? 1 : 0}` : ''; + reading.textContent = `${last.instr.op} read ${[...readNow].join(' and ')}${what ? ` and ${what}` : ''}.`; + } else if (frame.next && readNext.size) { + reading.textContent = `Next, ${frame.next.op} will read ${[...readNext].join(' and ')}.`; + } else reading.textContent = ''; + } + + return { node: panel(opts.title ?? 'Flags', el('div', {}, [body, note, reading])), update }; +} + +/* ------------------------------------------------------------------ */ +/* 4. Memory strip */ +/* ------------------------------------------------------------------ */ + +/** + * Each memory region as a strip of bytes with its positions written + * underneath, in the style of the paper strip from lesson 1. + * + * options: { + * window: { start: 'r8', length: 'rcx', region: 'T' } shaded run + * marks: [ { label: '.hit', position: 'r8', className: 'match', region: 'T' } ] + * showAddresses: false, write 0x1000+ instead of 0,1,2... + * regions: ['T','P'] which to draw, default all + * } + */ +export function memoryView(trace, options = {}) { + const opts = { showAddresses: false, marks: [], ...options }; + const body = el('div', { class: 'a86-mem' }); + const built = []; + + const initial = trace.states[0].mem; + const wanted = opts.regions + ? initial.filter((r) => opts.regions.includes(r.name)) + : initial.filter((r) => r.kind === 'text' || r.bytes.length <= 64); + + for (const region of wanted) { + const strip = el('div', { class: 'a86-strip' }); + const cells = []; + for (let k = 0; k < region.bytes.length; k++) { + const letter = el('span', { class: 'a86-cell-letter' }); + const posn = el('span', { class: 'a86-cell-pos', text: opts.showAddresses ? formatHex(BigInt(region.addr + k), 2).slice(2) : String(k) }); + const cell = el('div', { class: 'a86-cell' }, [letter, posn]); + strip.appendChild(cell); + cells.push({ cell, letter, addr: region.addr + k, offset: k }); + } + const label = el('div', { class: 'a86-strip-label' }, [ + el('code', { text: region.label }), + el('span', { class: 'a86-strip-addr', text: formatHex(BigInt(region.addr), 4) }), + ]); + body.appendChild(el('div', { class: 'a86-strip-wrap' }, [label, strip])); + built.push({ region, cells }); + } + + function update(frame) { + const reads = new Set(); + const writes = new Set(); + for (const a of frame.last?.accesses || []) { + for (let k = 0; k < a.size; k++) (a.kind === 'read' ? reads : writes).add(a.addr + k); + } + + // Marks accumulate: scan every step so far for the declared label. + const marks = new Map(); + for (const m of opts.marks) { + for (let s = 0; s < frame.t; s++) { + const step = trace.steps[s]; + if (!step.instr.labelsHere.includes(m.label)) continue; + const pos = valueOf(trace.states[s], m.position); + if (pos != null) marks.set(`${m.region || wanted[0]?.name}:${Number(pos)}`, m.className || 'match'); + } + } + + for (const b of built) { + const bytes = frame.state.mem.find((r) => r.name === b.region.name)?.bytes || b.region.bytes; + let winStart = null; let winEnd = null; + if (opts.window && (opts.window.region || wanted[0]?.name) === b.region.name) { + const s = valueOf(frame.state, opts.window.start); + const l = valueOf(frame.state, opts.window.length); + if (s != null) { winStart = Number(s); winEnd = winStart + Number(l ?? 1n); } + } + for (const c of b.cells) { + c.letter.textContent = byteChar(bytes[c.offset]); + c.cell.classList.toggle('is-read', reads.has(c.addr)); + c.cell.classList.toggle('is-write', writes.has(c.addr)); + c.cell.classList.toggle('in-window', winStart != null && c.offset >= winStart && c.offset < winEnd); + const mark = marks.get(`${b.region.name}:${c.offset}`); + c.cell.classList.toggle('is-match', mark === 'match'); + } + } + } + + return { node: panel(opts.title ?? 'Memory', body, 'a86-mem-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 5. Effective address */ +/* ------------------------------------------------------------------ */ + +/** base + index*scale + displacement, with every part shown separately. */ +export function addressView(trace, options = {}) { + const body = el('div', { class: 'a86-addr' }); + + function update(frame) { + clear(body); + const ea = frame.last?.effective; + if (!ea) { + add(body, el('p', { class: 'a86-muted', text: 'The last instruction named no memory address.' })); + return; + } + const parts = []; + if (ea.base) parts.push({ role: 'base', expr: ea.base, value: ea.baseVal }); + if (ea.index) parts.push({ role: `index \u00d7 ${ea.scale}`, expr: `${ea.index} * ${ea.scale}`, value: ea.indexVal * BigInt(ea.scale) }); + if (ea.disp) parts.push({ role: 'displacement', expr: String(ea.disp), value: ea.disp }); + + const row = el('div', { class: 'a86-addr-sum' }); + parts.forEach((p, i) => { + if (i) add(row, el('span', { class: 'a86-addr-op', text: '+' })); + add(row, el('div', { class: 'a86-addr-part' }, [ + el('span', { class: 'a86-addr-role', text: p.role }), + el('code', { class: 'a86-addr-expr', text: p.expr }), + el('code', { class: 'a86-addr-val', text: formatHex(p.value, 4) }), + ])); + }); + add(row, el('span', { class: 'a86-addr-op', text: '=' })); + add(row, el('div', { class: 'a86-addr-part is-result' }, [ + el('span', { class: 'a86-addr-role', text: 'address' }), + el('code', { class: 'a86-addr-expr', text: ea.source }), + el('code', { class: 'a86-addr-val', text: formatHex(BigInt(ea.addr), 4) }), + ])); + add(body, row); + + const access = (frame.last.accesses || [])[0]; + const what = frame.last.addressOnly + ? 'lea stops here: it stores the address and reads no memory.' + : access + ? `${access.kind === 'read' ? 'Loaded' : 'Stored'} ${access.size} byte${access.size > 1 ? 's' : ''}: ${formatHex(access.value, access.size)}${access.size === 1 ? ` (${byteChar(Number(access.value))})` : ''}` + : ''; + add(body, el('p', { class: 'a86-addr-note', text: what })); + } + + return { node: panel(options.title ?? 'Address arithmetic', body), update }; +} + +/* ------------------------------------------------------------------ */ +/* 6. Data flow */ +/* ------------------------------------------------------------------ */ + +/** + * Where this instruction's values came from and where they went, drawn + * from the operand list rather than from any per-program table. + */ +export function dataflowView(trace, options = {}) { + const host = el('div', { class: 'a86-flow' }); + + function box(x, y, w, h, label, value, cls = '') { + return sv('g', { class: `a86-flow-box ${cls}` }, [ + sv('rect', { x, y, width: w, height: h, rx: 3 }), + sv('text', { x: x + 8, y: y + 17, class: 'a86-flow-label', text: label }), + sv('text', { x: x + 8, y: y + 36, class: 'a86-flow-value', text: value }), + ]); + } + + function arrow(x1, y1, x2, y2, cls = '') { + return sv('path', { class: `a86-flow-arrow ${cls}`, d: `M ${x1} ${y1} L ${x2} ${y2}`, 'marker-end': 'url(#a86-arrowhead)' }); + } + + function update(frame) { + clear(host); + const step = frame.last; + if (!step) { + add(host, el('p', { class: 'a86-muted', text: 'Step forward to see values move.' })); + return; + } + const cls = OPS[step.instr.op].cls; + const sources = []; + const dest = step.instr.operands[0]; + + step.instr.operands.forEach((o, i) => { + const isDestOnly = i === 0 && (cls.move || cls.write) && !cls.alu && !cls.compare; + if (isDestOnly) return; + if (o.kind === 'reg') sources.push({ label: o.name, value: formatSigned(regValue(trace.states[frame.t - 1], o.name), REGS[o.name].size) }); + else if (o.kind === 'imm') sources.push({ label: 'constant', value: formatSigned(o.value, 8) }); + else if (o.kind === 'mem') { + const a = (step.accesses || []).find((x) => x.kind === 'read'); + sources.push({ label: o.source, value: a ? `${formatHex(a.value, a.size)}${a.size === 1 ? ` "${byteChar(Number(a.value))}"` : ''}` : 'address only' }); + } else if (o.kind === 'label') sources.push({ label: 'target', value: o.name }); + }); + + const W = 700; const H = 150; + const svgEl = sv('svg', { viewBox: `0 0 ${W} ${H}`, class: 'a86-flow-svg', role: 'img' }, [ + sv('defs', {}, [sv('marker', { id: 'a86-arrowhead', viewBox: '0 0 10 10', refX: 9, refY: 5, markerWidth: 6, markerHeight: 6, orient: 'auto-start-reverse' }, [sv('path', { d: 'M 0 0 L 10 5 L 0 10 z' })])]), + ]); + + const bw = 180; const bh = 46; + sources.slice(0, 2).forEach((s, i) => { + const y = sources.length > 1 ? 18 + i * 66 : 52; + add(svgEl, box(0, y, bw, bh, s.label, s.value, 'is-source')); + add(svgEl, arrow(bw + 4, y + bh / 2, 258, H / 2)); + }); + + const opName = step.instr.op; + add(svgEl, sv('g', { class: 'a86-flow-op' }, [ + sv('rect', { x: 262, y: 44, width: 150, height: 62, rx: 31 }), + sv('text', { x: 337, y: 70, class: 'a86-flow-opname', 'text-anchor': 'middle', text: opName }), + sv('text', { x: 337, y: 90, class: 'a86-flow-opkind', 'text-anchor': 'middle', text: cls.alu ? 'arithmetic unit' : cls.control ? 'branch unit' : 'copy' }), + ])); + + const outs = []; + if (step.diff.regs.length) { + for (const d of step.diff.regs) outs.push({ label: d.name, value: `${formatSigned(d.before)} \u2192 ${formatSigned(d.after)}`, cls: 'is-dest' }); + } + if (step.diff.mem.length) { + outs.push({ label: dest?.source || 'memory', value: step.diff.mem.map((m) => formatHex(BigInt(m.after), 1)).join(' '), cls: 'is-dest' }); + } + if (step.diff.flags.length) { + outs.push({ label: 'flags', value: step.diff.flags.map((f) => `${f.name}=${f.after ? 1 : 0}`).join(' '), cls: 'is-flags' }); + } + if (step.branch) { + outs.push({ label: 'next instruction', value: step.branch.taken ? `jump to ${step.branch.target}` : 'the following line', cls: 'is-control' }); + } + if (!outs.length) outs.push({ label: 'effect', value: 'nothing changed', cls: '' }); + + outs.slice(0, 3).forEach((o, i) => { + const y = outs.length === 1 ? 52 : 8 + i * 48; + add(svgEl, arrow(416, H / 2, 494, y + bh / 2)); + add(svgEl, box(498, y, 202, bh, o.label, o.value, o.cls)); + }); + + add(host, svgEl); + add(host, el('p', { class: 'a86-flow-gloss', text: `${opName}: ${OPS[opName].gloss}.` })); + } + + return { node: panel(options.title ?? 'What moved', host, 'a86-flow-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 7. Datapath */ +/* ------------------------------------------------------------------ */ + +const PATH_NODES = { + ip: { x: 8, y: 96, w: 96, h: 42, label: 'address of\nnext instruction' }, + fetch: { x: 122, y: 96, w: 80, h: 42, label: 'fetch' }, + decode: { x: 220, y: 96, w: 84, h: 42, label: 'decode' }, + regfile: { x: 322, y: 22, w: 106, h: 54, label: 'register file' }, + alu: { x: 322, y: 96, w: 106, h: 42, label: 'arithmetic unit' }, + agu: { x: 322, y: 158, w: 106, h: 42, label: 'address adder' }, + dcache: { x: 454, y: 158, w: 106, h: 42, label: 'data memory' }, + flags: { x: 454, y: 96, w: 106, h: 42, label: 'flags' }, + branch: { x: 584, y: 96, w: 100, h: 42, label: 'branch unit' }, + writeback: { x: 584, y: 22, w: 100, h: 54, label: 'write back' }, +}; + +const PATH_EDGES = [ + ['ip', 'fetch'], ['fetch', 'decode'], ['decode', 'regfile'], ['decode', 'alu'], + ['decode', 'agu'], ['regfile', 'alu'], ['agu', 'dcache'], ['alu', 'flags'], + ['dcache', 'writeback'], ['alu', 'writeback'], ['regfile', 'writeback'], + ['flags', 'branch'], ['branch', 'ip'], +]; + +/** + * Which parts of a simplified machine this instruction uses. The active set + * comes from the instruction's own class, so a new instruction added to + * interp.js lights the right boxes with no change here. + */ +export function classifyInstruction(instr) { + const cls = OPS[instr.op].cls; + const active = new Set(['ip', 'fetch', 'decode']); + const edges = new Set(['ip>fetch', 'fetch>decode']); + const touchesMem = instr.operands.some((o) => o.kind === 'mem'); + const readsReg = instr.operands.some((o) => o.kind === 'reg' || (o.kind === 'mem' && (o.base || o.index))); + + if (readsReg) { active.add('regfile'); edges.add('decode>regfile'); } + if (touchesMem || cls.address) { active.add('agu'); edges.add('decode>agu'); } + if (touchesMem && !cls.address) { active.add('dcache'); edges.add('agu>dcache'); } + if (cls.alu) { active.add('alu'); edges.add('decode>alu'); if (readsReg) edges.add('regfile>alu'); } + if (cls.flags) { active.add('flags'); edges.add('alu>flags'); } + if (cls.conditional) { active.add('branch'); active.add('flags'); edges.add('flags>branch'); } + if (cls.control) { active.add('branch'); edges.add('branch>ip'); } + if (cls.write) { + active.add('writeback'); + if (active.has('dcache')) edges.add('dcache>writeback'); + else if (cls.alu) edges.add('alu>writeback'); + else edges.add('regfile>writeback'); + } + return { active, edges }; +} + +/** A block diagram with only the parts this instruction uses lit up. */ +export function datapathView(trace, options = {}) { + const W = 708; const H = 222; + const svgEl = sv('svg', { viewBox: `0 0 ${W} ${H}`, class: 'a86-path-svg', role: 'img' }); + add(svgEl, sv('defs', {}, [sv('marker', { id: 'a86-pathhead', viewBox: '0 0 10 10', refX: 9, refY: 5, markerWidth: 5, markerHeight: 5, orient: 'auto-start-reverse' }, [sv('path', { d: 'M 0 0 L 10 5 L 0 10 z' })])])); + + const edgeEls = {}; + for (const [a, b] of PATH_EDGES) { + const p = PATH_NODES[a]; const q = PATH_NODES[b]; + const x1 = p.x + p.w; const y1 = p.y + p.h / 2; + const x2 = q.x; const y2 = q.y + q.h / 2; + const d = a === 'branch' && b === 'ip' + ? `M ${p.x + p.w / 2} ${p.y + p.h} L ${p.x + p.w / 2} ${H - 8} L ${PATH_NODES.ip.x + PATH_NODES.ip.w / 2} ${H - 8} L ${PATH_NODES.ip.x + PATH_NODES.ip.w / 2} ${PATH_NODES.ip.y + PATH_NODES.ip.h}` + : x2 >= x1 ? `M ${x1} ${y1} C ${x1 + 22} ${y1}, ${x2 - 22} ${y2}, ${x2} ${y2}` + : `M ${p.x + p.w / 2} ${p.y} L ${p.x + p.w / 2} ${q.y + q.h / 2} L ${x2} ${y2}`; + const path = sv('path', { class: 'a86-path-edge', d, 'marker-end': 'url(#a86-pathhead)' }); + edgeEls[`${a}>${b}`] = path; + add(svgEl, path); + } + + const nodeEls = {}; + for (const [key, n] of Object.entries(PATH_NODES)) { + const g = sv('g', { class: 'a86-path-node' }); + add(g, sv('rect', { x: n.x, y: n.y, width: n.w, height: n.h, rx: 4 })); + n.label.split('\n').forEach((line, i, all) => { + add(g, sv('text', { + x: n.x + n.w / 2, + y: n.y + n.h / 2 + 4 + (i - (all.length - 1) / 2) * 13, + 'text-anchor': 'middle', class: 'a86-path-label', text: line, + })); + }); + nodeEls[key] = g; + add(svgEl, g); + } + + const caption = el('p', { class: 'a86-path-caption' }); + + function update(frame) { + const instr = frame.last?.instr || frame.next; + const { active, edges } = instr ? classifyInstruction(instr) : { active: new Set(), edges: new Set() }; + for (const [k, g] of Object.entries(nodeEls)) g.classList.toggle('is-active', active.has(k)); + for (const [k, p] of Object.entries(edgeEls)) p.classList.toggle('is-active', edges.has(k)); + const parts = [...active].filter((k) => !['ip', 'fetch', 'decode'].includes(k)).map((k) => PATH_NODES[k].label.replace('\n', ' ')); + const wide = instr && OPS[instr.op].cls.vector ? ' Sixteen byte lanes move through it at once.' : ''; + caption.textContent = instr ? `${instr.op} uses ${parts.join(', ') || 'nothing past decode'}.${wide}` : ''; + } + + return { node: panel(options.title ?? 'Which parts of the machine ran', el('div', {}, [svgEl, caption]), 'a86-path-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 8. Plain-English explanation */ +/* ------------------------------------------------------------------ */ + +const FAULT_TEXT = { + 'page-fault': 'Page fault. The program touched an address no region maps. The processor stopped it and handed control to the operating system, and a segmentation fault on Linux is exactly this event.', + 'divide-error': 'Divide error. The processor refuses to divide by zero, or to produce a quotient too wide for its register, and raises an exception instead. Linux delivers it to the program as SIGFPE.', + 'bad-return': 'Bad return. ret takes whatever sits at the top of the stack as the address to go back to. The value on top here was not the address of any instruction.', + alignment: 'General protection fault. An aligned instruction was handed an address that is not a multiple of 16.', + 'stack-overflow': 'Stack overflow. Every call and push moved rsp further down, until it left the stack\u2019s region entirely. The access below the region is a page fault like any other; the operating system reports it as a crash because nothing is mapped there.', +}; + +/** One sentence naming what just happened, built from the recorded diff. */ +export function explainView(trace, options = {}) { + const body = el('div', { class: 'a86-explain' }); + + function update(frame) { + clear(body); + if (!frame.last) { + add(body, el('p', {}, [ + 'Nothing has run yet. Next: ', + el('code', { text: frame.next ? frame.next.text : '(end)' }), + frame.next ? ` \u2014 ${OPS[frame.next.op].gloss}.` : '', + ])); + return; + } + const s = frame.last; + const bits = []; + if (s.syscall) { + const sc = s.syscall; + const shown = sc.name === 'write' ? `write(${sc.fd}, ${formatHex(sc.args[1], 4)}, ${sc.args[2]})` : sc.name === 'exit' || sc.name === 'exit_group' ? `${sc.name}(${sc.args[0]})` : `${sc.name}(\u2026)`; + bits.push(`the kernel ran ${shown}${sc.ret !== null ? ` and returned ${formatSigned(BigInt.asUintN(64, sc.ret))}` : ''}`); + if (sc.text && sc.name === 'write') bits.push(`the terminal printed ${JSON.stringify(sc.text)}`); + bits.push('syscall overwrote rcx and r11 on the way in, as it always does'); + } + for (const d of s.diff.regs) { + if (s.syscall && (d.name === 'rcx' || d.name === 'r11')) continue; + bits.push(`${d.name} became ${d.vector ? formatHex(d.after, 16) : formatSigned(d.after)}`); + } + const memBytes = s.diff.mem; + if (memBytes.length > 4) bits.push(`${memBytes.length} bytes of memory from ${formatHex(BigInt(memBytes[0].addr), 4)} changed`); + else for (const m of memBytes) bits.push(`the byte at ${formatHex(BigInt(m.addr), 4)} became ${formatHex(BigInt(m.after), 1)}`); + if (s.diff.flags.length) bits.push(`the flags moved to ${s.diff.flags.map((f) => `${f.name}=${f.after ? 1 : 0}`).join(', ')}`); + if (s.branch?.kind === 'conditional') { + bits.push(s.branch.taken + ? `the condition held, so execution jumped to ${s.branch.target}` + : 'the condition failed, so execution carried on to the next line'); + } + if (s.branch?.kind === 'unconditional') bits.push(`execution jumped to ${s.branch.target}`); + if (s.branch?.kind === 'call') bits.push(`the return address went onto the stack and execution jumped to ${s.branch.target}`); + if (s.branch?.kind === 'return') bits.push(`execution went back to ${formatHex(s.branch.address, 4)}, the address the call had left on the stack`); + if (!bits.length) bits.push('nothing observable changed'); + + add(body, el('p', {}, [ + el('code', { class: 'a86-explain-instr', text: s.instr.text }), + ` \u2014 ${OPS[s.instr.op].gloss}. `, + el('span', { class: 'a86-explain-effect', text: `${bits.join('; ')}.` }), + ])); + const fault = trace.stats.fault; + if (frame.next) { + add(body, el('p', { class: 'a86-muted' }, ['Next: ', el('code', { text: frame.next.text }), '. What will it do?'])); + } else if (fault && frame.t === trace.count) { + add(body, el('p', { class: 'a86-explain-fault' }, [ + el('strong', { text: 'The next instruction, ' }), + el('code', { text: trace.program.instrs[fault.instr].text }), + el('strong', { text: ', faulted. ' }), + `${FAULT_TEXT[fault.kind] || fault.message}`, + ])); + } else if (frame.state.exitCode != null) { + add(body, el('p', { class: 'a86-muted', text: `The program asked the kernel to end it, with exit code ${frame.state.exitCode}.` })); + } else { + add(body, el('p', { class: 'a86-muted', text: 'The program has stopped.' })); + } + } + + return { node: panel(options.title ?? 'What just happened', body), update }; +} + +/* ------------------------------------------------------------------ */ +/* 9. Counters */ +/* ------------------------------------------------------------------ */ + +/** + * Running tallies up to the current step. `only` picks which to show. + * + * From the trace: instructions, comparisons, reads, writes, branches, + * taken, mispredicts, syscalls. + * From the hardware models (computed only when asked for): cycles (the + * out-of-order core), cycles-in-order (the five-stage pipeline), uops, + * l1d-misses, l1i-misses, l2-misses, memory (accesses that reached DRAM), + * tlb-misses. + */ +export function countersView(trace, options = {}) { + const opts = { only: null, ...options }; + const body = el('div', { class: 'a86-counters' }); + const defs = [ + { key: 'instructions', label: 'instructions run' }, + { key: 'comparisons', label: 'comparisons' }, + { key: 'reads', label: 'memory loads' }, + { key: 'writes', label: 'memory stores' }, + { key: 'branches', label: 'conditional branches' }, + { key: 'taken', label: 'branches taken' }, + { key: 'mispredicts', label: 'mispredicted' }, + { key: 'syscalls', label: 'system calls' }, + { key: 'uops', label: 'micro-ops', hw: true }, + { key: 'cycles', label: 'cycles, out-of-order core', hw: true }, + { key: 'cycles-in-order', label: 'cycles, five-stage pipeline', hw: true }, + { key: 'l1d-misses', label: 'L1d misses', hw: true }, + { key: 'l1i-misses', label: 'L1i misses', hw: true }, + { key: 'l2-misses', label: 'L2 misses', hw: true }, + { key: 'memory', label: 'trips to memory', hw: true }, + { key: 'tlb-misses', label: 'TLB misses', hw: true }, + ].filter((d) => (opts.only ? opts.only.includes(d.key) : !d.hw && d.key !== 'syscalls')); + if (opts.only) defs.sort((a, b) => opts.only.indexOf(a.key) - opts.only.indexOf(b.key)); + + const cards = defs.map((d) => { + const value = el('span', { class: 'a86-counter-value', text: '0' }); + body.appendChild(el('div', { class: 'a86-counter' }, [value, el('span', { class: 'a86-counter-label', text: d.label })])); + return { ...d, value }; + }); + + const running = buildRunning(trace); + const hw = defs.some((d) => d.hw) ? hardwareFor(trace) : null; + + function hwValue(key, t) { + if (!t) return 0; + const k = t - 1; + if (key === 'cycles') { const c = hw.core(); return (c.insts[k]?.retire ?? 0) + 1; } + if (key === 'cycles-in-order') return hw.pipeline().rows[k].E.WB + 1; + if (key === 'uops') { const c = hw.core(); let n = 0; for (let j = 0; j <= k; j++) if (!c.insts[j].fusedInto) n += c.insts[j].uops.length; return n; } + const m = hw.memory().running[t]; + return { 'l1d-misses': m.l1dMisses, 'l1i-misses': m.l1iMisses, 'l2-misses': m.l2Misses, memory: m.memory, 'tlb-misses': m.tlbMisses }[key]; + } + + function update(frame) { + const r = running[Math.min(frame.t, running.length - 1)]; + for (const c of cards) c.value.textContent = String(c.hw ? hwValue(c.key, frame.t) : (r[c.key] ?? 0)); + } + + return { node: panel(opts.title ?? 'Cost so far', body, 'a86-counters-panel'), update }; +} + +function buildRunning(trace) { + const rows = []; + const acc = { instructions: 0, comparisons: 0, reads: 0, writes: 0, branches: 0, taken: 0, mispredicts: 0, syscalls: 0 }; + rows.push({ ...acc }); + for (const s of trace.steps) { + acc.instructions++; + if (s.syscall) acc.syscalls++; + if (OPS[s.instr.op].cls.compare) acc.comparisons++; + for (const a of s.accesses) { if (a.kind === 'read') acc.reads++; else acc.writes++; } + if (s.branch?.kind === 'conditional') { + acc.branches++; + if (s.branch.taken) acc.taken++; + if (s.prediction && !s.prediction.correct) acc.mispredicts++; + } + rows.push({ ...acc }); + } + return rows; +} + +/* ------------------------------------------------------------------ */ +/* 10. Branch report */ +/* ------------------------------------------------------------------ */ + +/** One row per conditional branch in the program, with its prediction record. */ +export function branchView(trace, options = {}) { + const body = el('div', { class: 'a86-branch' }); + const table = el('table', { class: 'a86-table' }); + const head = el('thead', {}, [el('tr', {}, [ + el('th', { text: 'branch' }), el('th', { text: 'reached' }), el('th', { text: 'taken' }), + el('th', { text: 'mispredicted' }), el('th', { text: 'rate' }), + ])]); + const tbody = el('tbody'); + add(table, [head, tbody]); + add(body, table); + const note = el('p', { class: 'a86-muted', text: `Predictor: ${trace.spec.predictor || 'two-bit'} saturating counter, one per branch.` }); + add(body, note); + + const sites = [...new Set(trace.steps.filter((s) => s.branch?.kind === 'conditional').map((s) => s.instr.index))]; + + function update(frame) { + clear(tbody); + for (const idx of sites) { + let seen = 0; let taken = 0; let miss = 0; + for (let k = 0; k < frame.t; k++) { + const s = trace.steps[k]; + if (s.instr.index !== idx || s.branch?.kind !== 'conditional') continue; + seen++; + if (s.branch.taken) taken++; + if (s.prediction && !s.prediction.correct) miss++; + } + const instr = trace.program.instrs[idx]; + add(tbody, el('tr', { class: seen && miss / seen > 0.1 ? 'is-costly' : '' }, [ + el('td', {}, [el('code', { text: instr.text })]), + el('td', { text: String(seen) }), + el('td', { text: String(taken) }), + el('td', { text: String(miss) }), + el('td', { text: seen ? `${((miss / seen) * 100).toFixed(1)}%` : '\u2014' }), + ])); + } + } + + return { node: panel(options.title ?? 'Branch record', body), update }; +} + +/* ------------------------------------------------------------------ */ +/* 11. Trace table */ +/* ------------------------------------------------------------------ */ + +/** + * A table that grows one row at a time, in the style of the hand-traced + * tables in lessons 1 and 3. + * + * options: { + * rowsAt: '.scan' a row each time this label is reached + * | { matches: 'cmp r8b' } a row each time this instruction runs + * rowValues: 'before' | 'after' read state before or after that + * instruction, default 'before' + * columns: [ + * { label: 'i', reg: 'rcx' }, + * { label: 'T[i]', byteAt: { base: 'rdi', index: 'rcx' } }, + * { label: 'is it b?', compare: [{ byteAt: { base: 'rdi', index: 'rcx' } }, { reg: 'dl' }] }, + * { label: 'ZF', flag: 'ZF' }, + * { label: 'total', reg: 'rax' } + * ] + * } + */ +export function traceTableView(trace, options = {}) { + const opts = { columns: [], rowsAt: null, rowValues: 'before', maxRows: 60, ...options }; + const table = el('table', { class: 'a86-table a86-trace' }); + const head = el('thead', {}, [el('tr', {}, [el('th', { text: 'step' }), ...opts.columns.map((c) => el('th', { text: c.label }))])]); + const tbody = el('tbody'); + add(table, [head, tbody]); + const wrap = el('div', { class: 'a86-trace-wrap' }, [table]); + + const wantLabel = typeof opts.rowsAt === 'string' && opts.rowsAt.startsWith('.') ? opts.rowsAt : opts.rowsAt?.label; + const wantText = typeof opts.rowsAt === 'string' && !opts.rowsAt.startsWith('.') ? opts.rowsAt : opts.rowsAt?.matches; + + const normalize = (x) => String(x).replace(/\s+/g, ' ').trim().toLowerCase(); + + function isRowStep(s) { + if (!opts.rowsAt) return true; + if (wantLabel) return s.instr.labelsHere.includes(wantLabel); + if (wantText) return normalize(s.instr.text).includes(normalize(wantText)); + return false; + } + + const rowSteps = []; + for (let k = 0; k < trace.steps.length; k++) if (isRowStep(trace.steps[k])) rowSteps.push(k); + + /** The snapshot a row reads from: before that instruction, or after it. */ + function stateForRow(k) { + return opts.rowValues === 'after' ? trace.states[Math.min(k + 1, trace.states.length - 1)] : trace.states[k]; + } + + function byteValue(state, spec) { + const base = valueOf(state, spec.base) ?? 0n; + const index = valueOf(state, spec.index) ?? 0n; + const disp = BigInt(spec.disp ?? 0); + try { return readByte(state, Number(base + index * BigInt(spec.scale ?? 1) + disp)); } catch { return null; } + } + + function operandValue(state, spec) { + if (spec.reg) return regValue(state, spec.reg) & 0xffn; + if (spec.byteAt) { const b = byteValue(state, spec.byteAt); return b == null ? null : BigInt(b); } + if (spec.value != null) return BigInt(spec.value); + return null; + } + + function cellValue(state, col) { + if (col.reg) return formatSigned(regValue(state, col.reg), REGS[col.reg.toLowerCase()].size); + if (col.flag) return state.flags[col.flag] ? '1' : '0'; + if (col.yesNo) return state.flags[col.yesNo] ? 'yes' : 'no'; + if (col.hex) return formatHex(regValue(state, col.hex)); + if (col.byteAt) { const b = byteValue(state, col.byteAt); return b == null ? '\u2014' : byteChar(b); } + if (col.compare) { + const a = operandValue(state, col.compare[0]); + const b = operandValue(state, col.compare[1]); + if (a == null || b == null) return '\u2014'; + return a === b ? 'yes' : 'no'; + } + return ''; + } + + function update(frame) { + clear(tbody); + const visible = rowSteps.filter((k) => k < frame.t).slice(-opts.maxRows); + for (const k of visible) { + const state = stateForRow(k); + add(tbody, el('tr', { class: k === frame.t - 1 ? 'is-current' : '' }, [ + el('td', { class: 'a86-trace-step', text: String(k) }), + ...opts.columns.map((c) => el('td', {}, [el('code', { text: cellValue(state, c) })])), + ])); + } + if (!visible.length) add(tbody, el('tr', {}, [el('td', { colspan: String(opts.columns.length + 1), class: 'a86-muted', text: 'Step forward to fill this in.' })])); + wrap.scrollTop = wrap.scrollHeight; + } + + return { node: panel(opts.title ?? 'Trace', wrap), update }; +} + + +/* ------------------------------------------------------------------ */ +/* 12. Vector registers */ +/* ------------------------------------------------------------------ */ + +/** + * A 128-bit register drawn as what it is: sixteen byte lanes side by side. + * Lane 0 sits on the left, which is also the lowest bit of a pmovmskb + * result, and the mask row makes that correspondence visible. + * + * options: { + * watch: ['xmm0', 'xmm1'], default: every vector register the program touches + * roles: { xmm1: 'sixteen copies of b' }, + * maskFrom: 'r8d' draw this register's low sixteen bits under the lanes + * } + */ +export function vectorView(trace, options = {}) { + const opts = { roles: {}, maskFrom: null, ...options }; + const touched = new Set(opts.watch || []); + if (!opts.watch) { + for (const ins of trace.program.instrs) { + for (const o of ins.operands) { + if (o.kind === 'reg' && REGS[o.name].vector) touched.add(REGS[o.name].base); + } + } + } + const order = [...touched]; + + const body = el('div', { class: 'a86-vecs' }); + const rows = order.map((name) => { + const strip = el('div', { class: 'a86-vec-strip' }); + const cells = []; + for (let k = 0; k < 16; k++) { + const val = el('span', { class: 'a86-vec-val' }); + const idx = el('span', { class: 'a86-vec-lane', text: String(k) }); + const cell = el('div', { class: 'a86-vec-cell' }, [val, idx]); + strip.appendChild(cell); + cells.push({ cell, val }); + } + const row = el('div', { class: 'a86-vec-row' }, [ + el('div', { class: 'a86-vec-label' }, [ + el('code', { text: name }), + opts.roles[name] ? el('span', { class: 'a86-vec-role', text: opts.roles[name] }) : null, + ]), + strip, + ]); + body.appendChild(row); + return { name, row, cells }; + }); + + let maskRow = null; + if (opts.maskFrom) { + const strip = el('div', { class: 'a86-vec-strip' }); + const cells = []; + for (let k = 0; k < 16; k++) { + const val = el('span', { class: 'a86-vec-val' }); + const cell = el('div', { class: 'a86-vec-cell is-bit' }, [val, el('span', { class: 'a86-vec-lane', text: String(k) })]); + strip.appendChild(cell); + cells.push({ cell, val }); + } + maskRow = { + cells, node: el('div', { class: 'a86-vec-row' }, [ + el('div', { class: 'a86-vec-label' }, [el('code', { text: opts.maskFrom }), el('span', { class: 'a86-vec-role', text: 'bit per lane' })]), + strip, + ]) + }; + body.appendChild(maskRow.node); + } + + const note = el('p', { class: 'a86-vec-note' }); + add(body, note); + + function render(cellSet, bytes, changed) { + bytes.forEach((b, k) => { + const c = cellSet[k]; + c.cell.classList.remove('is-yes', 'is-no', 'is-letter'); + if (b === 0xff) { c.val.textContent = '1'; c.cell.classList.add('is-yes'); } + else if (b === 0x00) { c.val.textContent = '0'; c.cell.classList.add('is-no'); } + else { c.val.textContent = byteChar(b); c.cell.classList.add('is-letter'); } + c.cell.classList.toggle('is-changed', changed); + }); + } + + function update(frame) { + const changed = new Set((frame.last?.diff.regs || []).filter((d) => d.vector).map((d) => d.name)); + for (const r of rows) { + render(r.cells, lanes(frame.state.regs[r.name]), changed.has(r.name)); + } + if (maskRow) { + const v = regValue(frame.state, opts.maskFrom); + const bits = []; + for (let k = 0; k < 16; k++) bits.push(Number((v >> BigInt(k)) & 1n) ? 0xff : 0x00); + render(maskRow.cells, bits, false); + } + const vc = frame.last?.vectorCompare; + note.textContent = vc + ? 'One instruction just compared sixteen pairs of bytes. Each lane holds its own verdict.' + : (frame.last && OPS[frame.last.instr.op].cls.vector ? `${frame.last.instr.op} worked on all sixteen lanes at once.` : ''); + } + + return { node: panel(opts.title ?? 'Vector registers, sixteen lanes each', body, 'a86-vec-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* 13. Predictor comparison */ +/* ------------------------------------------------------------------ */ + +/** + * The same program run under several branch predictors, side by side. This + * view re-runs the spec once per predictor when it is built, so it shows + * whole-program totals rather than anything step-dependent. + * + * options: { kinds: ['always-taken', 'one-bit', 'two-bit'] } + */ +export function predictorsView(trace, options = {}) { + const opts = { kinds: ['always-taken', 'one-bit', 'two-bit'], ...options }; + const body = el('div', {}); + const table = el('table', { class: 'a86-table' }); + add(table, el('thead', {}, [el('tr', {}, [ + el('th', { text: 'predictor' }), el('th', { text: 'branches' }), + el('th', { text: 'mispredicted' }), el('th', { text: 'rate' }), + ])])); + const tbody = el('tbody'); + add(table, tbody); + add(body, table); + + const runs = opts.kinds.map((kind) => { + try { + const t = run({ ...trace.spec, predictor: kind }); + return { kind, branches: t.stats.branches, mispredicts: t.stats.mispredicts, sites: t.stats.branchSites }; + } catch (e) { + return { kind, error: e.message }; + } + }); + + for (const r of runs) { + if (r.error) { add(tbody, el('tr', {}, [el('td', { colspan: '4', class: 'a86-error', text: `${r.kind}: ${r.error}` })])); continue; } + const rate = r.branches ? (r.mispredicts / r.branches) * 100 : 0; + add(tbody, el('tr', { class: rate > 10 ? 'is-costly' : '' }, [ + el('td', {}, [el('code', { text: r.kind })]), + el('td', { text: String(r.branches) }), + el('td', { text: String(r.mispredicts) }), + el('td', { text: `${rate.toFixed(1)}%` }), + ])); + } + + const best = runs.filter((r) => !r.error).sort((a, b) => a.mispredicts - b.mispredicts)[0]; + add(body, el('p', { + class: 'a86-muted', text: best + ? `Cheapest here: ${best.kind}, missing ${best.mispredicts} of ${best.branches} branches. A predictor that guesses well on one program can guess badly on another, which is why this table changes with the sequence.` + : '' + })); + + return { node: panel(opts.title ?? 'The same program, three predictors', body, 'a86-span-2'), update() { } }; +} + +/* ------------------------------------------------------------------ */ +/* Stepper and composer */ +/* ------------------------------------------------------------------ */ + +const VIEW_REGISTRY = { + code: codeView, + registers: registerView, + flags: flagsView, + memory: memoryView, + address: addressView, + dataflow: dataflowView, + datapath: datapathView, + explain: explainView, + counters: countersView, + branches: branchView, + trace: traceTableView, + vectors: vectorView, + predictors: predictorsView, + ...HW_VIEWS, +}; + +export function registerViewType(name, factory) { VIEW_REGISTRY[name] = factory; } + +/** Every view name a spec can use, for the sandbox and the manual. */ +export function viewNames() { return Object.keys(VIEW_REGISTRY); } + +/** A widget-shaped box for a program that would not assemble. */ +function assemblyFailure(config, err) { + const root = el('div', { class: `a86 a86-${config.layout || 'one-column'}` }); + if (config.title) add(root, el('h3', { class: 'a86-title', text: config.title })); + add(root, el('p', { class: 'a86-error', text: err.message })); + const lines = String(config.code || '').replace(/\r/g, '').split('\n'); + const listing = el('div', { class: 'a86-code a86-code-failed' }); + lines.forEach((l, i) => { + add(listing, el('div', { class: `a86-code-row${i === err.line ? ' is-error' : ''}` }, [ + el('span', { class: 'a86-code-count', text: String(i + 1) }), + el('span', { class: 'a86-code-marker', text: i === err.line ? '\u2716' : '' }), + el('code', { class: 'a86-code-text', text: l }), + ])); + }); + add(root, panel('Program', listing, 'a86-code-panel is-wide')); + return root; +} + +/** + * Build a whole widget from one dictionary. + * + * mountWidget(node, { + * code: '...assembly...', + * registers: { rdi: 0x1000, rsi: 7, rdx: 65 }, + * memory: [{ name: 'T', label: 'T', addr: 0x1000, text: 'GATTACA' }], + * views: [ + * 'code', + * ['registers', { watch: ['rax','rcx'], roles: { rax: 'total', rcx: 'i' } }], + * 'memory', 'explain' + * ], + * layout: 'two-column', + * start: 0, + * speed: 300 + * }); + * + * Returns { node, trace, goto, step, play, pause, reset, onChange, hardware }. + * A program that will not assemble returns the same shape, with `error` + * set and a listing that marks the offending line. + */ +export function mountWidget(target, spec, extra = {}) { + const config = { views: ['code', 'registers', 'explain'], layout: 'two-column', speed: 280, start: 0, ...spec, ...extra }; + let trace; + try { + trace = config.trace || run(config); + } catch (err) { + // A line the assembler refuses is a lesson, not a crash: show where. + const root = assemblyFailure(config, err); + if (target) { clear(target); target.appendChild(root); } + const noop = () => { }; + return { node: root, trace: null, error: err, goto: noop, play: noop, pause: noop, fullscreen: noop, step: noop, reset: noop, t: 0, onChange: noop, hardware: null }; + } + + const root = el('div', { class: `a86 a86-${config.layout}` }); + if (config.title) add(root, el('h3', { class: 'a86-title', text: config.title })); + if (config.blurb) add(root, el('p', { class: 'a86-blurb', text: config.blurb })); + + const listeners = []; + let t = Math.max(0, Math.min(config.start, trace.count)); + let timer = null; + + /* controls */ + const slider = el('input', { type: 'range', min: '0', max: String(trace.count), value: String(t), class: 'a86-slider', 'aria-label': 'step' }); + const readout = el('span', { class: 'a86-readout' }); + const playBtn = el('button', { class: 'a86-btn', type: 'button', text: 'Play' }); + const fullBtn = el('button', { + class: 'a86-btn a86-btn-full', type: 'button', text: 'Fullscreen', + title: 'Open this widget at full window size (Esc to close)', + 'aria-pressed': 'false', + }); + const bar = el('div', { class: 'a86-controls' }, [ + el('button', { class: 'a86-btn', type: 'button', text: 'Reset', onclick: () => { pause(); goto(0); } }), + el('button', { class: 'a86-btn', type: 'button', text: 'Back', onclick: () => { pause(); goto(t - 1); } }), + playBtn, + el('button', { class: 'a86-btn a86-btn-primary', type: 'button', text: 'Step', onclick: () => { pause(); goto(t + 1); } }), + el('button', { class: 'a86-btn', type: 'button', text: 'End', onclick: () => { pause(); goto(trace.count); } }), + slider, readout, fullBtn, + ]); + slider.addEventListener('input', () => { pause(); goto(Number(slider.value)); }); + playBtn.addEventListener('click', () => (timer ? pause() : play())); + fullBtn.addEventListener('click', () => setFullscreen(!full)); + add(root, bar); + + /** + * Fullscreen is a fixed overlay rather than the browser's own fullscreen + * mode, so it behaves the same everywhere and needs no permission. The + * page body stops scrolling underneath while it is open. + */ + let full = false; + function setFullscreen(on) { + full = Boolean(on); + root.classList.toggle('is-fullscreen', full); + fullBtn.textContent = full ? 'Close' : 'Fullscreen'; + fullBtn.setAttribute('aria-pressed', full ? 'true' : 'false'); + const body = typeof document !== 'undefined' ? document.body : null; + if (body && body.style) body.style.overflow = full ? 'hidden' : ''; + if (full) root.focus?.(); + } + + /* views */ + const grid = el('div', { class: 'a86-grid' }); + add(root, grid); + const views = []; + for (const entry of config.views) { + const [name, options] = Array.isArray(entry) ? entry : [entry, {}]; + const factory = VIEW_REGISTRY[name]; + if (!factory) { add(grid, el('p', { class: 'a86-error', text: `No view named "${name}".` })); continue; } + let v; + try { + v = factory(trace, { ...(config.viewOptions?.[name] || {}), ...options }); + } catch (err) { + add(grid, el('p', { class: 'a86-error', text: `The ${name} view could not be built: ${err.message}` })); + continue; + } + if (options?.span) v.node.classList.add(`a86-span-${options.span}`); + v.name = name; + views.push(v); + add(grid, v.node); + } + + const FAULT_NAME = { 'page-fault': 'Page fault', 'divide-error': 'Divide error', 'bad-return': 'Bad return address', alignment: 'Alignment fault', 'stack-overflow': 'Stack overflow' }; + if (trace.stats.error) { + const f = trace.stats.fault; + add(root, el('p', { class: 'a86-error' }, [ + f && FAULT_NAME[f.kind] ? el('strong', { text: `${FAULT_NAME[f.kind]} after step ${trace.count}. ` }) : null, + trace.stats.error, + ])); + } + if (trace.stats.truncated) add(root, el('p', { class: 'a86-error', text: `Stopped after ${trace.count} instructions. Raise maxSteps to see more.` })); + + function draw() { + const frame = frameAt(trace, t); + for (const v of views) { + try { v.update(frame); v.node.classList.remove('has-failed'); } catch (err) { + if (!v.node.classList.contains('has-failed')) { + v.node.classList.add('has-failed'); + add(v.node, el('p', { class: 'a86-error', text: `The ${v.name} view failed at step ${t}: ${err.message}` })); + console.error(err); // eslint-disable-line no-console + } + } + } + slider.value = String(t); + readout.textContent = `step ${t} of ${trace.count}`; + playBtn.textContent = timer ? 'Pause' : 'Play'; + for (const fn of listeners) fn(frame); + } + + function goto(next) { t = Math.max(0, Math.min(next, trace.count)); draw(); } + function play() { + if (timer) return; + if (t >= trace.count) t = 0; + timer = setInterval(() => (t >= trace.count ? pause() : goto(t + 1)), config.speed); + draw(); + } + function pause() { if (timer) { clearInterval(timer); timer = null; draw(); } } + + root.tabIndex = 0; + root.addEventListener('keydown', (e) => { + if (/^(INPUT|TEXTAREA|SELECT)$/.test(e.target.tagName) && e.target !== slider) return; + if (e.key === 'Home') { pause(); goto(0); e.preventDefault(); } + if (e.key === 'End') { pause(); goto(trace.count); e.preventDefault(); } + if (e.key === 'ArrowRight') { pause(); goto(t + 1); e.preventDefault(); } + if (e.key === 'ArrowLeft') { pause(); goto(t - 1); e.preventDefault(); } + if (e.key === ' ') { timer ? pause() : play(); e.preventDefault(); } + if (e.key === 'Escape' && full) { setFullscreen(false); e.preventDefault(); } + }); + + if (target) { clear(target); target.appendChild(root); } + draw(); + + return { + node: root, trace, goto, play, pause, + fullscreen: setFullscreen, + step: () => goto(t + 1), + reset: () => goto(0), + get t() { return t; }, + get hardware() { return hardwareFor(trace); }, + onChange: (fn) => { listeners.push(fn); fn(frameAt(trace, t)); }, + }; +} + +/** For Observable JS, where a cell returns its node rather than being handed one. */ +export function widget(spec, extra = {}) { + return mountWidget(null, spec, extra).node; +} + +export { run, assemble, fromObjdump, hardwareFor, PRESETS, resolveMachine }; diff --git a/src/engines/pseudo/analyze.js b/src/engines/pseudo/analyze.js new file mode 100644 index 0000000..60c8013 --- /dev/null +++ b/src/engines/pseudo/analyze.js @@ -0,0 +1,738 @@ +/** + * pseudo/analyze.js + * + * What a program does with its data, worked out without running it. + * + * analyze(program) -> { + * units: [ unit ], one per subroutine, plus 'main' when it has lines + * } + * + * unit = { + * name, kind ('main' | 'Function' | 'Procedure'), node, body, + * cfg: { nodes, entry, exit }, control flow, one node per step + * defs, uses, every definition and every read + * reach: Map(useKey -> [def]), which definitions each read can see + * loops: [ loopInfo ], nesting, what each loop changes and tests + * graph: { nodes, edges }, which values feed which + * roles: Map(line -> role), the four jobs: setup, visit, test, update, report + * accumulators: [ ... ], names that collect a result across passes + * pattern, 'count', 'find-all', ... or null + * work, how many times the innermost lines run, in symbols + * } + */ + +import { sourceOf } from './parse.js'; +import { namesIn } from './interp.js'; + +const up = (s) => String(s).toUpperCase(); + +/* ------------------------------------------------------------------ */ +/* Walking the tree */ +/* ------------------------------------------------------------------ */ + +/** The statement lists nested directly inside a statement. */ +export function childBlocks(s) { + switch (s.kind) { + case 'If': return [...s.branches.map((b) => b.body), ...(s.elseBody ? [s.elseBody] : [])]; + case 'Case': return [...s.arms.map((a) => a.body), ...(s.otherwise ? [s.otherwise] : [])]; + case 'For': case 'ForEach': case 'While': case 'Repeat': return [s.body]; + case 'Function': case 'Procedure': return [s.body]; + default: return []; + } +} + +/** Visit every statement, with the chain of enclosing statements. */ +export function walkStatements(body, fn, parents = []) { + for (const s of body) { + if (fn(s, parents) === false) continue; + for (const b of childBlocks(s)) walkStatements(b, fn, [...parents, s]); + } +} + +/** The expressions a statement evaluates directly (not those of nested statements). */ +export function statementExprs(s) { + switch (s.kind) { + case 'Assign': return [s.value, ...targetExprs(s.target)]; + case 'Output': return s.values; + case 'Return': return s.value ? [s.value] : []; + case 'Call': case 'BuiltinCall': return s.args; + case 'ExprStmt': return [s.expr]; + case 'If': return s.branches.map((b) => b.cond); + case 'Case': return [s.subject, ...s.arms.flatMap((a) => a.labels.flatMap((l) => (l.value ? [l.value] : [l.low, l.high])))].filter(Boolean); + case 'For': return [s.start, s.stop, s.step].filter(Boolean); + case 'ForEach': return s.iterable ? [s.iterable] : []; + case 'While': case 'Repeat': return s.cond ? [s.cond] : []; + case 'Assert': case 'Invariant': case 'Decreases': case 'Require': case 'Ensure': return [s.expr]; + default: return []; + } +} + +/** Index and field expressions inside an assignment target, which are read. */ +function targetExprs(t) { + const out = []; + let x = t; + while (x && (x.kind === 'Index' || x.kind === 'Field')) { + if (x.kind === 'Index') out.push(...x.indices); + x = x.target; + } + return out; +} + +/** The variable at the root of a target: T for T[I].Name. */ +export function rootName(t) { + let x = t; + while (x && (x.kind === 'Index' || x.kind === 'Field')) x = x.target; + return x && x.kind === 'Name' ? x.name : null; +} + +/** Visit every expression node inside an expression. */ +export function walkExpr(e, fn, bound = new Set()) { + if (!e || typeof e !== 'object') return; + if (fn(e, bound) === false) return; + if (e.kind === 'Quant' || e.kind === 'SetBuilder') { + walkExpr(e.domain, fn, bound); + const b = new Set(bound); b.add(up(e.var)); + walkExpr(e.body, fn, b); + return; + } + for (const k of ['left', 'right', 'arg', 'target', 'lo', 'hi']) if (e[k]) walkExpr(e[k], fn, bound); + for (const k of ['operands', 'indices', 'args', 'items']) if (Array.isArray(e[k])) for (const x of e[k]) walkExpr(x, fn, bound); + if (Array.isArray(e.entries)) for (const x of e.entries) { walkExpr(x.key, fn, bound); walkExpr(x.value, fn, bound); } +} + +/* ------------------------------------------------------------------ */ +/* Control flow */ +/* ------------------------------------------------------------------ */ + +class CFG { + constructor() { this.nodes = []; } + node(kind, line, props = {}) { + const n = { id: this.nodes.length, kind, line, succ: [], pred: [], defs: [], uses: [], ...props }; + this.nodes.push(n); + return n; + } + link(a, b) { if (!a.succ.includes(b.id)) { a.succ.push(b.id); b.pred.push(a.id); } } +} + +/** + * A definition: { id, var, kind, node, line, stmt }. + * strong replaces the whole value (X <- ..., INPUT X, a counter moving) + * weak changes part of it (X[I] <- ..., APPEND(X, ...)) + * unset DECLARE of a name that starts with no value, or a counter after its loop + * init DECLARE of a collection, which starts empty + * param a parameter, set by the caller + */ +function addDef(cfg, n, name, kind, stmt) { + const d = { id: -1, var: up(name), name, kind, node: n.id, line: n.line, stmt }; + n.defs.push(d); + return d; +} + +function addUses(n, exprs, kind = 'use') { + for (const e of exprs) { + walkExpr(e, (x, bound) => { + if (x.kind === 'Name' && !bound.has(up(x.name)) && x.sym && ['var', 'param', 'counter', 'each'].includes(x.sym.kind)) { + n.uses.push({ var: up(x.name), name: x.name, kind, expr: x, line: x.line }); + } + }); + } +} + +function buildCFG(unit) { + const cfg = new CFG(); + const entry = cfg.node('entry', unit.node ? unit.node.line : 1); + const exit = cfg.node('exit', unit.node ? unit.node.endLine : null); + for (const p of unit.params || []) addDef(cfg, entry, p.name, 'param', null); + + const seq = (body, next, ctx) => { + let cur = next; + for (let i = body.length - 1; i >= 0; i--) cur = stmt(body[i], cur, ctx); + return cur; + }; + + const stmt = (s, next, ctx) => { + switch (s.kind) { + case 'If': { + let falseTarget = s.elseBody ? seq(s.elseBody, next, ctx) : next; + for (let i = s.branches.length - 1; i >= 0; i--) { + const b = s.branches[i]; + const t = cfg.node('test', b.line, { stmt: s, expr: b.cond, branch: i }); + addUses(t, [b.cond]); + cfg.link(t, seq(b.body, next, ctx)); + cfg.link(t, falseTarget); + falseTarget = t; + } + return falseTarget; + } + case 'Case': { + const t = cfg.node('select', s.line, { stmt: s }); + addUses(t, statementExprs(s)); + for (const a of s.arms) cfg.link(t, seq(a.body, next, ctx)); + cfg.link(t, s.otherwise ? seq(s.otherwise, next, ctx) : next); + return t; + } + case 'For': { + const init = cfg.node('for-init', s.line, { stmt: s }); + addUses(init, statementExprs(s)); + const counterName = s.var; + addDef(cfg, init, counterName, 'strong', s); + const head = cfg.node('for-head', s.line, { stmt: s, loop: s }); + head.uses.push({ var: up(counterName), name: counterName, kind: 'use', line: s.line, implicit: true }); + const nx = cfg.node('for-next', s.endLine ?? s.line, { stmt: s, loop: s }); + nx.uses.push({ var: up(counterName), name: counterName, kind: 'use', line: nx.line, implicit: true }); + addDef(cfg, nx, counterName, 'strong', s); + for (const inv of s.invariants || []) addUses(head, [inv.expr], 'spec'); + let after = next; + if (s.declaredCounter) { + const gone = cfg.node('loop-exit', s.endLine ?? s.line, { stmt: s }); + addDef(cfg, gone, counterName, 'unset', s); + cfg.link(gone, next); + after = gone; + } + cfg.link(init, head); + const body = seq(s.body, nx, { brk: after, cont: nx }); + cfg.link(head, body); + cfg.link(head, after); + cfg.link(nx, head); + return init; + } + case 'ForEach': { + const init = cfg.node('each-init', s.line, { stmt: s }); + addUses(init, statementExprs(s)); + const head = cfg.node('each-head', s.line, { stmt: s, loop: s }); + addDef(cfg, head, s.var, 'strong', s); + for (const inv of s.invariants || []) addUses(head, [inv.expr], 'spec'); + cfg.link(init, head); + const body = seq(s.body, head, { brk: next, cont: head }); + cfg.link(head, body); + cfg.link(head, next); + return init; + } + case 'While': { + const head = cfg.node('while-head', s.line, { stmt: s, loop: s, expr: s.cond }); + addUses(head, [s.cond]); + for (const inv of s.invariants || []) addUses(head, [inv.expr], 'spec'); + if (s.decreases) addUses(head, [s.decreases.expr], 'spec'); + const body = seq(s.body, head, { brk: next, cont: head }); + cfg.link(head, body); + cfg.link(head, next); + return head; + } + case 'Repeat': { + const until = cfg.node('until', s.endLine ?? s.line, { stmt: s, loop: s, expr: s.cond }); + if (s.cond) addUses(until, [s.cond]); + const top = cfg.node('repeat-head', s.line, { stmt: s, loop: s }); + for (const inv of s.invariants || []) addUses(top, [inv.expr], 'spec'); + const body = seq(s.body, until, { brk: next, cont: until }); + cfg.link(top, body); + cfg.link(until, next); + cfg.link(until, top); + return top; + } + case 'Break': { const n = cfg.node('break', s.line, { stmt: s }); cfg.link(n, ctx?.brk ? cfg.nodes[ctx.brk.id] : exit); return n; } + case 'Continue': { const n = cfg.node('continue', s.line, { stmt: s }); cfg.link(n, ctx?.cont ? cfg.nodes[ctx.cont.id] : exit); return n; } + case 'Return': { const n = cfg.node('return', s.line, { stmt: s }); addUses(n, statementExprs(s)); cfg.link(n, exit); return n; } + default: { + const n = cfg.node(s.kind.toLowerCase(), s.line, { stmt: s }); + simpleEffects(cfg, n, s); + cfg.link(n, next); + return n; + } + } + }; + + const first = seq(unit.body, exit, null); + cfg.link(entry, first); + // unit-level contracts read parameters + for (const r of unit.requires || []) addUses(entry, [r.expr], 'spec'); + for (const e of unit.ensures || []) addUses(exit, [e.expr], 'spec'); + return { cfg, entry, exit }; +} + +function simpleEffects(cfg, n, s) { + switch (s.kind) { + case 'Declare': + for (const nm of s.names) { + const k = s.resolved?.k; + // a collection starts empty, and an ARRAY or record exists with its cells unset + addDef(cfg, n, nm.name, ['LIST', 'HASHMAP', 'SET', 'ARRAY', 'RECORD'].includes(k) ? 'init' : 'unset', s); + } + return; + case 'Assign': { + addUses(n, statementExprs(s)); + const root = rootName(s.target); + if (!root) return; + if (s.target.kind === 'Name') addDef(cfg, n, root, 'strong', s); + else { + n.uses.push({ var: up(root), name: root, kind: 'use', line: s.line, partial: true }); + addDef(cfg, n, root, 'weak', s); + } + return; + } + case 'Input': { const root = rootName(s.target); if (root) addDef(cfg, n, root, s.target.kind === 'Name' ? 'strong' : 'weak', s); addUses(n, targetExprs(s.target)); return; } + case 'BuiltinCall': { + const [first, ...rest] = s.args; + addUses(n, rest); + addUses(n, targetExprs(first || {})); + const root = first ? rootName(first) : null; + if (root) { n.uses.push({ var: up(root), name: root, kind: 'use', line: s.line, partial: true }); addDef(cfg, n, root, 'weak', s); } + return; + } + case 'Call': { + const sub = s.sub; + s.args.forEach((a, i) => { + const p = sub?.params?.[i]; + if (p && p.mode === 'BYREF' && a.kind === 'Name') { + // A BYREF argument is read only if the callee reads the parameter + // before writing it; the coach checks that, so the use is tagged. + const before = n.uses.length; + addUses(n, [a]); + for (let k = before; k < n.uses.length; k++) Object.assign(n.uses[k], { byref: up(p.name), callee: sub.name }); + addDef(cfg, n, a.name, 'strong', s); + } else addUses(n, [a]); + }); + return; + } + case 'Assert': case 'Invariant': case 'Decreases': addUses(n, statementExprs(s), 'spec'); return; + default: addUses(n, statementExprs(s)); + } +} + +/* ------------------------------------------------------------------ */ +/* Reaching definitions */ +/* ------------------------------------------------------------------ */ + +/** + * The classic forward problem, solved to its least fixed point. A set of + * definitions is a bit set (one bit per definition, W words per node), and + * the nodes are visited in reverse postorder from the entry, so a pass + * carries a definition along a whole straight run of statements and the + * passes needed grow with loop nesting, not with the length of the program. + * Each bit is a problem of its own, so a very long program is solved a + * slice of definitions at a time, which bounds the memory the sets take. + */ +function reaching(cfg) { + const defs = []; + for (const n of cfg.nodes) for (const d of n.defs) { d.id = defs.length; defs.push(d); } + const byVar = new Map(); + for (const d of defs) { if (!byVar.has(d.var)) byVar.set(d.var, []); byVar.get(d.var).push(d.id); } + const N = cfg.nodes.length; + const order = reversePostorder(cfg); + const hits = cfg.nodes.map((n) => n.uses.map(() => [])); + const words = Math.max(1, (defs.length + 31) >>> 5); + const W = Math.max(1, Math.min(words, Math.floor((1 << 22) / Math.max(1, N)))); + const OUT = new Uint32Array(N * W); + const IN = new Uint32Array(N * W); + const next = new Uint32Array(W); + for (let lo = 0; lo < defs.length || lo === 0; lo += W * 32) { + const hi = lo + W * 32; + const inSlice = (id) => id >= lo && id < hi; + const has = (base, id) => (IN[base + ((id - lo) >>> 5)] & (1 << ((id - lo) & 31))) !== 0; + // A name defined on many lines (T <- T + 1 down a long routine) has its definitions in this slice as one mask, so + // replacing it, and finding which of them reach a use, costs the slice's words and not its definitions one by one: + // definition by definition, a routine of 8,000 such lines took a second to analyze. + const masks = new Map(); + const maskOf = (v) => { + const ids = byVar.get(v); + if (!ids || ids.length <= 64) return null; + let m = masks.get(v); + if (!m) { m = new Uint32Array(W); for (const o of ids) if (inSlice(o)) m[(o - lo) >>> 5] |= 1 << ((o - lo) & 31); masks.set(v, m); } + return m; + }; + // out = in, less every definition of a name a node replaces, plus its own + const transfer = (n) => { + for (const d of n.defs) { + if (d.kind !== 'weak') { + const m = maskOf(d.var); + if (m) for (let w = 0; w < W; w++) next[w] &= ~m[w]; + else for (const o of byVar.get(d.var)) if (inSlice(o)) next[(o - lo) >>> 5] &= ~(1 << ((o - lo) & 31)); + } + if (inSlice(d.id)) next[(d.id - lo) >>> 5] |= 1 << ((d.id - lo) & 31); + } + }; + OUT.fill(0); + let changed = true; + while (changed) { + changed = false; + for (const id of order) { + const n = cfg.nodes[id]; + const base = id * W; + IN.fill(0, base, base + W); + for (const p of n.pred) { const pb = p * W; for (let w = 0; w < W; w++) IN[base + w] |= OUT[pb + w]; } + next.set(IN.subarray(base, base + W)); + transfer(n); + for (let w = 0; w < W; w++) if (next[w] !== OUT[base + w]) { OUT.set(next, base); changed = true; break; } + } + } + // For each use, the definitions of its variable that reach it. A node's + // own weak definition does not reach its own use (the use happens first). + for (const n of cfg.nodes) { + n.uses.forEach((u, k) => { + const m = maskOf(u.var); + if (!m) { for (const i of byVar.get(u.var) || []) if (inSlice(i) && has(n.id * W, i)) hits[n.id][k].push(defs[i]); return; } + // the same definitions, in the same (increasing) order, read a word at a time + const base = n.id * W; + for (let w = 0; w < W; w++) { + let x = IN[base + w] & m[w]; + while (x) { const b = 31 - Math.clz32(x & -x); x &= x - 1; hits[n.id][k].push(defs[lo + w * 32 + b]); } + } + }); + } + } + const uses = []; + for (const n of cfg.nodes) n.uses.forEach((u, k) => uses.push({ ...u, node: n.id, defs: hits[n.id][k] })); + return { defs, uses }; +} + +/** Every node, those the entry reaches in reverse postorder first, then the rest (code after a RETURN). */ +function reversePostorder(cfg) { + const N = cfg.nodes.length; + const order = []; + const seen = new Uint8Array(N); + const postorder = (root) => { + if (seen[root]) return; + const post = []; + const stack = [[root, 0]]; + seen[root] = 1; + while (stack.length) { + const top = stack[stack.length - 1]; + const succ = cfg.nodes[top[0]].succ; + if (top[1] < succ.length) { + const s = succ[top[1]++]; + if (!seen[s]) { seen[s] = 1; stack.push([s, 0]); } + } else { post.push(top[0]); stack.pop(); } + } + for (let i = post.length - 1; i >= 0; i--) order.push(post[i]); + }; + postorder(0); + for (let i = 0; i < N; i++) postorder(i); + return order; +} + +/* ------------------------------------------------------------------ */ +/* Loops */ +/* ------------------------------------------------------------------ */ + +function loopInfo(body) { + const loops = []; + walkStatements(body, (s, parents) => { + if (!['For', 'ForEach', 'While', 'Repeat'].includes(s.kind)) return; + const enclosing = parents.filter((p) => ['For', 'ForEach', 'While', 'Repeat'].includes(p.kind)); + const changes = new Set(); + let breaks = 0; let returns = 0; + walkStatements(s.body, (x) => { + if (x.kind === 'Assign' || x.kind === 'Input') { const r = rootName(x.target); if (r) changes.add(up(r)); } + if (x.kind === 'BuiltinCall' && x.args[0]) { const r = rootName(x.args[0]); if (r) changes.add(up(r)); } + if (x.kind === 'Call') x.args.forEach((a, i) => { if (x.sub?.params?.[i]?.mode === 'BYREF' && a.kind === 'Name') changes.add(up(a.name)); }); + if (x.kind === 'Break') breaks++; + if (x.kind === 'Return') returns++; + if (x.kind === 'For') changes.add(up(x.var)); + }); + if (s.kind === 'For') changes.add(up(s.var)); + const tests = s.kind === 'While' || s.kind === 'Repeat' ? [...namesIn(s.cond || {})].map(up) : s.kind === 'For' ? [up(s.var)] : []; + loops.push({ node: s, line: s.line, endLine: s.endLine, depth: enclosing.length, parent: enclosing[enclosing.length - 1] || null, changes, tests, breaks, returns }); + }); + return loops; +} + +/* ------------------------------------------------------------------ */ +/* The value graph */ +/* ------------------------------------------------------------------ */ + +/** + * Nodes are the unit's inputs, its variables, and its outputs. An edge + * from A to B says a value of A is used to compute B ('data'), or that A + * decides whether B changes ('control'). Accumulators get an edge to + * themselves, which is what makes them accumulators. + */ +function valueGraph(unit, loops) { + const nodes = new Map(); + const edges = new Map(); + const node = (id, props) => { if (!nodes.has(id)) nodes.set(id, { id, ...props, lines: new Set() }); return nodes.get(id); }; + const edge = (from, to, kind, line) => { + const key = `${from}>${to}>${kind}`; + if (!edges.has(key)) edges.set(key, { from, to, kind, lines: new Set() }); + edges.get(key).lines.add(line); + }; + for (const p of unit.params || []) node(up(p.name), { label: p.name, role: 'input', type: p.type }); + for (const sym of unit.symbols || []) if (sym.kind === 'var') node(up(sym.name), { label: sym.name, role: 'variable', type: sym.type, line: sym.line }); + + const exprNames = (es) => { const out = new Set(); for (const e of es) walkExpr(e, (x, bound) => { if (x.kind === 'Name' && !bound.has(up(x.name)) && x.sym && ['var', 'param', 'counter', 'each', 'const', 'define'].includes(x.sym.kind)) out.add(x); }); return [...out]; }; + const ensure = (x) => { + const id = up(x.name); + const kind = x.sym?.kind; + return node(id, { label: x.sym?.name || x.name, role: kind === 'param' ? 'input' : kind === 'counter' || kind === 'each' ? 'counter' : kind === 'const' || kind === 'define' ? 'constant' : 'variable', type: x.sym?.type }); + }; + + // The names a block's conditions read, the first reference to each, worked out once per block: an ELSEIF + // chain of n arms has n conditions, and every statement inside it would otherwise walk all of them. + const firstByName = (xs) => { const seen = new Map(); for (const x of xs) { const k = up(x.name); if (!seen.has(k)) seen.set(k, x); } return [...seen.values()]; }; + const controlCache = new Map(); + const controlNames = (p) => { + if (!controlCache.has(p)) { + const controls = p.kind === 'If' ? p.branches.map((b) => b.cond) : p.kind === 'While' || p.kind === 'Repeat' ? [p.cond] : p.kind === 'Case' ? [p.subject] : []; + controlCache.set(p, firstByName(exprNames(controls.filter(Boolean)))); + } + return controlCache.get(p); + }; + walkStatements(unit.body, (s, parents) => { + const ctlNames = firstByName(parents.flatMap(controlNames)); + const into = (targetId, valueExprs) => { + for (const x of exprNames(valueExprs)) { ensure(x); edge(up(x.name), targetId, 'data', s.line); } + for (const x of ctlNames) { ensure(x); if (up(x.name) !== targetId) edge(up(x.name), targetId, 'control', s.line); } + nodes.get(targetId)?.lines.add(s.line); + }; + switch (s.kind) { + case 'Assign': { + const r = rootName(s.target); + if (!r) break; + const id = up(r); + node(id, { label: r, role: 'variable' }); + into(id, [s.value, ...targetExprs(s.target)]); + if (s.target.kind !== 'Name') edge(id, id, 'data', s.line); + break; + } + case 'Input': { const r = rootName(s.target); if (r) { node('INPUT', { label: 'INPUT', role: 'source' }); edge('INPUT', up(r), 'data', s.line); } break; } + case 'BuiltinCall': { + const r = s.args[0] && rootName(s.args[0]); + if (r) { const id = up(r); node(id, { label: r, role: 'variable' }); into(id, s.args.slice(1)); edge(id, id, 'data', s.line); } + break; + } + case 'Call': + s.args.forEach((a, i) => { + if (s.sub?.params?.[i]?.mode === 'BYREF' && a.kind === 'Name') { const id = up(a.name); into(id, s.args); } + }); + break; + case 'For': { const id = up(s.var); node(id, { label: s.var, role: 'counter' }); into(id, statementExprs(s)); break; } + case 'ForEach': { const id = up(s.var); node(id, { label: s.var, role: 'counter' }); into(id, statementExprs(s)); break; } + case 'Return': if (s.value) { node('RETURN', { label: 'RETURN', role: 'output' }); into('RETURN', [s.value]); } break; + case 'Output': node('OUTPUT', { label: 'OUTPUT', role: 'output' }); into('OUTPUT', s.values); break; + default: break; + } + }); + // Mark accumulators: a self edge. + for (const e of edges.values()) if (e.from === e.to && nodes.get(e.from)) nodes.get(e.from).accumulator = true; + return { nodes: [...nodes.values()].map((n) => ({ ...n, lines: [...n.lines].sort((a, b) => a - b) })), edges: [...edges.values()].map((e) => ({ ...e, lines: [...e.lines].sort((a, b) => a - b) })) }; +} + +/* ------------------------------------------------------------------ */ +/* The four jobs */ +/* ------------------------------------------------------------------ */ + +/** + * Accumulators: names given a starting value outside a loop and then + * changed inside it using their own old value (Total <- Total + 1), or by + * a built-in that adds to them (APPEND, ADD, Tally[B] <- ...). + */ +function findAccumulators(unit, loops) { + const isLoop = (p) => ['For', 'ForEach', 'While', 'Repeat'].includes(p.kind); + // One walk: an update belongs to its innermost loop, and is listed with that loop's, in the order of `loops`. + const byLoop = new Map(loops.map((L) => [L.node, []])); + walkStatements(unit.body, (s, chain) => { + let at = chain.length - 1; + while (at >= 0 && !isLoop(chain[at])) at--; + if (at >= 0) byLoop.get(chain[at])?.push([s, chain.slice(at + 1)]); + }); + const out = []; + const seen = new Set(); + for (const L of loops) { + for (const [s, parents] of byLoop.get(L.node)) { + let name = null; let how = null; + if (s.kind === 'Assign') { + const r = rootName(s.target); + if (!r) continue; + const self = [...namesIn(s.value)].some((n) => up(n) === up(r)); + if (s.target.kind === 'Name' && self) { name = r; how = describeUpdate(s); } + else if (s.target.kind === 'Index' && s.target.target.kind === 'Name' && s.target.type && unit.symbols?.find((x) => up(x.name) === up(r))?.type?.k === 'HASHMAP') { name = r; how = 'tally'; } + else if (s.target.kind === 'Name' && s.value.type?.k === 'BOOL' && (s.value.kind === 'Bool')) { name = r; how = 'flag'; } + else if (s.target.kind === 'Name' && parents.some((p) => p.kind === 'If' && condCompares(p, r))) { name = r; how = 'best'; } + } else if (s.kind === 'BuiltinCall' && ['APPEND', 'ADD', 'INSERT'].includes(s.name)) { + name = rootName(s.args[0]); how = s.name === 'ADD' ? 'collect-set' : 'collect'; + } + if (!name) continue; + const key = `${up(name)}@${L.line}`; + if (seen.has(key)) continue; + seen.add(key); + out.push({ name, var: up(name), how, loop: L.node, line: s.line, stmt: s }); + } + } + return out; +} + +function condCompares(ifStmt, name) { + return ifStmt.branches.some((b) => { + let hit = false; + walkExpr(b.cond, (x) => { if ((x.kind === 'Binary' || x.kind === 'Chain') && ['<', '>', '<=', '>='].includes(x.op || x.ops?.[0])) { if ([...namesIn(x)].some((n) => up(n) === up(name))) hit = true; } }); + return hit; + }); +} + +function describeUpdate(s) { + const v = s.value; + if (v.kind === 'Binary' && v.op === '+' && ((v.right.kind === 'Num' && v.right.value === 1) || (v.left.kind === 'Num' && v.left.value === 1))) return 'count'; + if (v.kind === 'Binary' && v.op === '+') return 'sum'; + if (v.kind === 'Binary' && v.op === '&') return 'build'; + if (v.kind === 'Binary' && v.op === '*') return 'product'; + if (v.kind === 'CallExpr' && ['MAX', 'MIN'].includes(v.builtin)) return 'best'; + return 'update'; +} + +/** Tag each line with one of the four jobs, plus reporting. */ +function assignRoles(unit, loops, accumulators) { + const roles = new Map(); + const put = (line, role, why) => { if (line != null && !roles.has(line)) roles.set(line, { role, why }); }; + const accNames = new Set(accumulators.filter((a) => a.how !== 'flag').map((a) => a.var)); + const flagNames = new Set(accumulators.filter((a) => a.how === 'flag').map((a) => a.var)); + const isFlagCond = (s) => s.kind === 'If' && s.branches.length === 1 && s.branches[0].cond.kind === 'Name' && flagNames.has(up(s.branches[0].cond.name)); + const outer = loops.filter((l) => l.depth === 0).map((l) => l.node); + const firstLoopLine = outer.length ? Math.min(...outer.map((l) => l.line)) : Infinity; + const lastLoopEnd = outer.length ? Math.max(...outer.map((l) => l.endLine ?? l.line)) : -Infinity; + + walkStatements(unit.body, (s, parents) => { + const inLoop = parents.some((p) => ['For', 'ForEach', 'While', 'Repeat'].includes(p.kind)); + switch (s.kind) { + case 'Declare': case 'Constant': put(s.line, 'setup', 'names what the algorithm works with'); break; + case 'Assign': { + const r = up(rootName(s.target) || ''); + if (!inLoop && s.line < firstLoopLine) put(s.line, 'setup', accNames.has(r) ? `gives ${rootName(s.target)} its starting value` : `measures or prepares ${rootName(s.target)}`); + else if (inLoop && flagNames.has(r)) put(s.line, 'test', s.value.kind === 'Bool' && s.value.value ? `assumes a yes until the test finds evidence against it` : `records the evidence the test found`); + else if (inLoop && accNames.has(r)) put(s.line, 'update', `changes the answer so far, ${rootName(s.target)}`); + else if (inLoop) { + const flag = s.value.kind === 'Bool'; + put(s.line, flag && parents.length && !parents.some((p) => p.kind === 'If') ? 'setup' : flag ? 'test' : 'helper', flag ? `resets or settles the flag ${rootName(s.target)}` : 'works out a value used later in this pass'); + } else if (s.line > lastLoopEnd) put(s.line, 'report', 'prepares the answer after the scan'); + else put(s.line, 'helper', ''); + break; + } + case 'BuiltinCall': put(s.line, inLoop ? 'update' : 'setup', inLoop ? `adds to ${rootName(s.args[0])}` : ''); break; + case 'For': case 'ForEach': case 'While': case 'Repeat': + if (inLoop && !accumulators.some((a) => accNames.has(a.var) && a.loop === s)) { + put(s.line, 'test', 'an inner loop that works out the answer for the current position'); + if (s.endLine) put(s.endLine, 'test', ''); + break; + } + put(s.line, 'visit', s.kind === 'For' ? `moves ${s.var} through the positions to visit` : s.kind === 'ForEach' ? `visits each value of ${sourceOf(s.iterable)}` : 'repeats until the condition settles it'); + if (s.endLine) put(s.endLine, 'visit', s.kind === 'For' ? `moves ${s.var} on to the next position` : 'goes back to the top of the loop'); + break; + case 'If': case 'Case': + if (inLoop && isFlagCond(s)) { + put(s.line, 'update', 'acts on the answer the test settled'); + if (s.endLine) put(s.endLine, 'update', ''); + } else if (inLoop) { + put(s.line, 'test', 'decides something about the current position'); + for (const b of s.branches || []) if (b.line !== s.line) put(b.line, 'test', 'tests the next case'); + if (s.elseLine) put(s.elseLine, 'test', 'the case where the test failed'); + if (s.endLine) put(s.endLine, 'test', ''); + } else put(s.line, s.line > lastLoopEnd ? 'report' : 'helper', ''); + break; + case 'Break': put(s.line, 'test', 'stops the scan once the answer is settled'); break; + case 'Continue': put(s.line, 'test', 'skips the rest of this pass'); break; + case 'Return': put(s.line, 'report', 'hands the answer back'); break; + case 'Output': put(s.line, inLoop ? 'update' : 'report', 'shows a result'); break; + default: break; + } + }); + return roles; +} + +/** A name for what the main loop builds. */ +function patternOf(accumulators, loops, unit) { + if (!loops.length) return null; + const main = accumulators.filter((a) => loops.find((l) => l.node === a.loop)?.depth === 0); + const hows = new Set((main.length ? main : accumulators).map((a) => a.how)); + const hasBreak = loops.some((l) => l.depth === 0 && l.breaks > 0); + if (hows.has('tally')) return 'tally'; + if (hows.has('collect')) { + const acc = (main.length ? main : accumulators).find((a) => a.how === 'collect'); + const v = acc.stmt.args[1]; + const counter = acc.loop.kind === 'For' ? up(acc.loop.var) : null; + return v && v.kind === 'Name' && counter && up(v.name) === counter ? 'find-all' : 'filter'; + } + if (hows.has('count')) return 'count'; + if (hows.has('best')) return 'find-best'; + if (hows.has('sum')) return 'sum'; + if (hows.has('build')) return 'build'; + if (hows.has('flag')) return hasBreak ? 'check' : 'check'; + if (hasBreak || loops.some((l) => l.depth === 0 && l.node.kind === 'While')) return 'find-first'; + return null; +} + +export const PATTERN_NAMES = { + count: 'Count', sum: 'Total', 'find-all': 'Find them all', filter: 'Filter', tally: 'Tally', + 'find-best': 'Find the best', 'find-first': 'Find the first', check: 'Check every position', build: 'Build a new sequence', +}; + +/* ------------------------------------------------------------------ */ +/* Counting the work */ +/* ------------------------------------------------------------------ */ + +/** How many passes a loop makes, in the program's own symbols. */ +function passes(loop) { + const s = loop.node; + if (s.kind === 'For') { + const a = s.start; const b = s.stop; + if (!a || !b) return '?'; + const step = s.step ? (s.step.kind === 'Num' ? s.step.value : null) : 1; + let span; + if (a.kind === 'Num' && a.value === 0) span = sourceOf(b); + else if (b.kind === 'Binary' && b.op === '+' && b.right.kind === 'Num' && a.kind === 'Num' && a.value === b.right.value) span = sourceOf(b.left); + else span = `${wrapSum(sourceOf(b))} - ${wrapSum(sourceOf(a))}`; + if (step === 1) return span; + if (step === -1) return `${wrapSum(sourceOf(a))} - ${wrapSum(sourceOf(b))}`; + return step ? `(${span}) / ${Math.abs(step)}` : `(${span}) / ${sourceOf(s.step)}`; + } + if (s.kind === 'ForEach') return `LENGTH(${sourceOf(s.iterable)})`; + if (s.decreases) return `at most ${sourceOf(s.decreases.expr)}, as measured on entry`; + return null; +} +const wrapSum = (x) => (/[+-]/.test(x) ? `(${x})` : x); + +function workOf(loops) { + const innermost = loops.filter((l) => !loops.some((m) => m.parent === l.node)); + return innermost.map((l) => { + const chain = []; + let cur = l; + while (cur) { chain.unshift(cur); cur = loops.find((m) => m.node === cur.parent); } + const parts = chain.map(passes); + const known = parts.every(Boolean); + const exits = chain.some((c) => c.breaks > 0 || c.returns > 0); + return { + line: l.line, endLine: l.endLine, depth: chain.length, + expr: known ? parts.map((p) => (chain.length > 1 && /[ +-]/.test(p) ? `(${p})` : p)).join(' × ') : null, + unknown: !known, + atMost: exits, + loops: chain.map((c) => c.line), + }; + }); +} + +/* ------------------------------------------------------------------ */ +/* Entry point */ +/* ------------------------------------------------------------------ */ + +export function analyzeUnit(unit) { + const { cfg, entry, exit } = buildCFG(unit); + const { defs, uses } = reaching(cfg); + const loops = loopInfo(unit.body); + const accumulators = findAccumulators(unit, loops); + const roles = assignRoles(unit, loops, accumulators); + const graph = valueGraph(unit, loops); + return { + ...unit, + cfg: { nodes: cfg.nodes, entry: entry.id, exit: exit.id }, + defs, uses, loops, accumulators, roles, graph, + pattern: patternOf(accumulators, loops, unit), + work: workOf(loops), + }; +} + +export function analyze(program) { + const units = []; + for (const sub of program.subs.values()) { + units.push(analyzeUnit({ name: sub.name, kind: sub.kind, node: sub.node, body: sub.node.body, params: sub.params, symbols: sub.symbols || [], requires: sub.requires, ensures: sub.ensures, sub })); + } + if (program.main.body.some((s) => s.kind !== 'Declare' || true) && program.main.body.length) { + units.push(analyzeUnit({ name: 'main program', kind: 'main', node: null, body: program.main.body, params: [], symbols: program.main.symbols })); + } + return { units }; +} diff --git a/src/engines/pseudo/ast.ts b/src/engines/pseudo/ast.ts new file mode 100644 index 0000000..72f48cd --- /dev/null +++ b/src/engines/pseudo/ast.ts @@ -0,0 +1,472 @@ +/** + * pseudo/ast.ts + * + * The shapes the pseudo engine passes between its stages, as types: the lexer's tokens, the syntax tree the + * parser builds (and the fields the checker and the interpreter write onto it), the type representation the + * checker computes, symbols and routines, diagnostics, run-time values, and the trace a run records. + * + * Each shape was read off the code that builds it and the code that reads it (parse, check, interp, and the + * modules that consume them). Where a field is sometimes absent, the type says so. A field the checker writes + * onto a node is optional on the node's type, since the tree exists before the checker runs. + * + * This module holds types only; it emits an empty JavaScript module. + */ + +import type { PArray, PRecord, UNSET } from './lang.js'; + +/* ------------------------------------------------------------------ */ +/* Positions and diagnostics */ +/* ------------------------------------------------------------------ */ + +/** Anything that sits at a place in the text: a token, a node, a parameter. Lines and columns count from 1. */ +export interface Loc { line: number; col: number } + +export type Severity = 'error' | 'warning' | 'note'; + +/** A mechanical repair: replace columns [col, endCol) of `line` with `text`. */ +export interface Fix { line: number; col: number; endCol: number; text: string } + +/** What a diagnostic may carry besides its severity, code, line and message. The coach adds `source`. */ +export interface DiagExtra { + col?: number; + /** An anchor in the pseudocode guide that explains the rule. */ + guide?: string; + fix?: Fix; + hint?: string; + [key: string]: unknown; +} + +/** One finding about a program (`diag` in parse.ts builds it). */ +export interface Diag extends DiagExtra { + severity: Severity; + code: string; + line: number; + col: number; + message: string; +} + +/* ------------------------------------------------------------------ */ +/* Tokens */ +/* ------------------------------------------------------------------ */ + +interface TokenBase { + line: number; + col: number; + /** The column just past the token. */ + end: number; + /** The text as written; absent on the line-end and end-of-text tokens. */ + raw?: string; + /** An operator from another notation, as written (':=' for an op token whose `v` is '<-'). */ + wrote?: string; + /** A FLOAT literal's digits, underscores removed. */ + text?: string; + /** Set by the parser on an IF that follows ELSE on the same line. */ + elseIf?: boolean; +} +export interface NumberToken extends TokenBase { t: 'int' | 'float'; v: number } +export interface TextToken extends TokenBase { t: 'name' | 'str' | 'char' | 'blank' | 'kw' | 'op' | 'nl' | 'eof'; v: string } +export type Token = NumberToken | TextToken; +export type TokenKind = Token['t']; + +/** One line of the source as the lexer saw it. */ +export interface LineInfo { n: number; text: string; indent: number; blank: boolean; onlyComment: boolean } + +/** A `//` comment; `standalone` when nothing precedes it on its line. */ +export interface Comment { line: number; col: number; text: string; standalone: boolean } + +export interface Lexed { tokens: Token[]; diags: Diag[]; comments: Comment[]; lines: LineInfo[] } + +/* ------------------------------------------------------------------ */ +/* Types as written (the parser's type expressions) */ +/* ------------------------------------------------------------------ */ + +export type ScalarName = 'INTEGER' | 'FLOAT' | 'CHAR' | 'STRING' | 'BOOL' | SizedName; +export type SizedName = 'INT8' | 'INT16' | 'INT32' | 'INT64' | 'UINT8' | 'UINT16' | 'UINT32' | 'UINT64' | 'FLOAT32' | 'FLOAT64'; + +/** + * A type as the parser reads it. A blank or an error made for a parameter the parser could not read has no place + * of its own (`subroutine` in parse.ts), so `line` and `col` are optional there. + */ +export type TypeExpr = + | { tk: 'scalar'; name: ScalarName; line: number; col: number } + | { tk: 'named'; name: string; line: number; col: number } + | { tk: 'list' | 'set'; of: TypeExpr; line: number; col: number } + | { tk: 'array'; dims: Expr[]; of: TypeExpr; line: number; col: number } + | { tk: 'map'; key: TypeExpr; val: TypeExpr; line: number; col: number } + | { tk: 'blank'; text: string; line?: number; col?: number } + | { tk: 'error'; line?: number; col?: number }; + +/* ------------------------------------------------------------------ */ +/* Types the checker computes */ +/* ------------------------------------------------------------------ */ + +/** + * The checker's types. Every member may be asked for any of the others' fields and answers undefined (the + * engine reads `t.of`, `t.bits` and `t.name` of a type it has not narrowed), so the fields a member lacks are + * declared `?: undefined` through `NoTypeFields`. + */ +interface NoTypeFields { + bits?: undefined; signed?: undefined; name?: undefined; of?: undefined; dims?: undefined; dimExprs?: undefined; + key?: undefined; val?: undefined; fields?: undefined; fieldNames?: undefined; node?: undefined; +} +type Without = Omit; + +/** INTEGER, or a sized whole number (INT8 ... UINT64) with its width. */ +export interface IntType extends Without<'bits' | 'signed' | 'name'> { k: 'INTEGER'; bits?: number; signed?: boolean; name?: string } +/** FLOAT, FLOAT64 (bits 64) or FLOAT32 (bits 32). */ +export interface FloatType extends Without<'bits' | 'name'> { k: 'FLOAT'; bits?: number; name?: string } +export interface SimpleType extends NoTypeFields { k: 'CHAR' | 'STRING' | 'BOOL' | 'VOID' } +/** ANY: a type the checker could not work out (after an error), or one that matches every other. */ +export interface AnyType extends NoTypeFields { k: 'ANY' } +export interface ListType extends Without<'of'> { k: 'LIST'; of: PType } +/** An ARRAY with one or two dimensions; `dimExprs` holds the lengths as written, when the type came from a declaration. */ +export interface ArrayType extends Without<'of' | 'dims' | 'dimExprs'> { k: 'ARRAY'; of: PType; dims: number; dimExprs?: Expr[] } +export interface MapType extends Without<'key' | 'val'> { k: 'HASHMAP'; key: PType; val: PType } +/** A SET; one made with TYPE X = SET OF ... carries its name and its definition. */ +export interface SetType extends Without<'of' | 'name' | 'node'> { k: 'SET'; of: PType; name?: string; node?: SetTypeDefStmt } +/** The positions [lo, hi): what a range such as [0, N) is. */ +export interface RangeType extends Without<'of'> { k: 'RANGE'; of: PType } +/** A record made with TYPE ... ENDTYPE. */ +export interface RecordType extends Without<'name' | 'fields' | 'fieldNames' | 'node'> { + k: 'RECORD'; name: string; fields: Map; fieldNames: string[]; node: TypeDefStmt; +} + +export type PType = IntType | FloatType | SimpleType | AnyType | ListType | ArrayType | MapType | SetType | RangeType | RecordType; +export type TypeKind = PType['k']; + +/** + * A type as a module that types them loosely hands it over: the prover's `TypeLike` (logic.ts, vcgen.ts) and the + * JavaScript callers. The functions of lang.ts that the prover calls accept it and read it as a PType. + */ +export interface LooseType { k: string } + +/* ------------------------------------------------------------------ */ +/* Expressions */ +/* ------------------------------------------------------------------ */ + +/** What every expression node carries. `type` is the checker's; `paren` marks one written in brackets. */ +interface ExprBase extends Loc { + type?: PType; + paren?: boolean; +} + +/** A number literal. `negated` marks a literal the parser folded a minus sign into. */ +export interface NumExpr extends ExprBase { + kind: 'Num'; value: number; float: boolean; text?: string; negated?: boolean; + /** Set by the checker once a literal-range error has been said for it. */ + rangeSaid?: boolean; + /** The FLOAT32 the literal becomes, kept by the interpreter once computed. */ + single?: number; +} +export interface StrExpr extends ExprBase { kind: 'Str'; value: string } +export interface CharExpr extends ExprBase { kind: 'Char'; value: string } +/** TRUE or FALSE; `missing` marks the TRUE the parser puts where a condition was left out. */ +export interface BoolExpr extends ExprBase { kind: 'Bool'; value: boolean; missing?: boolean } +/** A template's blank, such as , or `...`. */ +export interface BlankExpr extends ExprBase { kind: 'Blank'; text: string } +export interface NameExpr extends ExprBase { kind: 'Name'; name: string; sym?: Sym } +/** A type's name standing as an argument: MAX_VALUE(INT8), WRAP(X, UINT8). */ +export interface TypeRefExpr extends ExprBase { kind: 'TypeRef'; name: 'INTEGER' | 'FLOAT' | SizedName } +export interface UnaryExpr extends ExprBase { kind: 'Unary'; op: 'NOT' | '-'; arg: Expr } +export interface BinaryExpr extends ExprBase { kind: 'Binary'; op: string; left: Expr; right: Expr } +/** A chain of comparisons: 0 <= I < N. */ +export interface ChainExpr extends ExprBase { kind: 'Chain'; ops: string[]; operands: Expr[] } +export interface IndexExpr extends ExprBase { kind: 'Index'; target: Expr; indices: Expr[] } +export interface FieldExpr extends ExprBase { kind: 'Field'; target: Expr; field: string } +/** + * A call that stands where a value stands. `convert` marks a sized type's conversion, INT32(X). The checker + * writes `sub` for a call of a routine, `builtin` for a built-in function, and `isOld` for OLD(X) in an ENSURE. + */ +export interface CallExprNode extends ExprBase { + kind: 'CallExpr'; name: string; args: Expr[]; openCol?: number; convert?: boolean; + sub?: Sub; builtin?: string; isOld?: boolean; +} +/** `maybeRange` marks [A, B] where a range [A, B) may have been meant. */ +export interface ListLitExpr extends ExprBase { kind: 'ListLit'; items: Expr[]; maybeRange?: boolean } +export interface SetLitExpr extends ExprBase { kind: 'SetLit'; items: Expr[] } +export interface MapLitExpr extends ExprBase { kind: 'MapLit'; entries: Array<{ key: Expr; value: Expr }> } +export interface RangeExpr extends ExprBase { kind: 'Range'; lo: Expr; hi: Expr } +/** { I IN [0, N) : T[I] == B } */ +export interface SetBuilderExpr extends ExprBase { kind: 'SetBuilder'; var: string; domain: Expr; body: Expr } +/** FORALL, EXISTS or SUM over a domain. */ +export interface QuantExpr extends ExprBase { kind: 'Quant'; q: 'FORALL' | 'EXISTS' | 'SUM'; var: string; domain: Expr; body: Expr } +/** What the parser leaves where it could not read a value. */ +export interface ErrorExpr extends ExprBase { kind: 'Error' } + +export type Expr = + | NumExpr | StrExpr | CharExpr | BoolExpr | BlankExpr | NameExpr | TypeRefExpr | UnaryExpr | BinaryExpr | ChainExpr + | IndexExpr | FieldExpr | CallExprNode | ListLitExpr | SetLitExpr | MapLitExpr | RangeExpr | SetBuilderExpr + | QuantExpr | ErrorExpr; +export type ExprKind = Expr['kind']; + +/* ------------------------------------------------------------------ */ +/* Statements */ +/* ------------------------------------------------------------------ */ + +type StmtBase = Loc; + +/** One name of a DECLARE line; `blank` for a template's blank. */ +export interface DeclName { name: string; line: number; col: number; blank?: boolean } +export interface DeclareStmt extends StmtBase { kind: 'Declare'; names: DeclName[]; type: TypeExpr; resolved?: PType } +export interface ConstantStmt extends StmtBase { kind: 'Constant'; name: string; value: Expr; type?: PType } +export interface DefineStmt extends StmtBase { kind: 'Define'; name: string; values: Expr[]; typeName: string | null; type?: PType } +export interface FieldDecl { name: string; type: TypeExpr; line: number; col: number } +export interface TypeDefStmt extends StmtBase { kind: 'TypeDef'; name: string; fields: FieldDecl[]; endLine: number | null } +export interface SetTypeDefStmt extends StmtBase { kind: 'SetTypeDef'; name: string; of: TypeExpr } + +/** REQUIRE, ENSURE, INVARIANT, DECREASES and ASSERT. The checker marks a contract it moved (`lifted`) or reported. */ +export interface ContractStmt extends StmtBase { + kind: 'Require' | 'Ensure' | 'Invariant' | 'Decreases' | 'Assert'; expr: Expr; + lifted?: boolean; reported?: boolean; +} + +/** What the checker gathers onto a loop: its INVARIANT lines and its DECREASES. */ +interface LoopContracts { invariants?: ContractStmt[]; decreases?: ContractStmt | null } + +export interface IfBranch { cond: Expr; body: Stmt[]; line: number } +export interface IfStmt extends StmtBase { + kind: 'If'; branches: IfBranch[]; elseBody: Stmt[] | null; elseLine: number | null; endLine: number | null; + /** Absent on an IF whose closer is missing before any ELSEIF or ELSE was read. */ + viaElseIf?: boolean; +} +export type CaseLabel = { value: Expr; low?: undefined; high?: undefined } | { low: Expr; high: Expr; value?: undefined }; +export interface CaseArm { labels: CaseLabel[]; body: Stmt[]; line: number } +export interface CaseStmt extends StmtBase { + kind: 'Case'; subject: Expr | null; arms: CaseArm[]; otherwise: Stmt[] | null; otherwiseLine: number | null; endLine: number | null; +} +export interface ForStmt extends StmtBase, LoopContracts { + kind: 'For'; var: string; varCol: number; start: Expr | null; stop: Expr | null; step: Expr | null; body: Stmt[]; endLine: number | null; + /** The name after NEXT, once read (null when NEXT stands alone). */ + nextName?: string | null; + parallel?: boolean; + /** The checker's: the variable this loop counts with when it was declared beforehand, and the counter's symbol. */ + declaredCounter?: Sym; + counterSym?: Sym; +} +export interface ForEachStmt extends StmtBase, LoopContracts { + kind: 'ForEach'; var: string; varCol: number; iterable: Expr | null; body: Stmt[]; endLine: number | null; + nextName?: string | null; + /** The checker's: a SET or HASHMAP is visited in no promised order; the visited value's symbol. */ + unordered?: boolean; + eachSym?: Sym; +} +export interface WhileStmt extends StmtBase, LoopContracts { kind: 'While'; cond: Expr; body: Stmt[]; endLine: number | null } +export interface RepeatStmt extends StmtBase, LoopContracts { kind: 'Repeat'; body: Stmt[]; cond: Expr | null; endLine: number | null } +export type LoopStmt = ForStmt | ForEachStmt | WhileStmt | RepeatStmt; + +export type ParamMode = 'BYVAL' | 'BYREF'; +/** A parameter as written. `explicit` when BYVAL or BYREF was written out; `blank` for a template's blank. */ +export interface ParamDecl { name: string; mode: ParamMode; explicit?: boolean; blank?: boolean; type: TypeExpr; line: number; col: number } +export interface SubStmt extends StmtBase { + kind: 'Function' | 'Procedure'; name: string; nameCol: number; params: ParamDecl[]; returns: TypeExpr | null; body: Stmt[]; endLine: number | null; +} +export interface BreakStmt extends StmtBase { kind: 'Break' } +export interface ContinueStmt extends StmtBase { kind: 'Continue' } +export interface ReturnStmt extends StmtBase { kind: 'Return'; value: Expr | null } +export interface InputStmt extends StmtBase { kind: 'Input'; target: Expr | null } +export interface OutputStmt extends StmtBase { kind: 'Output'; values: Expr[] } +/** CALL Name(args). The checker writes `sub`, or `builtin` for a built-in procedure called with CALL. */ +export interface CallStmt extends StmtBase { kind: 'Call'; name: string; nameCol: number; args: Expr[]; sub?: Sub; builtin?: boolean } +/** USE module, USE module.Routine, or USE IEEE. */ +export interface UseStmt extends StmtBase { kind: 'Use'; module: string; moduleCol: number; routine: string | null; routineCol: number | null; ieee: boolean } +/** A template's blank on a line of its own. */ +export interface BlankStmt extends StmtBase { kind: 'BlankStmt'; text: string } +/** A built-in procedure: APPEND(L, X). The checker writes the type of the collection it changes. */ +export interface BuiltinCallStmt extends StmtBase { kind: 'BuiltinCall'; name: string; args: Expr[]; targetType?: PType } +/** A call on a line of its own without CALL (an error unless it is a procedure). The checker writes `sub`. */ +export interface ExprStmt extends StmtBase { kind: 'ExprStmt'; expr: Expr; sub?: Sub } +export interface AssignStmt extends StmtBase { kind: 'Assign'; target: Expr; value: Expr } + +export type Stmt = + | DeclareStmt | ConstantStmt | DefineStmt | TypeDefStmt | SetTypeDefStmt | ContractStmt | IfStmt | CaseStmt + | ForStmt | ForEachStmt | WhileStmt | RepeatStmt | SubStmt | BreakStmt | ContinueStmt | ReturnStmt | InputStmt + | OutputStmt | CallStmt | UseStmt | BlankStmt | BuiltinCallStmt | ExprStmt | AssignStmt; +export type StmtKind = Stmt['kind']; + +/** The whole text, as the parser reads it. */ +export interface ProgramNode extends Loc { kind: 'Program'; body: Stmt[] } + +/** What `parse` returns. */ +export interface Parsed { + ast: ProgramNode; + diags: Diag[]; + tokens: Token[]; + lines: LineInfo[]; + comments: Comment[]; + source: string; +} + +/* ------------------------------------------------------------------ */ +/* Symbols, routines, the checked program */ +/* ------------------------------------------------------------------ */ + +interface SymBase { name: string; type: PType; line: number; col: number; readOnly?: boolean } +export interface ParamSym extends SymBase { kind: 'param'; mode: ParamMode } +export interface VarSym extends SymBase { kind: 'var'; node: DeclareStmt } +export interface ConstSym extends SymBase { kind: 'const'; node: ConstantStmt } +export interface DefineSym extends SymBase { kind: 'define'; node: DefineStmt } +export interface CounterSym extends SymBase { kind: 'counter'; loop: ForStmt } +export interface EachSym extends SymBase { kind: 'each'; loop: ForEachStmt } +/** A quantifier's or set builder's own name, inside its body. */ +export interface BoundSym extends SymBase { kind: 'bound' } +/** RESULT inside a FUNCTION's ENSURE; it has no place of its own. */ +export interface ResultSym { kind: 'result'; name: string; type: PType; line?: undefined; col?: undefined; readOnly?: undefined } + +/** What a name refers to, as the checker records it on a Name node. */ +export type Sym = ParamSym | VarSym | ConstSym | DefineSym | CounterSym | EachSym | BoundSym | ResultSym; +export type SymKind = Sym['kind']; + +/** A routine's parameter, with its type resolved. */ +export interface Param { name: string; type: PType; mode: ParamMode; line: number; col: number } + +/** Where a routine brought in with USE came from (`resolveUses` in services/pseudo-study.ts writes it). */ +export interface LibOrigin { module: string; id: string; program: Program; source: string } + +/** + * A FUNCTION or PROCEDURE as the checker gathers it. `returns` is null until its parameters are resolved, then + * the declared type (ANY when none was written, VOID for a PROCEDURE). + */ +export interface Sub { + name: string; + kind: 'Function' | 'Procedure'; + node: SubStmt; + params: Param[]; + returns: PType | null; + requires: ContractStmt[]; + ensures: ContractStmt[]; + decreases: ContractStmt | null; + /** The routine's own names, once its body is checked. */ + scope?: ScopeLike; + symbols?: Sym[]; + lib?: LibOrigin; +} + +/** A scope as the checker builds it (the class is `Scope` in check.ts). */ +export interface ScopeLike { + parent: ScopeLike | null; + owner: string; + map: Map; + get(name: string): Sym | undefined; + own(name: string): Sym | undefined; +} + +/** A template blank still to fill in. */ +export interface BlankSpot { line: number; col: number; text: string } + +/** What `check` returns as `program`: the tree with every name and type worked out. */ +export interface Program { + ast: ProgramNode; + types: Map; + setTypes: Map; + constants: Map; + defines: Map; + subs: Map; + libs: Map; + uses: UseStmt[]; + ieee: boolean; + main: { body: Stmt[]; scope: ScopeLike; symbols: Sym[] }; + blanks: BlankSpot[]; + /** Copied from the parse by `check`. */ + source?: string; + lines?: LineInfo[]; + comments?: Comment[]; +} + +/* ------------------------------------------------------------------ */ +/* Run-time values */ +/* ------------------------------------------------------------------ */ + +/** The positions [lo, hi), as a range such as [0, N) evaluates. */ +export interface RangeValue { range: true; lo: number; hi: number } + +/** + * A value a run holds. INTEGER and FLOAT are numbers, CHAR and STRING strings, BOOL a boolean; a LIST is an array, + * an ARRAY a PArray, a HASHMAP a Map, a SET a Set, a record a PRecord. UNSET is the contents of a name declared and + * never given a value; a range is what [A, B) evaluates to. A PROCEDURE's call gives undefined. + */ +export type Value = + | number | string | boolean | typeof UNSET | Value[] | PArray | PRecord | Map | Set | RangeValue | undefined; + +/* ------------------------------------------------------------------ */ +/* Runs */ +/* ------------------------------------------------------------------ */ + +/** A run-time error's kind: the rule that stopped the run. */ +export type ErrorKind = + | 'index' | 'unset' | 'key' | 'divide' | 'overflow' | 'input' | 'require' | 'ensure' | 'invariant' | 'decreases' + | 'assert' | 'limit' | 'depth' | 'value' | 'return' | 'race' | 'internal'; + +/** A read or write a step made: a name, and the position inside it when only one was touched. */ +export interface Access { name: string; path: Value[] | null; frame: number | undefined } + +export type StepKind = + | 'declare' | 'assign' | 'input' | 'output' | 'test' | 'select' | 'loop-start' | 'loop-test' | 'loop-end' | 'next' + | 'break' | 'continue' | 'return' | 'call' | 'check' | 'builtin' | 'expr' | 'error' | 'unrecorded'; + +/** One step of a recorded run. */ +export interface Step { + line: number | null; + kind: StepKind; + text: string; + reads: Access[]; + writes: Access[]; + depth: number; + sub: string | undefined; + /** A test's answer (the condition's value), the CASE arm's line that was taken, or whether a loop goes on. */ + outcome?: Value | null; + value?: Value; + target?: string; + output?: string; + pass?: number; + callee?: string; + library?: string; + errorKind?: ErrorKind; +} + +/** A name's box in a frame. A BYREF parameter shares its caller's slot. */ +export interface Slot { + name: string; + /** RESULT's slot holds the routine's return type, which is null for a routine the checker had not resolved. */ + type?: PType | null; + value: Value; + line?: number; + kind?: 'var' | 'param' | 'const' | 'define' | 'counter' | 'each' | 'bound' | 'result'; + /** For a declared FOR counter after its loop: the line of the loop's NEXT. */ + afterLoop?: number | null; +} + +/** One name in a snapshot. */ +export interface VarSnapshot { name: string; type: PType | null | undefined; value: Value; slot: Slot; line: number | undefined; kind: Slot['kind'] } +export interface FrameSnapshot { id: number; name: string; vars: Record; callLine: number | null } +/** The state before a step: every live frame's names, how much output there is, and the loops in progress. */ +export interface State { + frames: FrameSnapshot[]; + outputLength: number; + loops: Array<{ line: number; pass: number; kind: LoopStmt['kind'] }>; +} + +/** A call to make instead of the main program. */ +export interface RunCall { name: string; args?: Value[] } + +export interface RunOptions { + call?: RunCall | null; + /** What INPUT reads, one line at a time. */ + input?: unknown[]; + maxSteps?: number; + /** Positions FORALL, EXISTS, SUM and sets written by rule may visit over the run (default: 100 per step allowed). */ + maxVisits?: number; + record?: boolean; + /** Values the snapshots may copy before recording stops. */ + maxRecorded?: number; + contracts?: boolean; + seed?: number; + /** Visit SET members and HASHMAP keys in a random order. */ + shuffle?: boolean; + /** Calls deeper than this stop the run. */ + maxDepth?: number; + /** Step into a routine from the library instead of running it as one step. */ + stepIntoLibrary?: boolean; +} + +/** How much a run did. */ +export interface Work { steps: number; compares: number; letterCompares: number; reads: number; writes: number; calls: number; contractChecks: number } diff --git a/src/engines/pseudo/check.ts b/src/engines/pseudo/check.ts new file mode 100644 index 0000000..f24e69c --- /dev/null +++ b/src/engines/pseudo/check.ts @@ -0,0 +1,1507 @@ +/** + * pseudo/check.ts + * + * Names and types. The checker works out what every name refers to and what + * type every expression has, writes both onto the tree, and reports each + * place where the notation's rules are broken. It also gathers each + * subroutine's contracts (REQUIRE, ENSURE, DECREASES) and each loop's + * (INVARIANT, DECREASES) into fields of their own, so the interpreter and the + * verifier find them without searching. + * + * check(parsed) -> { program, diags } + * + * program = { + * ast, types, setTypes, constants, defines, subs, main, + * blanks: [ { line, col, text } ], template blanks still to fill + * } + */ + +import { + T, BUILTINS, MUTATORS, typeName, sameType, assignable, isNumeric, isAny, isText, + SIZED, isSizedInt, isFloat32, intRange, intRangeText, TYPE_ARGS, SOFT_BUILTINS, + knownDims, +} from './lang.js'; +import { diag, sourceOf } from './parse.js'; +import { editDistance } from '../shared/text.js'; +import type { + PType, RecordType, SetType, SetTypeDefStmt, Loc, Diag, DiagExtra, Parsed, Program, Sub, Sym, ConstSym, DefineSym, CounterSym, + EachSym, BlankSpot, TypeExpr, Expr, NumExpr, StrExpr, CharExpr, BoolExpr, BinaryExpr, CallExprNode, IndexExpr, NameExpr, + SetBuilderExpr, QuantExpr, Stmt, ContractStmt, LoopStmt, ForStmt, ForEachStmt, CaseStmt, ReturnStmt, AssignStmt, CallStmt, + BuiltinCallStmt, ExprStmt, UseStmt, +} from './ast.js'; + +/** A SET type made with TYPE X = SET OF ...: it has its name and its definition. */ +type NamedSetType = SetType & { name: string; node: SetTypeDefStmt }; +/** Two operands, as +, DIV and the bit built-ins (GCD(A, B) checked as A and B) have them. */ +type Pair = Loc & { left: Expr; right: Expr; op?: string }; +/** What `expr` is asked to allow or expect of an expression. */ +interface ExprOpts { expect?: PType | null; rangeOk?: boolean; quiet?: boolean; typeArg?: boolean; target?: boolean } +/** What the checker is inside of: a CONSTANT's or DEFINE's values (literals and other constants only), or a body. */ +interface ConstCtx { scope: Scope; constOnly: true; sub?: undefined; loops?: undefined; inEnsure?: undefined } +interface BodyCtx { scope: Scope; sub: Sub | null; loops: LoopStmt[]; counters: never[]; inEnsure?: boolean; constOnly?: undefined } +type Ctx = ConstCtx | BodyCtx; +/** Routines from the library (UPPER name -> checked routine), and where each USE brought one in. */ +export interface CheckOptions { libs?: Map; libOrigin?: Map } + +/* ------------------------------------------------------------------ */ +/* Helpers */ +/* ------------------------------------------------------------------ */ + +const up = (s: unknown): string => String(s).toUpperCase(); + +/** Function names from Python and Mojo, and what this notation writes instead. */ +const FOREIGN_FUNCTIONS: Record = { + LEN: 'The length of a STRING or a collection is LENGTH(T).', + RANGE: 'A loop over positions is FOR I <- 0 TO N ... NEXT I, and a range in a condition or a set is [0, N).', + PRINT: 'Output is a statement of its own: OUTPUT X.', + STR: 'TO_STRING(X) turns a value into text.', + FLOAT: 'TO_FLOAT(X) turns a whole number or text into a FLOAT.', + APPEND: null, + ENUMERATE: 'FOR I <- 0 TO LENGTH(T) visits every position, and T[I] is the letter there.', + ZIP: 'Walk both with one counter: FOR I <- 0 TO LENGTH(A), reading A[I] and B[I].', + SUM: 'Add up in a loop with a running total, or write SUM K IN [0, N) : L[K] inside a specification.', + SORT: null, + SORTED: null, + REVERSED: 'Build the reverse in a loop that walks from the last position down: FOR I <- LENGTH(T) - 1 TO -1 STEP -1.', + JOIN: 'Join text with &, as in A & B.', + SPLIT: 'This notation has no SPLIT. Walk the text one letter at a time with a FOR loop.', + INPUT: 'INPUT is a statement of its own: INPUT X reads the next value into X.', +}; +for (const k of Object.keys(FOREIGN_FUNCTIONS)) if (!FOREIGN_FUNCTIONS[k]) delete FOREIGN_FUNCTIONS[k]; + +/** Type names from Python and Mojo, and this notation's word for each. */ +const FOREIGN_TYPES: Record = { + INT: 'INTEGER (or INT64 for a 64-bit whole number)', LONG: 'INT64', NUMBER: 'INTEGER or FLOAT', + STR: 'STRING', TEXT: 'STRING', DOUBLE: 'FLOAT64', REAL: 'FLOAT', SINGLE: 'FLOAT32', + BOOLEAN: 'BOOL', DICT: 'HASHMAP OF TO ', MAP: 'HASHMAP OF TO ', + ARRAYLIST: 'LIST OF ', +}; + +/** The closest few names to a misspelt one, by edit distance. */ +export function suggest(name: string, candidates: Iterable): string[] { + const n = up(name); + const scored: Array<[number, string]> = []; + for (const c of candidates) { + const d = editDistance(n, up(c)); + if (d <= Math.max(1, Math.floor(n.length / 3))) scored.push([d, c]); + } + scored.sort((a, b) => a[0] - b[0]); + return scored.slice(0, 2).map((x) => x[1]); +} + +const article = (t: PType | null | undefined): string => { + if (!t || t.k === 'ANY') return 'a value'; + const w = typeName(t); + return /^[AEIOU]/.test(w) ? `an ${w}` : `a ${w}`; +}; + +function elementOf(t: PType | null | undefined): PType | null { + if (!t) return T.ANY; + switch (t.k) { + case 'STRING': return T.CHAR; + case 'LIST': case 'SET': case 'ARRAY': return t.of; + case 'HASHMAP': return t.key; + case 'RANGE': return T.INTEGER; + default: return null; + } +} + +/** + * The type two numbers combine to. A FLOAT with anything is a FLOAT (FLOAT32 stays FLOAT32 next to a whole + * number; FLOAT32 next to FLOAT64 is a clash the caller reports). Two whole numbers are INTEGER, or the sized + * type when one is sized (two different widths are a clash the caller reports). + */ +function numericResult(a: PType, b: PType): PType { + if (isAny(a) || isAny(b)) return isAny(a) ? (isAny(b) ? T.ANY : b) : a; + if (a.k === 'FLOAT' || b.k === 'FLOAT') { + if (a.k === 'FLOAT' && b.k === 'FLOAT') return isFloat32(a) && isFloat32(b) ? a : (a.bits === 64 ? a : b.bits === 64 ? b : T.FLOAT); + return a.k === 'FLOAT' ? a : b; + } + if (a.bits) return a; + return b.bits ? b : T.INTEGER; +} + +/** A numeric literal, which takes the type of the sized value beside it: X + 1 for an INT8 X is INT8 arithmetic. */ +const isLiteral = (n: Expr | null | undefined): n is NumExpr => !!n && n.kind === 'Num'; + +/** Does a loop's body hold a BREAK that leaves this loop (one inside a nested loop leaves that one)? */ +function breaksOut(body: Stmt[] | null | undefined): boolean { + return (body || []).some((s) => s.kind === 'Break' + || (s.kind === 'If' && (s.branches.some((b) => breaksOut(b.body)) || breaksOut(s.elseBody))) + || (s.kind === 'Case' && (s.arms.some((a) => breaksOut(a.body)) || breaksOut(s.otherwise)))); +} + +/** Does a list of statements end every path in RETURN? */ +export function alwaysReturns(body: Stmt[]): boolean { + for (const s of body) { + if (s.kind === 'Return') return true; + // WHILE TRUE with no BREAK of its own is left only by a RETURN inside it: nothing after it is ever reached + if (s.kind === 'While' && s.cond?.kind === 'Bool' && s.cond.value === true && !breaksOut(s.body)) return true; + if (s.kind === 'If' && s.elseBody && s.branches.every((b) => alwaysReturns(b.body)) && alwaysReturns(s.elseBody)) return true; + if (s.kind === 'Case' && s.otherwise && s.arms.every((a) => alwaysReturns(a.body)) && alwaysReturns(s.otherwise)) return true; + } + return false; +} + +/* ------------------------------------------------------------------ */ +/* Scopes */ +/* ------------------------------------------------------------------ */ + +/** + * A scope maps an uppercase name to a symbol: + * { name, type, kind, line, col, readOnly } + * kind is 'var', 'param', 'counter', 'each', 'bound' (a quantifier's name), + * 'const', or 'define'. + */ +class Scope { + declare parent: Scope | null; + declare owner: string; + declare map: Map; + constructor(parent: Scope | null, owner: string) { this.parent = parent; this.owner = owner; this.map = new Map(); } + get(name: string): Sym | undefined { const k = up(name); return this.map.get(k) ?? this.parent?.get(k); } + own(name: string): Sym | undefined { return this.map.get(up(name)); } + set(name: string, sym: S): S { this.map.set(up(name), sym); return sym; } + all(): Map { const out = this.parent ? this.parent.all() : new Map(); for (const [k, v] of this.map) out.set(k, v); return out; } +} + +/* ------------------------------------------------------------------ */ +/* The checker */ +/* ------------------------------------------------------------------ */ + +class Checker { + declare parsed: Parsed; + declare libs: Map; + declare libOrigin: Map; + declare diags: Diag[]; + declare types: Map; + declare setTypes: Map; + declare constants: Map; + declare defines: Map; + declare subs: Map; + /** Routines the program cannot call by name (a second one of a name, one named after a built-in): each is still checked, with a record of its own. */ + orphans: Sub[] = []; + declare blanks: BlankSpot[]; + declare global: Scope; + declare mainScope: Scope; + declare ctx: Ctx | null; + constructor(parsed: Parsed, options: CheckOptions = {}) { + this.parsed = parsed; + this.libs = options.libs || new Map(); // routines from the library: UPPER name -> checked routine + this.libOrigin = options.libOrigin || new Map(); // UPPER name -> { module, line } of the USE that brought it + this.diags = []; + this.types = new Map(); // record types + this.setTypes = new Map(); // TYPE X = SET OF ... + this.constants = new Map(); + this.defines = new Map(); + this.subs = new Map(); + this.blanks = []; + this.global = new Scope(null, 'global'); + this.mainScope = new Scope(this.global, 'main'); + this.ctx = null; + } + + /** A routine defined in this program, or brought in from the library with USE. */ + findSub(name: string): Sub | undefined { const k = up(name); return this.subs.get(k) || this.libs.get(k); } + + error(code: string, node: Partial | null | undefined, message: string, extra: DiagExtra = {}): void { this.diags.push(diag('error', code, node?.line ?? 1, message, { col: node?.col ?? 1, ...extra })); } + warn(code: string, node: Partial | null | undefined, message: string, extra: DiagExtra = {}): void { this.diags.push(diag('warning', code, node?.line ?? 1, message, { col: node?.col ?? 1, ...extra })); } + note(code: string, node: Partial | null | undefined, message: string, extra: DiagExtra = {}): void { this.diags.push(diag('note', code, node?.line ?? 1, message, { col: node?.col ?? 1, ...extra })); } + + blank(node: Loc, text: string): void { + this.blanks.push({ line: node.line, col: node.col, text }); + this.diags.push(diag('error', 'blank', node.line, `<${text}> is a blank to fill in before this can run.`, { col: node.col })); + } + + /* ---------- pass 1: everything global ---------- */ + collect(body: Stmt[]): void { + for (const s of body) { + if (s.kind === 'TypeDef') { + if (this.types.has(up(s.name)) || this.setTypes.has(up(s.name))) this.error('duplicate', s, `A type called ${s.name} is already defined.`); + this.types.set(up(s.name), { k: 'RECORD', name: s.name, fields: new Map(), fieldNames: [], node: s }); + } else if (s.kind === 'SetTypeDef') { + this.setTypes.set(up(s.name), { k: 'SET', of: T.ANY, name: s.name, node: s }); + } else if (s.kind === 'Function' || s.kind === 'Procedure') { + if (s.name.startsWith('<')) this.subs.set(`${up(s.name)}#${s.line}`, { name: s.name, kind: s.kind, node: s, params: [], returns: null, requires: [], ensures: [], decreases: null }); + else if (this.subs.has(up(s.name))) { + this.error('duplicate', s, `${s.name} is already defined on line ${this.subs.get(up(s.name))!.node.line}.`); + this.orphans.push({ name: s.name, kind: s.kind, node: s, params: [], returns: null, requires: [], ensures: [], decreases: null }); + } + else if ((BUILTINS[up(s.name)] && !SOFT_BUILTINS.has(up(s.name))) || SIZED[up(s.name)]) { + this.error('builtin-name', s, `${up(s.name)} is a built-in, so a ${s.kind.toUpperCase()} cannot take its name.`); + this.orphans.push({ name: s.name, kind: s.kind, node: s, params: [], returns: null, requires: [], ensures: [], decreases: null }); + } + else if (this.libs.has(up(s.name))) { + this.error('library-clash', s, `${s.name} also comes from the library (USE ${this.libOrigin.get(up(s.name))?.module} on line ${this.libOrigin.get(up(s.name))?.line}). Rename this one, or bring in only the routines you need: USE ${this.libOrigin.get(up(s.name))?.module}..`, { guide: 'libraries' }); + this.subs.set(up(s.name), { name: s.name, kind: s.kind, node: s, params: [], returns: null, requires: [], ensures: [], decreases: null }); + } + else this.subs.set(up(s.name), { name: s.name, kind: s.kind, node: s, params: [], returns: null, requires: [], ensures: [], decreases: null }); + } + } + // Field and element types, now that every type name is known. + for (const rt of this.types.values()) { + for (const f of rt.node.fields) { + if (rt.fields.has(up(f.name)) && !f.name.startsWith('<')) this.error('duplicate', f, `${rt.name} already has a field called ${f.name}.`); + rt.fields.set(up(f.name), this.resolveType(f.type, f)); + rt.fieldNames.push(f.name); + } + } + // A record that holds a record of its own type (directly, through another record, or in an ARRAY, whose positions + // are records from the start) would never end. A LIST OF the type starts empty and is fine. + const inside = (t: PType | null | undefined): RecordType | null => (t?.k === 'ARRAY' ? inside(t.of) : t?.k === 'RECORD' ? t : null); + for (const rt of this.types.values()) { + const reaches = (t: PType | null | undefined, seen: Set): boolean => { const r = inside(t); return !!r && (r === rt || (!seen.has(r) && (seen.add(r), [...r.fields.values()].some((x) => reaches(x, seen))))); }; + for (const f of rt.node.fields) { + if (reaches(rt.fields.get(up(f.name)), new Set())) { + this.error('recursive-type', f, `${rt.name} cannot hold ${f.name} : ${typeName(rt.fields.get(up(f.name)))}: a ${rt.name} inside every ${rt.name} would never end. A LIST OF ${typeName(inside(rt.fields.get(up(f.name))))} starts empty and can hold as many as are needed.`, { guide: 'records' }); + rt.fields.set(up(f.name), T.ANY); + } + } + } + for (const st of this.setTypes.values()) st.of = this.member(this.resolveType(st.node.of, st.node), st.node.of || st.node); + for (const sub of [...this.subs.values(), ...this.orphans]) { + const s = sub.node; + for (const p of s.params) { + const type = this.resolveType(p.type, p); + if (p.mode === 'BYREF' && s.kind === 'Function') { + this.error('function-byref', p, `Parameters of a FUNCTION are always by value, so ${p.name} cannot be BYREF. A subroutine that changes its caller's variables is a PROCEDURE.`, { guide: 'passing-by-value-or-by-reference' }); + } + sub.params.push({ name: p.name, type, mode: s.kind === 'Function' ? 'BYVAL' : p.mode, line: p.line, col: p.col }); + } + sub.returns = s.kind === 'Function' ? (s.returns ? this.resolveType(s.returns, s) : T.ANY) : T.VOID; + } + // Constants and sets, in order, since one may use another. + for (const s of body) { + if (s.kind === 'Constant') { + if (this.constants.has(up(s.name))) { this.error('duplicate', s, `The constant ${s.name} is already defined.`); continue; } + this.ctx = { scope: this.global, constOnly: true }; + const type = this.expr(s.value, this.global); + this.ctx = null; + const sym: ConstSym = { name: s.name, type, kind: 'const', line: s.line, col: s.col, readOnly: true, node: s }; + this.constants.set(up(s.name), sym); + this.global.set(s.name, sym); + s.type = type; + } else if (s.kind === 'Define') { + const st = this.setTypes.get(up(s.typeName || '')); + if (!st && !String(s.typeName).startsWith('<')) this.error('unknown-type', s, `DEFINE ${s.name} names the set type ${s.typeName}, and no TYPE ${s.typeName} = SET OF ... is defined.`, { guide: 'set-for-membership' }); + const type = st || T.set(T.ANY); + this.ctx = { scope: this.global, constOnly: true }; + for (const v of s.values) { + const vt = this.expr(v, this.global); + if (st && !assignable(st.of, vt)) this.error('type', v, `${sourceOf(v)} is ${article(vt)}, and ${st.name} holds ${typeName(st.of)} values.`); + } + this.ctx = null; + const sym: DefineSym = { name: s.name, type, kind: 'define', line: s.line, col: s.col, readOnly: true, node: s }; + this.defines.set(up(s.name), sym); + this.global.set(s.name, sym); + s.type = type; + } + } + } + + resolveType(tn: TypeExpr | null | undefined, at: unknown): PType { + if (!tn) return T.ANY; + switch (tn.tk) { + case 'scalar': return T[tn.name]; + case 'list': return T.list(this.resolveType(tn.of, at)); + case 'set': return T.set(this.member(this.resolveType(tn.of, at), tn)); + case 'map': { + const key = this.resolveType(tn.key, at); + if (key.k === 'FLOAT') this.warn('float-key', tn, 'A FLOAT key finds its value only when the key is exactly equal, and two measurements rarely are. Round to an INTEGER first.'); + if (!['INTEGER', 'CHAR', 'STRING', 'BOOL', 'FLOAT', 'ANY'].includes(key.k)) this.error('bad-key', tn, `A HASHMAP key is a single value: INTEGER, CHAR, STRING, or BOOL, not ${typeName(key)}.`); + return T.map(key, this.resolveType(tn.val, at)); + } + case 'array': { + for (const d of tn.dims) { + const dt = this.expr(d, this.ctx?.scope || this.mainScope); + if (!isAny(dt) && dt.k !== 'INTEGER') this.error('type', d, `An ARRAY's length is a whole number, and ${sourceOf(d)} is ${article(dt)}.`); + } + const of = this.resolveType(tn.of, at); + // an ARRAY's positions hold values, and an ARRAY is not made position by position: a table has two lengths + if (of.k === 'ARRAY') this.error('array-of-array', tn, 'An ARRAY of ARRAYs is written as one ARRAY with two lengths: ARRAY[Rows, Columns] OF , read as A[Row, Column].', { guide: 'array-when-you-know-how-many' }); + return { ...T.array(of, tn.dims.length), dimExprs: tn.dims }; + } + case 'named': { + const r = this.types.get(up(tn.name)) || this.setTypes.get(up(tn.name)); + if (r) return r; + const s = suggest(tn.name, [...this.types.values(), ...this.setTypes.values()].map((x) => x.name).concat(['INTEGER', 'FLOAT', 'CHAR', 'STRING', 'BOOL'])); + const other = FOREIGN_TYPES[up(tn.name)]; + if (other) this.error('unknown-type', tn, `${tn.name} is not a type here. Write ${other}.`, { guide: 'the-types' }); + else this.error('unknown-type', tn, `${tn.name} is not a type.${s.length ? ` Did you mean ${s.join(' or ')}?` : ' Define it with TYPE ... ENDTYPE first.'}`, { guide: 'the-types' }); + return T.ANY; + } + // typed-port: a blank parameter written without a type, FUNCTION F(), has a blank type with no place of its + // own (parse.ts `subroutine`), so this blank is reported at line undefined, beside the one for the parameter. + case 'blank': this.blank(tn as Loc & typeof tn, tn.text); return T.ANY; + default: return T.ANY; + } + } + + /* ---------- pass 2: bodies ---------- */ + run(): Program { + const ast = this.parsed.ast; + this.collect(ast.body); + const mainBody: Stmt[] = []; + const byNode = new Map([...this.subs.values(), ...this.orphans].map((x) => [x.node as Stmt, x])); + for (const s of ast.body) { + if (['TypeDef', 'SetTypeDef', 'Constant', 'Define', 'Use'].includes(s.kind)) continue; + // each routine is checked with its own record: the one the first pass kept under its name, or, for a second + // routine of a name or one named after a built-in (both already reported), the one it set aside. Looking the + // record up by name alone checked the first body twice and the second never, and gave a routine named after a + // built-in a record with no kind, on which a REQUIRE below its first line made the checker throw. + if (s.kind === 'Function' || s.kind === 'Procedure') { this.subroutine(byNode.get(s)!); continue; } + mainBody.push(s); + } + this.ctx = { scope: this.mainScope, sub: null, loops: [], counters: [] }; + this.liftContracts(mainBody, 'main'); + this.block(mainBody, this.mainScope); + this.ctx = null; + return { + ast, + types: this.types, + setTypes: this.setTypes, + constants: this.constants, + defines: this.defines, + subs: this.subs, + libs: this.libs, + uses: ast.body.filter((s): s is UseStmt => s.kind === 'Use'), + ieee: ast.body.some((s) => s.kind === 'Use' && s.ieee), + main: { body: mainBody, scope: this.mainScope, symbols: [...this.mainScope.map.values()] }, + blanks: this.blanks, + }; + } + + /** REQUIRE and ENSURE at the head of a body move onto the subroutine. */ + liftContracts(body: Stmt[], where: 'main' | 'sub', sub: Sub | null = null): void { + let head = true; + for (let i = 0; i < body.length; i++) { + const s = body[i]; + const isContract = s.kind === 'Require' || s.kind === 'Ensure'; + if (isContract && where === 'main') { + this.error('contract-place', s, `${s.kind.toUpperCase()} belongs directly under a FUNCTION or PROCEDURE line. To check a condition in the main program, write ASSERT.`, { guide: 'contracts' }); + s.reported = true; + continue; + } + if (s.kind === 'Decreases' && where === 'sub' && head) { sub!.decreases = s; s.lifted = true; continue; } + if (isContract) { + if (!head) { this.error('contract-place', s, `${s.kind.toUpperCase()} goes directly under the ${sub!.kind.toUpperCase()} line, before any other line of the body.`, { guide: 'contracts' }); s.reported = true; } + else { (s.kind === 'Require' ? sub!.requires : sub!.ensures).push(s); s.lifted = true; } + continue; + } + head = false; + } + for (let i = body.length - 1; i >= 0; i--) if ((body[i] as ContractStmt).lifted) body.splice(i, 1); + } + + subroutine(sub: Sub): void { + const s = sub.node; + const scope = new Scope(this.global, s.name); + sub.scope = scope; + for (const p of sub.params) { + if (p.name.startsWith('<')) { this.blank(p, p.name.slice(1, -1)); continue; } + if (scope.own(p.name)) this.error('duplicate', p, `Two parameters are called ${p.name}.`); + if (this.global.get(p.name)) this.warn('shadow', p, `Parameter ${p.name} has the same name as the ${this.global.get(p.name)!.kind === 'const' ? 'constant' : 'set'} ${this.global.get(p.name)!.name}, which it hides inside ${s.name}.`); + scope.set(p.name, { name: p.name, type: p.type, kind: 'param', mode: p.mode, line: p.line, col: p.col }); + } + this.liftContracts(s.body, 'sub', sub); + this.ctx = { scope, sub, loops: [], counters: [] }; + for (const r of sub.requires) this.condition(r.expr, scope, 'REQUIRE'); + this.ctx.inEnsure = true; + for (const e of sub.ensures) this.condition(e.expr, scope, 'ENSURE'); + this.ctx.inEnsure = false; + if (sub.decreases) { + const dt = this.expr(sub.decreases.expr, scope); + if (!isAny(dt) && dt.k !== 'INTEGER') this.error('type', sub.decreases, 'DECREASES needs a whole number that gets smaller with every recursive call.'); + } + this.block(s.body, scope); + if (s.kind === 'Function' && !alwaysReturns(s.body) && s.endLine && !s.body.some((x) => x.kind === 'BlankStmt')) { + this.error('missing-return', { line: s.endLine, col: 1 }, `FUNCTION ${s.name} can reach ENDFUNCTION without a RETURN, and a function has to hand back ${article(sub.returns)} on every path.`, { guide: 'function' }); + } + sub.symbols = [...scope.map.values()]; + this.ctx = null; + } + + block(body: Stmt[], scope: Scope): void { + for (const s of body) this.statement(s, scope); + } + + /** Names a loop creates are visible only inside it. */ + childScope(scope: Scope): Scope { const c = new Scope(scope, scope.owner); c.map = scope.map; return c; } + + declareName(name: string, sym: Sym, scope: Scope, node: Stmt): Sym { + const k = up(name); + const prior = scope.own(name); + if (prior) { + if ((prior as { node?: unknown }).node === node) return prior; // the same DECLARE, reached again + this.error('duplicate', node, `${name} is already declared on line ${prior.line}${prior.kind === 'param' ? ' as a parameter' : ''}. One name per idea, throughout an algorithm.`, { guide: 'names' }); + return prior; + } + const g = this.global.get(name); + if (g) this.error('duplicate', node, `${name} is already the name of a ${g.kind === 'const' ? 'CONSTANT' : 'set'} defined on line ${g.line}.`); + if (this.findSub(k)) this.error('duplicate', node, `${name} is already the name of a ${this.findSub(k)!.kind.toUpperCase()}${this.libs.has(k) ? ' from the library' : ''}.`); + if ((BUILTINS[k] && !SOFT_BUILTINS.has(k)) || SIZED[k]) this.error('builtin-name', node, `${k} is a built-in, so a variable cannot take its name.`); + if (this.types.has(k) || this.setTypes.has(k)) this.error('duplicate', node, `${name} is already the name of a type.`); + return scope.set(name, sym); + } + + statement(s: Stmt, scope: Scope): void { + const ctx = this.ctx as BodyCtx; // a statement is always inside a body + switch (s.kind) { + case 'Declare': { + const type = this.resolveType(s.type, s); + s.resolved = type; + for (const n of s.names) { + if (n.blank) { this.blank(n, n.name.slice(1, -1)); continue; } + this.declareName(n.name, { name: n.name, type, kind: 'var', line: s.line, col: n.col, node: s }, scope, s); + } + return; + } + case 'TypeDef': case 'SetTypeDef': case 'Constant': case 'Define': + this.error('global-only', s, `${s.kind === 'TypeDef' || s.kind === 'SetTypeDef' ? 'TYPE' : s.kind === 'Constant' ? 'CONSTANT' : 'DEFINE'} lines go at the top level, outside every block.`); + return; + case 'Function': case 'Procedure': return; + case 'Use': this.error('use-place', s, 'USE lines go at the top of the program, outside every block.', { guide: 'libraries' }); return; + case 'Assign': this.assign(s, scope); return; + case 'Input': { + if (!s.target) return; + const t = this.target(s.target, scope, 'INPUT'); + if (t && !isAny(t) && !['INTEGER', 'FLOAT', 'CHAR', 'STRING', 'BOOL'].includes(t.k)) this.error('type', s, `INPUT fills one value, and ${sourceOf(s.target)} is ${article(t)}. Read the values one at a time in a loop.`, { guide: 'input-and-output' }); + return; + } + case 'Output': + for (const v of s.values) { + const t = this.expr(v, scope); + if (t.k === 'VOID') this.error('type', v, `${sourceOf(v)} hands nothing back, so there is nothing to show.`); + } + return; + case 'If': + for (const b of s.branches) { + this.condition(b.cond, scope, b === s.branches[0] ? 'IF' : 'ELSEIF'); + this.block(b.body, scope); + } + if (s.elseBody) this.block(s.elseBody, scope); + return; + case 'Case': this.caseStmt(s, scope); return; + case 'For': this.forStmt(s, scope); return; + case 'ForEach': this.forEach(s, scope); return; + case 'While': { + this.condition(s.cond, scope, 'WHILE'); + this.loopBody(s, scope); + return; + } + case 'Repeat': { + this.loopBody(s, scope); + if (s.cond) this.condition(s.cond, scope, 'UNTIL'); + return; + } + case 'Break': case 'Continue': + if (!ctx.loops.length) this.error('outside-loop', s, `${s.kind.toUpperCase()} works only inside a loop, and this line is not inside one.`, { guide: 'leaving-a-loop-early' }); + return; + case 'Return': this.returnStmt(s, scope); return; + case 'Call': this.callStmt(s, scope); return; + case 'BuiltinCall': this.builtinStmt(s, scope); return; + case 'ExprStmt': this.exprStmt(s, scope); return; + case 'Assert': this.condition(s.expr, scope, 'ASSERT'); return; + case 'Invariant': case 'Decreases': + this.error('contract-place', s, `${s.kind.toUpperCase()} goes at the top of a loop's body, before any other line of it.`, { guide: 'contracts' }); + this.expr(s.expr, scope); + return; + case 'Require': case 'Ensure': + if (!s.reported) this.error('contract-place', s, `${s.kind.toUpperCase()} goes directly under a FUNCTION or PROCEDURE line.`, { guide: 'contracts' }); + return; + case 'BlankStmt': this.blank(s, s.text); return; + default: return; + } + } + + /** INVARIANT and DECREASES at the top of a loop body belong to the loop. */ + loopBody(loop: LoopStmt, scope: Scope): void { + loop.invariants = []; + loop.decreases = null; + const body = loop.body; + let i = 0; + while (i < body.length && (body[i].kind === 'Invariant' || body[i].kind === 'Decreases')) { + if (body[i].kind === 'Invariant') loop.invariants.push(body[i] as ContractStmt); + else if (loop.decreases) this.error('duplicate', body[i], 'A loop has one DECREASES.'); + else loop.decreases = body[i] as ContractStmt; + i++; + } + body.splice(0, i); + (this.ctx as BodyCtx).loops.push(loop); + const inner = this.childScope(scope); + for (const inv of loop.invariants) this.condition(inv.expr, scope, 'INVARIANT'); + if (loop.decreases) { + const dt = this.expr(loop.decreases.expr, scope); + if (!isAny(dt) && dt.k !== 'INTEGER') this.error('type', loop.decreases, `DECREASES needs a whole number that gets smaller on every pass, and ${sourceOf(loop.decreases.expr)} is ${article(dt)}.`, { guide: 'contracts' }); + } + this.block(body, inner); + (this.ctx as BodyCtx).loops.pop(); + } + + forStmt(s: ForStmt, scope: Scope): void { + for (const [e, what] of [[s.start, 'start'], [s.stop, 'stop'], [s.step, 'step']] as Array<[Expr | null, string]>) { + if (!e) continue; + const t = this.expr(e, scope); + if (!isAny(t) && t.k !== 'INTEGER') { + this.error('type', e, `A FOR counter counts in whole numbers, and its ${what} value ${sourceOf(e)} is ${article(t)}.${t.k === 'FLOAT' ? ' Use INT or ROUND, or a WHILE loop for a FLOAT that changes.' : ''}`, { guide: 'for-when-you-know-how-many-times' }); + } + } + if (s.step && s.step.kind === 'Num' && s.step.value === 0) this.error('zero-step', s.step, 'STEP 0 never moves the counter, so the loop would never end.'); + if (s.var.startsWith('<')) { this.blank(s, s.var.slice(1, -1)); this.loopBody(s, scope); return; } + const existing = scope.get(s.var); + if (existing && (existing.kind === 'counter' || existing.kind === 'each')) { + this.error('counter-reuse', s, `${s.var} is already the counter of the loop on line ${existing.line}. A nested loop needs a counter of its own, such as J.`, { guide: 'nested-loops' }); + } + if (existing && existing.kind === 'var') { + if (!isAny(existing.type) && existing.type.k !== 'INTEGER') this.error('type', s, `${s.var} is declared ${typeName(existing.type)}, and a FOR counter holds whole numbers.`); + s.declaredCounter = existing; + } else if (existing && existing.kind === 'param') { + this.error('counter-param', s, `${s.var} is a parameter, and a FOR loop needs a counter of its own.`); + } else if (existing && (existing.kind === 'const' || existing.kind === 'define')) { + this.error('counter-const', s, `${s.var} is a constant, and a FOR loop needs a counter it can change.`); + } + const inner = this.childScope(scope); + const prior = inner.map.get(up(s.var)); + const sym: CounterSym = { name: s.var, type: T.INTEGER, kind: 'counter', line: s.line, col: s.varCol, readOnly: true, loop: s }; + inner.map.set(up(s.var), sym); + s.counterSym = sym; + this.loopBodyWith(s, inner); + if (prior) inner.map.set(up(s.var), prior); else inner.map.delete(up(s.var)); + } + + /** loopBody, with the counter already in scope for the invariants. */ + loopBodyWith(loop: ForStmt | ForEachStmt, inner: Scope): void { + loop.invariants = []; + loop.decreases = null; + const body = loop.body; + let i = 0; + while (i < body.length && (body[i].kind === 'Invariant' || body[i].kind === 'Decreases')) { + if (body[i].kind === 'Invariant') loop.invariants.push(body[i] as ContractStmt); + else loop.decreases = body[i] as ContractStmt; + i++; + } + body.splice(0, i); + (this.ctx as BodyCtx).loops.push(loop); + for (const inv of loop.invariants) this.condition(inv.expr, inner, 'INVARIANT'); + if (loop.decreases) this.expr(loop.decreases.expr, inner); + this.block(body, inner); + (this.ctx as BodyCtx).loops.pop(); + } + + forEach(s: ForEachStmt, scope: Scope): void { + let elem: PType | null = T.ANY; + if (s.iterable) { + const t = this.expr(s.iterable, scope, { rangeOk: true }); + if (t.k === 'ARRAY' && t.dims === 2) this.error('type', s.iterable, 'FOR EACH visits a one-dimensional collection. For a two-dimensional ARRAY, nest two FOR loops over ROWS and COLUMNS.'); + else if (!isAny(t)) { + elem = elementOf(t); + if (!elem) { this.error('type', s.iterable, `FOR EACH visits the values of a STRING, LIST, ARRAY, SET, or HASHMAP, and ${sourceOf(s.iterable)} is ${article(t)}.`); elem = T.ANY; } + } + if (t.k === 'SET' || t.k === 'HASHMAP') s.unordered = true; + } + const existing = scope.get(s.var); + if (existing && (existing.kind === 'var' || existing.kind === 'param')) this.error('each-shadow', s, `${s.var} is already a ${existing.kind === 'param' ? 'parameter' : 'variable'} (line ${existing.line}). FOR EACH needs a fresh name for the value it visits.`); + const inner = this.childScope(scope); + const prior = inner.map.get(up(s.var)); + const sym: EachSym = { name: s.var, type: elem, kind: 'each', line: s.line, col: s.varCol, readOnly: true, loop: s }; + inner.map.set(up(s.var), sym); + s.eachSym = sym; + this.loopBodyWith(s, inner); + if (prior) inner.map.set(up(s.var), prior); else inner.map.delete(up(s.var)); + } + + caseStmt(s: CaseStmt, scope: Scope): void { + const st = s.subject ? this.expr(s.subject, scope) : T.ANY; + const seen = new Map(); + for (const arm of s.arms) { + for (const lab of arm.labels) { + if (lab.value) { + const lt = this.expr(lab.value, scope); + this.caseLabelType(st, lt, lab.value, s); + if (['Num', 'Str', 'Char', 'Bool'].includes(lab.value.kind)) { + const key = `${lab.value.kind}:${(lab.value as NumExpr | StrExpr | CharExpr | BoolExpr).value}`; + if (seen.has(key)) this.warn('case-duplicate', lab.value, `${sourceOf(lab.value)} already has a branch on line ${seen.get(key)}, and CASE runs only the first branch that matches, so this one never runs.`, { guide: 'case' }); + else seen.set(key, arm.line); + } + } else { + const a = this.expr(lab.low, scope); + const b = this.expr(lab.high, scope); + this.caseLabelType(st, a, lab.low, s); + this.caseLabelType(st, b, lab.high, s); + if (!isAny(st) && !isNumeric(st) && st.k !== 'CHAR') this.error('type', lab.low, `A range in a CASE needs values that come in order, such as numbers, and the CASE tests ${article(st)}.`); + } + } + this.block(arm.body, scope); + } + if (s.otherwise) this.block(s.otherwise, scope); + } + + caseLabelType(subjectType: PType, labelType: PType, node: Expr, s: CaseStmt): void { + if (isAny(subjectType) || isAny(labelType)) return; + if (isNumeric(subjectType) && isNumeric(labelType)) return; + if (!sameType(subjectType, labelType)) { + // typed-port: `node` is any label, not only a literal; for a name or an expression its `value` is undefined, and the + // hint reads "write 'undefined'". + const extra = subjectType.k === 'CHAR' && labelType.k === 'STRING' + ? ` Single quotes make a CHAR: write '${(node as { value?: unknown }).value}'.` + : subjectType.k === 'STRING' && labelType.k === 'CHAR' ? ` Double quotes make a STRING: write "${(node as { value?: unknown }).value}".` : ''; + this.error('type', node, `CASE OF ${sourceOf(s.subject)} tests ${article(subjectType)}, and ${sourceOf(node)} is ${article(labelType)}, so this branch can never match.${extra}`, { guide: 'the-types' }); + } + } + + returnStmt(s: ReturnStmt, scope: Scope): void { + const sub = (this.ctx as BodyCtx).sub; + if (!sub) { this.error('return-main', s, 'RETURN belongs inside a FUNCTION or PROCEDURE. The main program ends at its last line.', { guide: 'function' }); if (s.value) this.expr(s.value, scope); return; } + if (sub.kind === 'Procedure') { + if (s.value) { this.error('procedure-value', s, `A PROCEDURE hands nothing back, so its RETURN takes no value. ${sub.name} would need to be a FUNCTION to return ${sourceOf(s.value)}.`, { guide: 'procedure' }); this.expr(s.value, scope); } + return; + } + if (!s.value) { this.error('return-value', s, `FUNCTION ${sub.name} has to hand back ${article(sub.returns)}, so its RETURN needs a value.`, { guide: 'function' }); return; } + const t = this.adopt(s.value, sub.returns, this.expr(s.value, scope)); + if (!assignable(sub.returns, t)) this.error('type', s.value, `FUNCTION ${sub.name} promises ${article(sub.returns)} with RETURNS, and ${sourceOf(s.value)} is ${article(t)}.${this.narrowHint(sub.returns!, t, s.value)}`, { guide: 'function' }); + else this.arrayLength(sub.returns, s.value, t, `The RESULT of ${sub.name}`); + } + + /** + * What a SET holds. A run finds a member by its value only for a single value (as a HASHMAP finds a key): a LIST, + * an ARRAY, a record or a collection would be found by identity, and could change after it was added (interp + * audit C5, prover audit O). So a SET's members are the types a HASHMAP key may be. + */ + member(of: PType, at: Partial | null | undefined): PType { + if (of && !['INTEGER', 'CHAR', 'STRING', 'BOOL', 'FLOAT', 'ANY'].includes(of.k)) { + this.error('bad-member', at, `A SET holds single values: INTEGER, CHAR, STRING, BOOL, or FLOAT, not ${typeName(of)}. To keep a collection of ${typeName(of)} values, use a LIST OF ${typeName(of)}.`, { guide: 'set-for-membership' }); + } + return of; + } + + /** + * A value going where an ARRAY of known lengths is declared (a store, an argument, a RETURN, a value APPENDed to + * a LIST OF ARRAY): when the value's lengths are known too (a list written out, or anything typed ARRAY[3]) and + * differ, it is an error here; otherwise the run checks (interp.js `fitNumber`). + */ + arrayLength(to: PType | null | undefined, value: Expr | null | undefined, t: PType | null | undefined, what: string): void { + const want = knownDims(to); + if (!want || !value) return; + const have = value.kind === 'ListLit' ? [value.items.length] : knownDims(t); + if (!have || (have.length === want.length && have.every((n, i) => n === want[i]))) return; + this.error('array-length', value, `${what} is an ARRAY of ${want.join(' by ')} positions, and ${sourceOf(value)} has ${have.join(' by ')}. An ARRAY keeps the length it was declared with; a LIST is the collection that grows and shrinks.`, { guide: 'array-when-you-know-how-many' }); + } + + narrowHint(to: PType, from: PType, value: Expr | null | undefined): string { + if (to.k === 'INTEGER' && from.k === 'INTEGER' && to.bits && from.bits) return ` Two different sized types do not mix silently: convert with ${to.name}(${sourceOf(value)}), which stops the run if the value does not fit.`; + if (isFloat32(to) && from.k === 'FLOAT') return ` A FLOAT64 rounds when it becomes a FLOAT32, so the conversion is written out: FLOAT32(${sourceOf(value)}).`; + if (to.k === 'INTEGER' && from.k === 'FLOAT') { + if (value && value.kind === 'Binary' && value.op === '/') return ' / always gives a FLOAT, even when both sides are whole. Use DIV for a whole-number answer.'; + return ' Use INT or ROUND to turn a FLOAT into a whole number, and decide which one you mean.'; + } + if (to.k === 'CHAR' && from.k === 'STRING' && value?.kind === 'Str' && Array.from(value.value).length === 1) return ` Single quotes make a CHAR: '${value.value}'.`; + return ''; + } + + assign(s: AssignStmt, scope: Scope): void { + const tt = this.target(s.target, scope, 'assign'); + const vt = this.adopt(s.value, tt, this.expr(s.value, scope, { expect: tt })); + if (!tt || isAny(tt) || isAny(vt)) return; + if (vt.k === 'VOID') { this.error('type', s.value, `${sourceOf(s.value)} hands nothing back, so there is nothing to store.`); return; } + if (tt.k === 'STRING' && vt.k === 'CHAR') return; + if (!assignable(tt, vt)) { + this.error('type', s.value, `${sourceOf(s.target)} holds ${typeName(tt)} values, and ${sourceOf(s.value)} is ${article(vt)}.${this.narrowHint(tt, vt, s.value)}`, { guide: 'the-types' }); + } else this.arrayLength(tt, s.value, vt, sourceOf(s.target)); + } + + /** Something that can receive a value; returns its type. */ + target(e: Expr | null, scope: Scope, why: string): PType { + if (!e) return T.ANY; + if (e.kind === 'Name') { + const sym = this.lookup(e, scope, why === 'assign' ? 'assign' : 'read'); + if (!sym) return T.ANY; + if (sym.kind === 'counter') this.error('counter-assign', e, `The FOR loop on line ${sym.line} owns ${sym.name}, and changing it inside the loop hides the loop's shape from a reader. Use a WHILE loop if the counter has to jump.`, { guide: 'for-when-you-know-how-many-times' }); + else if (sym.kind === 'each') this.error('counter-assign', e, `${sym.name} is the value FOR EACH is visiting, and it cannot be changed from inside the loop.`); + else if (sym.kind === 'const') this.error('const-assign', e, `${sym.name} is a CONSTANT, so its value cannot change.`, { guide: 'constants' }); + else if (sym.kind === 'define') this.error('const-assign', e, `${sym.name} is a set made with DEFINE, so its members are fixed.`); + e.type = sym.type; + return sym.type; + } + if (e.kind === 'Index') { + const t = this.expr(e, scope, { target: true }); + const base = this.rootType(e.target, scope); + if (base && base.k === 'STRING' && e.target.type?.k === 'STRING') { + this.error('string-write', e, `A STRING cannot be changed one position at a time. Build a new one with &, or keep the letters in a LIST OF CHAR while you change them.`, { guide: 'text-and-positions' }); + } + this.rootWritable(e, scope); + return t; + } + if (e.kind === 'Field') { const t = this.expr(e, scope); this.rootWritable(e, scope); return t; } + if (e.kind === 'Blank') { this.blank(e, e.text); return T.ANY; } + this.error('bad-target', e, `${sourceOf(e)} cannot receive a value. Only a name, a position such as T[I], or a field such as R.Name can.`); + return T.ANY; + } + + rootType(e: Expr, _scope?: Scope): PType | undefined { return e.type; } + + rootWritable(e: Expr, scope: Scope): void { + let r = e; + while (r.kind === 'Index' || r.kind === 'Field') r = r.target; + if (r.kind === 'Name') { + const sym = scope.get(r.name); + if (sym && (sym.kind === 'const' || sym.kind === 'define')) this.error('const-assign', r, `${sym.name} is fixed, so no part of it can change.`); + if (sym && sym.kind === 'each') this.error('counter-assign', r, `${sym.name} is the value FOR EACH is visiting. Changing it would not change the collection.`); + } + } + + lookup(e: NameExpr, scope: Scope, how = 'read'): Sym | null { + const name = e.name; + if (name.startsWith('<')) { this.blank(e, name.slice(1, -1)); return null; } + const sym = scope.get(name); + if (sym) { + if (sym.name !== name && sym.kind !== 'bound') { + this.note('name-case', e, `${name} is written ${sym.name} where it was ${sym.kind === 'param' ? 'named' : 'declared'} on line ${sym.line}. Names ignore case, so this is the same name; pick one spelling and hold it.`, { guide: 'names' }); + } + if (sym.kind === 'var' && sym.line > e.line && !this.ctx?.loops?.length) { + this.error('before-declare', e, `${name} is used on line ${e.line} and declared on line ${sym.line}. Declare a variable before using it.`, { guide: 'declaring' }); + } + e.sym = sym; + return sym; + } + if (this.ctx?.inEnsure && up(name) === 'RESULT' && this.ctx.sub?.kind === 'Function') { + e.sym = { name: 'RESULT', type: this.ctx.sub.returns!, kind: 'result' }; + return e.sym; + } + // A function reaching for a name in the main program. + const mainSym = this.mainScope.own(name); + if (this.ctx?.sub && mainSym) { + this.error('outer-name', e, `${name} belongs to the main program, and ${this.ctx.sub.name} can see only its own parameters and variables. Pass ${name} in as a parameter.`, { guide: 'function' }); + e.sym = mainSym; + return mainSym; + } + if (this.findSub(name)) { this.error('sub-as-value', e, `${name} is a ${this.findSub(name)!.kind.toUpperCase()}, and it needs brackets and its arguments to be called.`); return null; } + if (up(name) === 'RESULT') { this.error('result-place', e, 'RESULT means the value a FUNCTION returns, so it can appear only in that function\'s ENSURE lines.', { guide: 'contracts' }); return null; } + const pool = [...scope.all().values()].map((x) => x.name); + const s = suggest(name, pool); + const verb = how === 'assign' ? 'Declare it before assigning to it' : 'Declare it, and give it a value, before reading it'; + this.error('undeclared', e, `${name} has not been declared.${s.length ? ` Did you mean ${s.join(' or ')}?` : ` ${verb}: DECLARE ${name} : .`}`, { guide: 'declaring' }); + return null; + } + + condition(e: Expr | null, scope: Scope, word: string): void { + if (!e) return; + const t = this.expr(e, scope); + if (isAny(t) || t.k === 'BOOL') return; + let hint = ''; + if (isNumeric(t)) hint = ` Compare it with something, as in ${sourceOf(e)} > 0.`; + if (e.kind === 'Binary' && e.op === '<-') hint = ' <- stores a value; == compares two.'; + this.error('not-bool', e, `${word} needs a condition that is TRUE or FALSE, and ${sourceOf(e)} is ${article(t)}.${hint}`, { guide: 'comparison' }); + } + + callStmt(s: CallStmt, scope: Scope): void { + const sub = this.findSub(s.name); + if (!sub) { + if (BUILTINS[up(s.name)]?.kind === 'procedure') { + this.note('builtin-call', s, `${up(s.name)} is a built-in procedure and needs no CALL: write ${up(s.name)}(...) on its own.`); + this.builtinStmt({ ...s, kind: 'BuiltinCall', name: up(s.name) }, scope); + s.builtin = true; + return; + } + const hint = suggest(s.name, [...this.subs.values(), ...this.libs.values()].map((x) => x.name)); + this.error('unknown-sub', s, `No PROCEDURE is called ${s.name}.${hint.length ? ` Did you mean ${hint.join(' or ')}?` : ''}`, { guide: 'procedure' }); + for (const a of s.args) this.expr(a, scope); + return; + } + if (sub.kind === 'Function') { + this.error('call-function', s, `${sub.name} is a FUNCTION, so its answer would be thrown away. A function call stands where a value stands: X <- ${sub.name}(...).`, { guide: 'procedure' }); + } + s.sub = sub; + this.args(sub, s.args, s, scope); + } + + args(sub: Sub, args: Expr[], at: Loc, scope: Scope): void { + // Two BYREF parameters bound to one variable would be two names for one + // box, and every claim about either one would stop meaning what it says. + const byref = new Map(); + args.forEach((a, i) => { + const p = sub.params[i]; + if (!p || p.mode !== 'BYREF' || a.kind !== 'Name') return; + const k = up(a.name); + if (byref.has(k)) this.error('byref-alias', a, `${a.name} is passed to both ${byref.get(k)} and ${p.name}, which are BYREF. Each BYREF parameter needs a variable of its own, or a change through one name would silently change the other.`, { guide: 'passing-by-value-or-by-reference' }); + else byref.set(k, p.name); + }); + if (args.length !== sub.params.length) { + this.error('arg-count', at, `${sub.name} takes ${sub.params.length} ${sub.params.length === 1 ? 'value' : 'values'} (${sub.params.map((p) => p.name).join(', ') || 'none'}), and this call gives it ${args.length}.`, { guide: 'function' }); + } + args.forEach((a, i) => { + const p = sub.params[i]; + let t: PType | undefined = this.expr(a, scope); + if (!p) return; + if (p.mode !== 'BYREF') t = this.adopt(a, p.type); + if (p.mode === 'BYREF') { + if (a.kind !== 'Name') this.error('byref-arg', a, `Parameter ${p.name} of ${sub.name} is BYREF, so the call has to pass a variable it can change, not ${sourceOf(a)}.`, { guide: 'passing-by-value-or-by-reference' }); + else { + const sym = a.sym; + if (sym && (sym.kind === 'const' || sym.kind === 'define' || sym.kind === 'counter' || sym.kind === 'each')) this.error('byref-arg', a, `${sym.name} cannot be changed, so it cannot be passed to the BYREF parameter ${p.name}.`); + if (!isAny(t) && !isAny(p.type) && !sameType(t, p.type)) this.error('type', a, `BYREF parameter ${p.name} is ${article(p.type)}, and ${sourceOf(a)} is ${article(t)}. A BYREF argument has to match exactly, since the procedure writes back into it.`); + else this.arrayLength(p.type, a, t, `Parameter ${p.name} of ${sub.name}`); + } + return; + } + if (!isAny(t) && !isAny(p.type) && !assignable(p.type, t) && !(p.type.k === 'STRING' && t.k === 'CHAR')) { + const extra = p.type.k === 'CHAR' && t.k === 'STRING' && a.kind === 'Str' && Array.from(a.value).length === 1 ? ` Single quotes make a CHAR: '${a.value}'.` : ''; + this.error('type', a, `Parameter ${p.name} of ${sub.name} is ${article(p.type)}, and ${sourceOf(a)} is ${article(t)}.${extra}`, { guide: 'function' }); + } else this.arrayLength(p.type, a, t, `Parameter ${p.name} of ${sub.name}`); + }); + } + + exprStmt(s: ExprStmt, scope: Scope): void { + const e = s.expr; + if (e.kind === 'CallExpr') { + const sub = e.convert ? null : this.findSub(e.name); + if (sub && sub.kind === 'Procedure') { + this.error('missing-call', s, `Call a PROCEDURE with CALL, on a line of its own: CALL ${sourceOf(e)}.`, { guide: 'procedure', fix: { line: s.line, col: s.col, endCol: s.col, text: 'CALL ' } }); + s.sub = sub; + this.args(sub, e.args, e, scope); + return; + } + if (sub && sub.kind === 'Function') { + this.error('call-alone', s, `${sub.name} is a FUNCTION: a call stands where a value stands, so it goes inside an expression, and never on a line of its own. Store its answer, as in X <- ${sourceOf(e)}, or OUTPUT it.`, { guide: 'function' }); + this.expr(e, scope); + return; + } + if (BUILTINS[up(e.name)]?.kind === 'function') { + this.error('call-alone', s, `${up(e.name)} computes a value, and on a line of its own that value is thrown away. Store it or OUTPUT it.`); + this.expr(e, scope); + return; + } + this.expr(e, scope); + return; + } + this.error('bare-value', s, `${sourceOf(e)} on a line of its own does nothing.`); + } + + builtinStmt(s: BuiltinCallStmt, scope: Scope): void { + const name = s.name; + const spec = BUILTINS[name]; + const [first, ...rest] = s.args; + const need = ({ APPEND: 2, INSERT: 3, REMOVE: 2, ADD: 2, DELETE: 2, SORT: 1 } as Record)[name]; + if (s.args.length !== need) { + this.error('arg-count', s, `${spec.sig} takes ${need} ${need === 1 ? 'value' : 'values'}, and this line gives it ${s.args.length}.`); + for (const a of s.args) this.expr(a, scope); + return; + } + if (!['Name', 'Index', 'Field'].includes(first.kind)) { + this.error('bad-target', first, `${name} changes its first argument, so that has to be a variable, not ${sourceOf(first)}.`); + for (const a of rest) this.expr(a, scope); + return; + } + const ct = this.target(first, scope, 'mutate'); + const types = rest.map((a) => this.expr(a, scope)); + if (isAny(ct)) return; + const want = (kinds: string[], what: string): boolean => { + if (!kinds.includes(ct.k)) { + let extra = ''; + if (ct.k === 'ARRAY' && name === 'APPEND') extra = ' An ARRAY\'s length is fixed when it is declared. Use a LIST when you do not know how many values are coming.'; + if (ct.k === 'STRING' && name === 'APPEND') extra = ' Join text with &: T <- T & \'A\'.'; + this.error('type', first, `${name} works on ${what}, and ${sourceOf(first)} is ${article(ct)}.${extra}`, { guide: 'collections' }); + return false; + } + return true; + }; + const checkElem = (t0: PType, node: Expr, of: PType | undefined, role: string): void => { + const t = this.adopt(node, of, t0); + if (!isAny(t) && !isAny(of) && !assignable(of, t) && !(of.k === 'STRING' && t.k === 'CHAR')) this.error('type', node, `${sourceOf(first)} holds ${typeName(of)} values, and the ${role} ${sourceOf(node)} is ${article(t)}.`); + else this.arrayLength(of, node, t, `Each value of ${sourceOf(first)}`); + }; + const checkIndex = (t: PType, node: Expr): void => { if (!isAny(t) && t.k !== 'INTEGER') this.error('type', node, `A position is a whole number, and ${sourceOf(node)} is ${article(t)}.`); }; + switch (name) { + case 'APPEND': if (want(['LIST'], 'a LIST')) checkElem(types[0], rest[0], ct.of, 'value'); break; + case 'INSERT': if (want(['LIST'], 'a LIST')) { checkIndex(types[0], rest[0]); checkElem(types[1], rest[1], ct.of, 'value'); } break; + case 'REMOVE': + if (ct.k === 'SET' || ct.k === 'HASHMAP') { this.error('type', s, `REMOVE takes a position out of a LIST. To take ${ct.k === 'SET' ? 'a member out of a SET' : 'a key out of a HASHMAP'}, use DELETE.`); break; } + if (want(['LIST'], 'a LIST')) checkIndex(types[0], rest[0]); + break; + case 'ADD': if (want(['SET'], 'a SET')) checkElem(types[0], rest[0], ct.of, 'member'); break; + case 'DELETE': + if (ct.k === 'LIST') { this.error('type', s, 'DELETE works on a SET or a HASHMAP. To take a position out of a LIST, use REMOVE.'); break; } + if (want(['SET', 'HASHMAP'], 'a SET or a HASHMAP')) checkElem(types[0], rest[0], ct.k === 'SET' ? ct.of : ct.key, ct.k === 'SET' ? 'member' : 'key'); + break; + case 'SORT': + if (want(['LIST', 'ARRAY'], 'a LIST or a one-dimensional ARRAY') && ct.k === 'ARRAY' && ct.dims === 2) this.error('type', first, 'SORT works on a one-dimensional collection.'); + break; + default: break; + } + s.targetType = ct; + } + + /* ---------- expressions ---------- */ + expr(e: Expr, scope: Scope, opts: ExprOpts = {}): PType { + const t = this.exprInner(e, scope, opts) || T.ANY; + e.type = t; + return t; + } + + exprInner(e: Expr, scope: Scope, opts: ExprOpts): PType | null { + if (!e) return T.ANY; + switch (e.kind) { + case 'Num': + // a literal no run could hold is an error before any run: a whole number past 2^53 - 1 would be rounded on + // the way in (9007199254740993 reads as ...992), and a FLOAT past about 1.8e308 is infinity + if (!e.rangeSaid && (e.float ? !Number.isFinite(e.value) : Math.abs(e.value) > Number.MAX_SAFE_INTEGER)) { + e.rangeSaid = true; + this.error('literal-range', e, e.float + ? 'This number is beyond the largest FLOAT, about 1.8 × 10^308.' + : 'This whole number is past 9,007,199,254,740,991, the largest this lab holds exactly, so it could not be read without rounding it.', { guide: 'sized-types' }); + } + return e.float ? T.FLOAT : T.INTEGER; + case 'Str': return T.STRING; + case 'Char': return T.CHAR; + case 'Bool': return T.BOOL; + case 'Blank': if (e.text !== '...') this.blank(e, e.text); return T.ANY; + case 'Error': return T.ANY; + case 'TypeRef': + if (!opts.typeArg) { this.error('type-as-value', e, `${e.name} is a type, and a value was expected here. To convert a number to it, write ${e.name}(X)${['INTEGER', 'FLOAT'].includes(e.name) ? ` with a sized type, or ${e.name === 'INTEGER' ? 'INT(X)' : 'TO_FLOAT(X)'}` : ''}.`, { guide: 'the-types' }); return T.ANY; } + return T[e.name] || T.ANY; + case 'Name': { + if (this.ctx?.constOnly) { + const sym = scope.get(e.name); + if (!sym || (sym.kind !== 'const' && sym.kind !== 'define')) { this.error('const-expr', e, `A CONSTANT's value can use only literals and other constants, and ${e.name} is not one.`, { guide: 'constants' }); return T.ANY; } + e.sym = sym; + return sym.type; + } + const sym = this.lookup(e, scope); + return sym ? sym.type : T.ANY; + } + case 'Unary': { + const t = this.expr(e.arg, scope); + if (e.op === 'NOT') { + if (!isAny(t) && t.k !== 'BOOL') this.error('type', e, `NOT needs a TRUE-or-FALSE value, and ${sourceOf(e.arg)} is ${article(t)}.`); + return T.BOOL; + } + if (!isAny(t) && !isNumeric(t)) this.error('type', e, `Only a number can be negated, and ${sourceOf(e.arg)} is ${article(t)}.`); + return isAny(t) ? T.ANY : t; + } + case 'Binary': return this.binary(e, scope, opts); + case 'Chain': { + for (let i = 0; i < e.ops.length; i++) { + const fake: BinaryExpr = { kind: 'Binary', op: e.ops[i], left: e.operands[i], right: e.operands[i + 1], line: e.line, col: e.col }; + this.binary(fake, scope, opts, i > 0); + } + return T.BOOL; + } + case 'Index': return this.index(e, scope); + case 'Field': { + const t = this.expr(e.target, scope); + if (isAny(t)) return T.ANY; + if (t.k !== 'RECORD') { this.error('type', e, `A dot reaches a field of a record, and ${sourceOf(e.target)} is ${article(t)}.`, { guide: 'record-for-grouping-related-fields' }); return T.ANY; } + const f = t.fields.get(up(e.field)); + if (!f) { const s = suggest(e.field, t.fieldNames); this.error('unknown-field', e, `${t.name} has no field called ${e.field}.${s.length ? ` Did you mean ${s.join(' or ')}?` : ` Its fields are ${t.fieldNames.join(', ')}.`}`); return T.ANY; } + return f; + } + case 'CallExpr': return this.call(e, scope); + case 'ListLit': { + let el: PType = T.ANY; + const want = opts.expect && ['LIST', 'ARRAY'].includes(opts.expect.k) ? opts.expect.of : null; + for (const it of e.items) { + const t = this.adopt(it, want, this.expr(it, scope)); + if (isAny(el)) el = t; + else if (isNumeric(el) && isNumeric(t)) { + if (isSizedInt(el) && isSizedInt(t) && !sameType(el, t)) this.error('mixed-width', it, `${sourceOf(it)} is ${t.name} where the values before it are ${el.name}, and two different sized types do not mix silently. Convert one so they agree, as in ${el.name}(${sourceOf(it)}).`, { guide: 'sized-types' }); + el = numericResult(el, t); + } + else if (!isAny(t) && !sameType(el, t)) this.error('type', it, `Every value in a list has one type, and ${sourceOf(it)} is ${article(t)} where the others are ${typeName(el)}.`); + } + // A list of whole-number literals going into a LIST OF FLOAT is a list of FLOAT literals from here on, as 3 + // stored in a FLOAT is 3.0: L <- [1, 2] reads, and the translators write 1.0, which is what a List[Float64] + // in Mojo holds (library/pseudo/programs/54-audit-rules.pseudo). A list that already has a FLOAT in it is + // a LIST OF FLOAT as written and stays as written. + if (want && want.k === 'FLOAT' && !isFloat32(want) && el.k === 'INTEGER' && !el.bits && e.items.length && e.items.every((it) => isLiteral(it) && !it.float)) { + for (const it of e.items) { (it as NumExpr).float = true; it.type = T.FLOAT; } + el = T.FLOAT; + } + if (e.maybeRange && !opts.quiet) this.note('closed-range', e, `[${sourceOf(e.items[0])}, ${sourceOf(e.items[1])}] is a LIST of two values. A range of positions is half-open and ends with a round bracket: [${sourceOf(e.items[0])}, ${sourceOf(e.items[1])}).`, { guide: 'text-and-positions' }); + return T.list(el); + } + case 'SetLit': { + let el: PType = T.ANY; + for (const it of e.items) { + const t = this.expr(it, scope); + if (isAny(el)) el = t; + else if (isNumeric(el) && isNumeric(t)) el = numericResult(el, t); // {1, 2.5} is a SET OF FLOAT, as [1, 2.5] is a LIST OF FLOAT + else if (!isAny(t) && !sameType(el, t)) this.error('type', it, `Every member of a set has one type, and ${sourceOf(it)} is ${article(t)}.`); + } + return T.set(this.member(el, e)); + } + case 'MapLit': { + let kt: PType = T.ANY; let vt: PType = T.ANY; + for (const x of e.entries) { + const a = this.expr(x.key, scope); const b = this.expr(x.value, scope); + if (isAny(kt)) kt = a; else if (!isAny(a) && !sameType(kt, a)) this.error('type', x.key, `Every key of a HASHMAP has one type, and ${sourceOf(x.key)} is ${article(a)}.`); + if (isAny(vt)) vt = b; else if (isNumeric(vt) && isNumeric(b)) vt = numericResult(vt, b); else if (!isAny(b) && !sameType(vt, b)) this.error('type', x.value, `Every value of a HASHMAP has one type, and ${sourceOf(x.value)} is ${article(b)}.`); + } + return T.map(kt, vt); + } + case 'Range': { + const a = this.expr(e.lo, scope); const b = this.expr(e.hi, scope); + for (const [x, n] of [[a, e.lo], [b, e.hi]] as Array<[PType, Expr]>) if (!isAny(x) && x.k !== 'INTEGER') this.error('type', n, `The ends of a range of positions are whole numbers, and ${sourceOf(n)} is ${article(x)}.`); + if (!opts.rangeOk) this.error('range-place', e, `A range such as [${sourceOf(e.lo)}, ${sourceOf(e.hi)}) can follow IN, as in I IN [0, N), and cannot stand on its own as a value.`); + return T.range(); + } + case 'SetBuilder': case 'Quant': return this.binder(e, scope); + default: return T.ANY; + } + } + + binder(e: SetBuilderExpr | QuantExpr, scope: Scope): PType { + const dt = e.domain ? this.expr(e.domain, scope, { rangeOk: true, quiet: false }) : T.ANY; + let vt: PType | null = T.ANY; + if (!isAny(dt)) { + vt = elementOf(dt); + if (!vt) { this.error('type', e.domain, `${e.kind === 'SetBuilder' ? 'A set written by rule' : e.q} ranges over a range such as [0, N), or over a collection, and ${sourceOf(e.domain)} is ${article(dt)}.`); vt = T.ANY; } + } + const inner = new Scope(scope, scope.owner); + inner.set(e.var, { name: e.var, type: vt, kind: 'bound', line: e.line, col: e.col, readOnly: true }); + if (scope.get(e.var) && scope.get(e.var)!.kind !== 'bound') this.warn('bound-shadow', e, `${e.var} is already a name here, and inside the ${e.kind === 'SetBuilder' ? 'set' : e.q} it means something else. A fresh name, such as K, reads more clearly.`); + const bt = this.expr(e.body, inner); + if (e.kind === 'SetBuilder' || e.q === 'FORALL' || e.q === 'EXISTS') { + if (!isAny(bt) && bt.k !== 'BOOL') this.error('type', e.body, `The test after the colon has to be TRUE or FALSE, and ${sourceOf(e.body)} is ${article(bt)}.`); + return e.kind === 'SetBuilder' ? T.set(this.member(vt, e)) : T.BOOL; + } + // SUM + if (!isAny(bt) && !isNumeric(bt)) this.error('type', e.body, `SUM adds numbers, and ${sourceOf(e.body)} is ${article(bt)}.`); + return isAny(bt) ? T.ANY : bt; + } + + binary(e: BinaryExpr, scope: Scope, opts: ExprOpts, leftDone = false): PType { + const op = e.op; + const lt = leftDone ? (e.left.type || T.ANY) : this.expr(e.left, scope); + const rt = this.expr(e.right, scope, { rangeOk: op === 'IN' }); + const any = isAny(lt) || isAny(rt); + switch (op) { + case 'AND': case 'OR': + for (const [t, n] of [[lt, e.left], [rt, e.right]] as Array<[PType, Expr]>) { + if (!isAny(t) && t.k !== 'BOOL') this.error('type', n, `${op} joins two TRUE-or-FALSE values, and ${sourceOf(n)} is ${article(t)}.${isText(t) || isNumeric(t) ? ` Each side needs its own comparison: ${op === 'OR' ? 'Base == \'G\' OR Base == \'C\'' : 'X > 0 AND X < 10'}.` : ''}`, { guide: 'compound-conditions' }); + } + return T.BOOL; + case '+': case '-': case '*': + if (any) return isNumeric(lt) ? lt : isNumeric(rt) ? rt : T.ANY; + if (isNumeric(lt) && isNumeric(rt)) return this.mix(e, lt, rt); + if (op === '+' && isText(lt) && isText(rt)) { this.error('type', e, `Text is joined with &, not +: ${sourceOf(e.left)} & ${sourceOf(e.right)}.`, { guide: 'joining-and-cutting-text' }); return T.STRING; } + if (!isNumeric(lt) || !isNumeric(rt)) { this.error('type', e, `${op} works on numbers, and ${!isNumeric(lt) ? sourceOf(e.left) : sourceOf(e.right)} is ${article(!isNumeric(lt) ? lt : rt)}.${isText(lt) && isNumeric(rt) ? ' TO_STRING turns a number into text, and & joins text.' : ''}`); return T.ANY; } + return numericResult(lt, rt); + case '/': + if (!any && (!isNumeric(lt) || !isNumeric(rt))) { this.error('type', e, `/ divides numbers, and ${!isNumeric(lt) ? sourceOf(e.left) : sourceOf(e.right)} is ${article(!isNumeric(lt) ? lt : rt)}.`); } + if (!any && isNumeric(lt) && isNumeric(rt) && (isFloat32(lt) || isFloat32(rt))) { const m = this.mix(e, lt, rt); return isFloat32(m) ? m : T.FLOAT; } + return lt.bits === 64 || rt.bits === 64 ? T.FLOAT64 : T.FLOAT; + case 'DIV': case 'MOD': + for (const [t, n] of [[lt, e.left], [rt, e.right]] as Array<[PType, Expr]>) { + if (!isAny(t) && t.k !== 'INTEGER') this.error('type', n, `${op} works on whole numbers, and ${sourceOf(n)} is ${article(t)}.${t.k === 'FLOAT' ? ' Use / to divide FLOAT values.' : ''}`, { guide: 'arithmetic' }); + } + if (!any && lt.k === 'INTEGER' && rt.k === 'INTEGER') return this.mix(e, lt, rt); + return T.INTEGER; + case '^': + if (!any && (!isNumeric(lt) || !isNumeric(rt))) { this.error('type', e, '^ raises a number to a power, and both sides have to be numbers.'); return T.ANY; } + if (any) return T.ANY; + // the power keeps the base's width: INT32 ^ 2 is INT32, whatever the exponent's type + if (lt.bits && rt.k === 'INTEGER') { this.adopt(e.right, lt.k === 'INTEGER' ? lt : null); return lt; } + return this.mix(e, lt, rt); + case '&': + for (const [t, n] of [[lt, e.left], [rt, e.right]] as Array<[PType, Expr]>) { + if (!isAny(t) && !isText(t)) this.error('type', n, `& joins text, and ${sourceOf(n)} is ${article(t)}. TO_STRING(${sourceOf(n)}) writes it as text first.`); + } + return T.STRING; + case '==': case '!=': + this.adoptPair(e, lt, rt); + this.comparable(e, lt, rt, true); + return T.BOOL; + case '<': case '<=': case '>': case '>=': + this.adoptPair(e, lt, rt); + this.comparable(e, lt, rt, false); + return T.BOOL; + case 'IN': { + if (any) return T.BOOL; + let el: PType | null; + if (rt.k === 'HASHMAP') el = rt.key; + else if (rt.k === 'STRING') { + if (lt.k === 'STRING') { this.error('type', e, `IN on a STRING asks whether one CHAR occurs in it. Finding a whole pattern inside a text is an algorithm, not an operator: see MATCH.`, { guide: 'putting-it-together' }); return T.BOOL; } + el = T.CHAR; + } else el = elementOf(rt); + if (!el) { this.error('type', e.right, `IN asks about membership of a SET, HASHMAP, LIST, ARRAY, STRING, or range, and ${sourceOf(e.right)} is ${article(rt)}.`); return T.BOOL; } + if (!isAny(el) && !(isNumeric(el) && isNumeric(lt)) && !sameType(el, lt)) { + const extra = el.k === 'CHAR' && lt.k === 'STRING' && e.left.kind === 'Str' ? ` Single quotes make a CHAR: '${e.left.value}'.` : el.k === 'STRING' && lt.k === 'CHAR' && e.left.kind === 'Char' ? ` Double quotes make a STRING: "${e.left.value}".` : ''; + this.error('type', e.left, `${sourceOf(e.right)} holds ${typeName(el)} ${rt.k === 'HASHMAP' ? 'keys' : 'values'}, and ${sourceOf(e.left)} is ${article(lt)}, so the answer is always FALSE.${extra}`, { guide: 'the-types' }); + } + return T.BOOL; + } + default: return T.ANY; + } + } + + adopt(node: Expr | null | undefined, want: PType | null | undefined, current: PType): PType; + adopt(node: Expr | null | undefined, want: PType | null | undefined, current?: PType): PType | undefined; + /** + * A numeric literal next to a sized value takes that value's type: X + 1 with an INT8 X is INT8 arithmetic, + * and 0.1 next to a FLOAT32 is the FLOAT32 nearest 0.1, as in Mojo. A whole-number literal that does not fit + * the type is an error here, before any run. Returns the node's type afterwards. + */ + adopt(node: Expr | null | undefined, want: PType | null | undefined, current: PType | undefined = node?.type): PType | undefined { + if (!isLiteral(node) || !want || isAny(want) || !isNumeric(want)) return current; + if (isSizedInt(want) && !node.float) { + const [lo, hi] = intRange(want); + if ((node.value < lo || node.value > hi) && !node.rangeSaid) this.error('literal-range', node, `${sourceOf(node)} does not fit ${want.name}, which holds ${intRangeText(want)}.`, { guide: 'sized-types' }); + node.type = want; + return want; + } + if (isFloat32(want)) { + if (Number.isFinite(node.value) && !Number.isFinite(Math.fround(node.value))) this.error('literal-range', node, `${sourceOf(node)} does not fit FLOAT32, whose largest value is about 3.4 × 10^38.`, { guide: 'sized-types' }); + node.type = want; + return want; + } + return current; + } + + /** In a binary operation, a literal on either side adopts the other side's sized type. */ + adoptPair(e: Pair, lt: PType, rt: PType): [PType, PType] { + if (isLiteral(e.left) && !isLiteral(e.right)) return [this.adopt(e.left, rt, lt), rt]; + if (isLiteral(e.right) && !isLiteral(e.left)) return [lt, this.adopt(e.right, lt, rt)]; + return [lt, rt]; + } + + /** The type of +, -, *, DIV, MOD on two numbers, after literals adopt; two widths that do not mix are an error. */ + mix(e: Pair, lt0: PType, rt0: PType): PType { + const [lt, rt] = this.adoptPair(e, lt0, rt0); + if (lt.k === 'INTEGER' && rt.k === 'INTEGER' && lt.bits && rt.bits && !sameType(lt, rt)) { + const [narrow, wide] = lt.bits <= rt.bits ? [e.left, rt] : [e.right, lt]; + this.error('mixed-width', e, `${sourceOf(e.left)} is ${lt.name} and ${sourceOf(e.right)} is ${rt.name}, and two different sized types do not mix silently. Convert one so both sides agree, as in ${wide.name}(${sourceOf(narrow)}).`, { guide: 'sized-types' }); + return T.ANY; + } + if (lt.k === 'FLOAT' && rt.k === 'FLOAT' && isFloat32(lt) !== isFloat32(rt)) { + const [s32, s64] = isFloat32(lt) ? [e.left, e.right] : [e.right, e.left]; + this.error('mixed-precision', e, `${sourceOf(s32)} is FLOAT32 and ${sourceOf(s64)} is a FLOAT64, and the two precisions do not mix silently. Write FLOAT32(${sourceOf(s64)}) to round it to single precision, or FLOAT64(${sourceOf(s32)}) to widen the other side.`, { guide: 'sized-types' }); + return T.ANY; + } + return numericResult(lt, rt); + } + + /** INT32(X), FLOAT32(X), ...: one number, converted to the named type. */ + conversion(e: CallExprNode, scope: Scope): PType { + const to = SIZED[up(e.name)]; + const ts = e.args.map((a) => this.expr(a, scope)); + if (ts.length !== 1) { this.error('arg-count', e, `${up(e.name)}(X) converts one number, and this call gives it ${ts.length}.`); return to; } + const t = ts[0]; + if (!isAny(t) && !isNumeric(t)) this.error('type', e.args[0], `${up(e.name)}(X) converts a number, and ${sourceOf(e.args[0])} is ${article(t)}.${isText(t) ? ' TO_INTEGER or TO_FLOAT reads a number out of text first.' : ''}`, { guide: 'sized-types' }); + else if (!isAny(t) && sameType(t, to) && t.name === to.name) this.note('already-type', e, `${sourceOf(e.args[0])} is already ${to.name}, so ${to.name}(...) changes nothing.`); + if (isLiteral(e.args[0]) && to.k === 'INTEGER' && !e.args[0].float) this.adopt(e.args[0], to); + // FLOAT32(0.1) is the single nearest the decimal written, not the single nearest its FLOAT64 + if (isLiteral(e.args[0]) && isFloat32(to) && e.args[0].float) this.adopt(e.args[0], to); + return to; + } + + comparable(e: BinaryExpr, lt: PType, rt: PType, equality: boolean): void { + if (isAny(lt) || isAny(rt)) return; + if (isNumeric(lt) && isNumeric(rt)) return; + if ((lt.k === 'CHAR' && rt.k === 'STRING') || (lt.k === 'STRING' && rt.k === 'CHAR')) { + const [cs, ss] = lt.k === 'CHAR' ? [e.left, e.right] : [e.right, e.left]; + const lit = ss.kind === 'Str' && Array.from(ss.value).length === 1 ? ` Write '${ss.value}' with single quotes to compare against the CHAR.` : ''; + this.error('char-string', e, `${sourceOf(cs)} is a CHAR and ${sourceOf(ss)} is a STRING. They are different types even when they hold the same letter, so ${e.op === '==' ? 'this comparison is never TRUE' : e.op === '!=' ? 'this comparison is always TRUE' : 'one cannot be ordered against the other'}.${lit}`, { + guide: 'the-types', + fix: lit ? { line: ss.line, col: ss.col, endCol: ss.col + (ss as StrExpr).value.length + 2, text: `'${(ss as StrExpr).value}'` } : undefined, + }); + return; + } + if (!sameType(lt, rt)) { this.error('type', e, `${sourceOf(e.left)} is ${article(lt)} and ${sourceOf(e.right)} is ${article(rt)}, and values of different types cannot be compared.`); return; } + if (!equality && !['CHAR', 'STRING', 'INTEGER', 'FLOAT'].includes(lt.k)) this.error('type', e, `${e.op} puts values in order, and ${typeName(lt)} values have no order.`); + } + + index(e: IndexExpr, scope: Scope): PType { + const t = this.expr(e.target, scope); + const its = e.indices.map((i) => this.expr(i, scope)); + for (let k = 0; k < its.length; k++) { + const it = its[k]; + if (t.k === 'HASHMAP') continue; + if (!isAny(it) && it.k !== 'INTEGER') { + this.error('type', e.indices[k], `A position is a whole number, and ${sourceOf(e.indices[k])} is ${article(it)}.${it.k === 'FLOAT' ? ' INT or ROUND turns it into one; decide which.' : ''}`, { guide: 'text-and-positions' }); + } + } + if (isAny(t)) return T.ANY; + switch (t.k) { + case 'STRING': + if (its.length !== 1) this.error('type', e, 'A STRING has one position per character, so it takes one index.'); + return T.CHAR; + case 'LIST': + if (its.length !== 1) this.error('type', e, `A LIST takes one index. For a list of lists, write ${sourceOf(e.target)}[I][J].`); + return t.of; + case 'ARRAY': + if (its.length !== t.dims) { + this.error('type', e, t.dims === 2 + ? `${sourceOf(e.target)} is a two-dimensional ARRAY, so it takes a row and a column: ${sourceOf(e.target)}[Row, Column].` + : `${sourceOf(e.target)} is a one-dimensional ARRAY, so it takes one index.`, { guide: 'array-when-you-know-how-many' }); + } + return t.of; + case 'HASHMAP': { + const kt = its[0]; + if (its.length !== 1) this.error('type', e, 'A HASHMAP takes one key in its brackets.'); + else if (!isAny(kt) && !sameType(t.key, kt) && !(isNumeric(t.key) && isNumeric(kt))) { + const extra = t.key.k === 'CHAR' && kt.k === 'STRING' && e.indices[0].kind === 'Str' ? ` Single quotes make a CHAR: '${e.indices[0].value}'.` : t.key.k === 'STRING' && kt.k === 'CHAR' ? ' Double quotes make a STRING.' : ''; + this.error('type', e.indices[0], `${sourceOf(e.target)} has ${typeName(t.key)} keys, and ${sourceOf(e.indices[0])} is ${article(kt)}.${extra}`); + } + return t.val; + } + case 'SET': this.error('type', e, `A SET has no positions, so ${sourceOf(e)} means nothing. Ask a SET whether something is in it, with IN.`, { guide: 'set-for-membership' }); return T.ANY; + default: this.error('type', e, `Square brackets reach a position in a STRING, LIST, ARRAY, or HASHMAP, and ${sourceOf(e.target)} is ${article(t)}.`); return T.ANY; + } + } + + call(e: CallExprNode, scope: Scope): PType | null { + const name = up(e.name); + if (this.ctx?.inEnsure && name === 'OLD') { + if (e.args.length !== 1) { this.error('arg-count', e, 'OLD takes one name: OLD(X) is the value X held when the subroutine was called.'); return T.ANY; } + const t = this.expr(e.args[0], scope); + e.isOld = true; + const a = e.args[0]; + if (a.kind !== 'Name' || (a.sym && a.sym.kind !== 'param')) { + const inner = a.kind === 'CallExpr' && a.args.length === 1 && a.args[0].kind === 'Name' ? `${a.name}(OLD(${a.args[0].name}))` : null; + this.error('old-name', e, `OLD takes the name of a parameter: OLD(X) is the value X held when the subroutine was called.${inner ? ` To use that value inside a call, write ${inner}.` : ''}`, { guide: 'contracts' }); + } + return t; + } + if (e.convert) return this.conversion(e, scope); + const sub = this.findSub(name); + if (sub) { + e.sub = sub; + if (sub.kind === 'Procedure') { + this.error('procedure-value', e, `${sub.name} is a PROCEDURE and hands nothing back, so it cannot stand where a value stands. CALL it on a line of its own.`, { guide: 'procedure' }); + for (const a of e.args) this.expr(a, scope); + return T.VOID; + } + this.args(sub, e.args, e, scope); + return sub.returns; + } + const b = BUILTINS[name]; + if (!b) { + const sym = scope.get(e.name); + if (sym) { this.error('not-callable', e, `${e.name} is ${sym.kind === 'var' ? 'a variable' : 'not a subroutine'}, so it cannot be called. Square brackets reach a position: ${e.name}[I].`); return T.ANY; } + const s = suggest(e.name, [...this.subs.values(), ...this.libs.values()].map((x) => x.name).concat(Object.keys(BUILTINS))); + const other = FOREIGN_FUNCTIONS[name]; + if (other) this.error('unknown-function', e, `No FUNCTION is called ${e.name}. ${other}`, { guide: 'function' }); + else this.error('unknown-function', e, `No FUNCTION is called ${e.name}.${s.length ? ` Did you mean ${s.join(' or ')}?` : ' Define it with FUNCTION ... ENDFUNCTION.'}`, { guide: 'function' }); + for (const a of e.args) this.expr(a, scope); + return T.ANY; + } + if (b.kind === 'procedure') { + this.error('procedure-value', e, `${name} changes a collection and hands nothing back, so it goes on a line of its own: ${name}(...).`); + for (const a of e.args) this.expr(a, scope); + return T.VOID; + } + e.builtin = name; + const ts = e.args.map((a, i) => this.expr(a, scope, { typeArg: TYPE_ARGS[name] === i })); + const argc = (n: number): boolean => { + if (ts.length !== n) { this.error('arg-count', e, `${b.sig} takes ${n} ${n === 1 ? 'value' : 'values'}, and this call gives it ${ts.length}.`); return false; } + return true; + }; + const num = (i: number): PType => { const t = ts[i]; if (!isAny(t) && !isNumeric(t)) this.error('type', e.args[i], `${name} works on numbers, and ${sourceOf(e.args[i])} is ${article(t)}.`); return t; }; + const int = (i: number): void => { const t = ts[i]; if (!isAny(t) && t.k !== 'INTEGER') this.error('type', e.args[i], `${name} needs a whole number here, and ${sourceOf(e.args[i])} is ${article(t)}.`); }; + switch (name) { + case 'LENGTH': { + if (!argc(1)) return T.INTEGER; + const t = ts[0]; + if (isAny(t)) return T.INTEGER; + if (t.k === 'ARRAY' && t.dims === 2) this.error('type', e, 'A two-dimensional ARRAY has two lengths: ROWS(A) and COLUMNS(A).'); + else if (!['STRING', 'LIST', 'ARRAY', 'SET', 'HASHMAP'].includes(t.k)) this.error('type', e.args[0], `LENGTH measures a STRING or a collection, and ${sourceOf(e.args[0])} is ${article(t)}.${t.k === 'CHAR' ? ' A CHAR is always exactly one character.' : ''}`); + return T.INTEGER; + } + case 'ROWS': case 'COLUMNS': + if (argc(1) && !isAny(ts[0]) && !(ts[0].k === 'ARRAY' && ts[0].dims === 2)) this.error('type', e.args[0], `${name} measures a two-dimensional ARRAY. For anything else, use LENGTH.`); + return T.INTEGER; + case 'INT': case 'ROUND': + if (argc(1)) { num(0); if (ts[0].k === 'INTEGER') this.note('already-whole', e, `${sourceOf(e.args[0])} is already an INTEGER, so ${name} changes nothing.`); } + return T.INTEGER; + case 'ABS': if (argc(1)) return isAny(num(0)) ? T.ANY : ts[0]; return T.ANY; + case 'MIN': case 'MAX': { + if (ts.length === 1) { + const t = ts[0]; + if (isAny(t)) return T.ANY; + const el = ['LIST', 'ARRAY', 'SET'].includes(t.k) ? t.of : null; + if (!el || (!isNumeric(el) && !isAny(el) && !isText(el))) { this.error('type', e, `${name} of one value needs a LIST, ARRAY, or SET of numbers; with separate values, give it two or more: ${name}(A, B).`); return T.ANY; } + return el; + } + if (ts.length < 2) { this.error('arg-count', e, `${name} needs at least two values, or one LIST.`); return T.ANY; } + let r: PType | null = null; + ts.forEach((t, i) => { + num(i); + // two different sized whole-number types do not mix here either: the answer would carry the first one's name + if (r && isSizedInt(r) && isSizedInt(t) && !sameType(r, t)) this.error('mixed-width', e.args[i], `${sourceOf(e.args[i])} is ${t.name} and the values before it are ${r.name}, and two different sized types do not mix silently. Convert one so they agree, as in ${r.name}(${sourceOf(e.args[i])}).`, { guide: 'sized-types' }); + r = r ? numericResult(r, t) : t; + }); + return r; + } + case 'SQRT': case 'EXP': case 'LOG': case 'LOG2': case 'RAND': case 'LOG10': case 'SIN': case 'COS': case 'TAN': if (argc(1)) num(0); return T.FLOAT; + case 'ATAN2': if (argc(2)) { num(0); num(1); } return T.FLOAT; + case 'PI': case 'RANDN': argc(0); return T.FLOAT; + case 'FLOOR': case 'CEIL': + if (argc(1)) { num(0); if (ts[0].k === 'INTEGER') this.note('already-whole', e, `${sourceOf(e.args[0])} is already an INTEGER, so ${name} changes nothing.`); } + return T.INTEGER; + case 'CHOOSE': if (argc(2)) { int(0); int(1); } return T.INTEGER; + case 'SUM_OF': case 'MEAN_OF': case 'VARIANCE_OF': case 'STDEV_OF': case 'MEDIAN_OF': case 'NORM_OF': { + if (!argc(1)) return name === 'SUM_OF' ? T.ANY : T.FLOAT; + const t = ts[0]; + const el = ['LIST', 'ARRAY'].includes(t.k) ? t.of : null; + if (!isAny(t) && (!el || (!isNumeric(el) && !isAny(el)))) { this.error('type', e.args[0], `${name} works on a LIST or ARRAY of numbers, and ${sourceOf(e.args[0])} is ${article(t)}.${t.k === 'SET' ? ' Turn the SET into a LIST first with SORTED.' : ''}`); return name === 'SUM_OF' ? T.ANY : T.FLOAT; } + if (name === 'SUM_OF') return el && !isAny(el) ? el : T.ANY; + return T.FLOAT; + } + case 'DOT': { + if (!argc(2)) return T.ANY; + const els = ts.map((t, i) => { + const el = ['LIST', 'ARRAY'].includes(t.k) ? t.of : null; + if (!isAny(t) && (!el || (!isNumeric(el) && !isAny(el)))) this.error('type', e.args[i], `DOT multiplies two LISTs or ARRAYs of numbers position by position, and ${sourceOf(e.args[i])} is ${article(t)}.`); + return el && !isAny(el) ? el : null; + }); + return els[0] && els[1] ? numericResult(els[0], els[1]) : T.ANY; + } + case 'FILL': + if (!argc(2)) return T.list(T.ANY); + int(0); + return T.list(ts[1]); + case 'RANDINT': if (argc(2)) { int(0); int(1); } return T.INTEGER; + case 'SLICE': { + if (!argc(3)) return T.ANY; + int(1); int(2); + const t = ts[0]; + if (!isAny(t) && !['STRING', 'LIST'].includes(t.k)) { this.error('type', e.args[0], `SLICE takes a run of positions from a STRING or a LIST, and ${sourceOf(e.args[0])} is ${article(t)}.`); return T.ANY; } + return t; + } + case 'UPPER': case 'LOWER': + if (argc(1) && !isAny(ts[0]) && !isText(ts[0])) this.error('type', e.args[0], `${name} changes letters, and ${sourceOf(e.args[0])} is ${article(ts[0])}.`); + return isText(ts[0]) ? ts[0] : T.STRING; + case 'SPLIT': + if (argc(2)) { + if (!isAny(ts[0]) && ts[0].k !== 'STRING') this.error('type', e.args[0], `SPLIT cuts a STRING into pieces, and ${sourceOf(e.args[0])} is ${article(ts[0])}.`); + if (!isAny(ts[1]) && !isText(ts[1])) this.error('type', e.args[1], `The separator of SPLIT is a STRING or a CHAR, and ${sourceOf(e.args[1])} is ${article(ts[1])}.`); + if (e.args[1].kind === 'Str' && e.args[1].value === '') this.error('value', e.args[1], 'SPLIT needs a separator with at least one character. To take a text apart character by character, use FOR EACH.'); + } + return T.list(T.STRING); + case 'JOIN': + if (argc(2)) { + if (!isAny(ts[0]) && !(ts[0].k === 'LIST' && (isAny(ts[0].of) || ts[0].of.k === 'STRING' || ts[0].of.k === 'CHAR'))) this.error('type', e.args[0], `JOIN glues a LIST OF STRING together, and ${sourceOf(e.args[0])} is ${article(ts[0])}.${ts[0].k === 'LIST' ? ' Turn each value into text first with TO_STRING.' : ''}`); + if (!isAny(ts[1]) && !isText(ts[1])) this.error('type', e.args[1], `The separator of JOIN is a STRING or a CHAR, and ${sourceOf(e.args[1])} is ${article(ts[1])}.`); + } + return T.STRING; + case 'INDEX_OF': + if (argc(2)) { + const t = ts[0]; + if (isAny(t)) return T.INTEGER; + if (t.k === 'STRING') { if (!isAny(ts[1]) && !isText(ts[1])) this.error('type', e.args[1], `INDEX_OF looks for a STRING or a CHAR inside a STRING, and ${sourceOf(e.args[1])} is ${article(ts[1])}.`); } + else if (t.k === 'LIST') { if (!isAny(ts[1]) && !isAny(t.of) && !sameType(ts[1], t.of)) this.error('type', e.args[1], `${sourceOf(e.args[0])} holds ${article(t.of)} at each position, and INDEX_OF is looking for ${article(ts[1])}.`); } + else this.error('type', e.args[0], `INDEX_OF searches a STRING or a LIST, and ${sourceOf(e.args[0])} is ${article(t)}.${t.k === 'HASHMAP' || t.k === 'SET' ? ' Ask IN instead: a HASHMAP or SET has no positions.' : ''}`); + } + return T.INTEGER; + case 'REPLACE': + if (argc(3)) { + if (!isAny(ts[0]) && ts[0].k !== 'STRING') this.error('type', e.args[0], `REPLACE works on a STRING, and ${sourceOf(e.args[0])} is ${article(ts[0])}.`); + for (const i of [1, 2]) if (!isAny(ts[i]) && !isText(ts[i])) this.error('type', e.args[i], `REPLACE swaps one piece of text for another, and ${sourceOf(e.args[i])} is ${article(ts[i])}.`); + if (e.args[1].kind === 'Str' && e.args[1].value === '') this.error('value', e.args[1], 'REPLACE needs something to look for: the piece to replace cannot be empty.'); + } + return T.STRING; + case 'ORD': + if (argc(1) && !isAny(ts[0]) && ts[0].k !== 'CHAR') this.error('type', e.args[0], `ORD numbers one CHAR, and ${sourceOf(e.args[0])} is ${article(ts[0])}.${ts[0].k === 'STRING' && e.args[0].kind === 'Str' ? ` Single quotes make a CHAR: '${e.args[0].value}'.` : ''}`); + return T.INTEGER; + case 'CHR': if (argc(1)) int(0); return T.CHAR; + case 'TO_STRING': argc(1); return T.STRING; + case 'TO_INTEGER': + if (argc(1) && !isAny(ts[0]) && !isText(ts[0])) this.error('type', e.args[0], ts[0].k === 'FLOAT' ? 'TO_INTEGER reads text. To turn a FLOAT into a whole number, use INT or ROUND.' : `TO_INTEGER reads a STRING of digits, and ${sourceOf(e.args[0])} is ${article(ts[0])}.`); + return T.INTEGER; + case 'TO_FLOAT': + if (argc(1) && !isAny(ts[0]) && !isText(ts[0]) && !isNumeric(ts[0])) this.error('type', e.args[0], `TO_FLOAT reads a STRING or converts a number, and ${sourceOf(e.args[0])} is ${article(ts[0])}.`); + return T.FLOAT; + case 'KEYS': + if (argc(1) && !isAny(ts[0]) && ts[0].k !== 'HASHMAP') { this.error('type', e.args[0], `KEYS lists the keys of a HASHMAP, and ${sourceOf(e.args[0])} is ${article(ts[0])}.`); return T.list(T.ANY); } + // typed-port: with the wrong number of values (argc says so and goes on), a first value that is not a HASHMAP + // has no key type, and the answer is a LIST OF undefined (shown as "LIST OF an unknown type"). + return T.list(isAny(ts[0]) ? T.ANY : ts[0].key as PType); + case 'SORTED': + if (argc(1) && !isAny(ts[0]) && !['LIST', 'SET', 'ARRAY'].includes(ts[0].k)) { this.error('type', e.args[0], `SORTED puts the values of a LIST or SET in order, and ${sourceOf(e.args[0])} is ${article(ts[0])}.`); return T.list(T.ANY); } + // typed-port: as KEYS: with the wrong number of values, a first value with no element type gives a LIST OF undefined. + return T.list(isAny(ts[0]) ? T.ANY : ts[0].of as PType); + // ---- numerical work + case 'ASIN': case 'ACOS': case 'ATAN': case 'SINH': case 'COSH': case 'TANH': case 'CBRT': case 'EXPM1': case 'LOG1P': + if (argc(1)) num(0); + return T.FLOAT; + case 'HYPOT': case 'POW': case 'COPYSIGN': if (argc(2)) { num(0); num(1); } return T.FLOAT; + case 'E': case 'INF': case 'NAN': argc(0); return T.FLOAT; + case 'TRUNC': if (argc(1)) { num(0); return ts[0].k === 'FLOAT' ? ts[0] : T.FLOAT; } return T.FLOAT; + case 'SIGN': if (argc(1)) return isAny(num(0)) ? T.ANY : ts[0]; return T.ANY; + case 'CLAMP': { + if (!argc(3)) return T.ANY; + ts.forEach((_, i) => num(i)); + if (ts.some((x) => isAny(x))) return T.ANY; + for (const i of [1, 2]) this.adopt(e.args[i], ts[0]); + // the answer may be either end, so its type is the widest of the three: CLAMP(I, 0.5, 2.5) is a FLOAT + return numericResult(numericResult(ts[0], e.args[1].type || ts[1]), e.args[2].type || ts[2]); + } + case 'FMA': { + if (!argc(3)) return T.FLOAT; + ts.forEach((_, i) => num(i)); + const f32 = ts.some((x) => isFloat32(x)); + if (f32) e.args.forEach((a) => this.adopt(a, T.FLOAT32)); + return f32 && e.args.every((a) => isFloat32(a.type) || a.type?.k === 'INTEGER') ? T.FLOAT32 : T.FLOAT; + } + case 'IS_NAN': case 'IS_INF': case 'IS_FINITE': if (argc(1)) num(0); return T.BOOL; + case 'IS_CLOSE': if (argc(4)) ts.forEach((_, i) => num(i)); return T.BOOL; + case 'ULP': case 'NEXT_AFTER': { + if (!argc(name === 'ULP' ? 1 : 2)) return T.FLOAT; + ts.forEach((_, i) => num(i)); + return isFloat32(ts[0]) ? ts[0] : T.FLOAT; + } + case 'EXACT': if (argc(1)) num(0); return T.STRING; + case 'MAX_VALUE': case 'MIN_VALUE': case 'EPSILON': { + if (!argc(1)) return T.ANY; + const a = e.args[0]; + if (a.kind !== 'TypeRef') { this.error('type', a, `${name} takes the name of a numeric type, as in ${name}(${name === 'EPSILON' ? 'FLOAT64' : 'INT32'}).`, { guide: 'sized-types' }); return T.ANY; } + const ty = T[a.name]; + if (name === 'EPSILON' && ty.k !== 'FLOAT') { this.error('type', a, `EPSILON measures the spacing of FLOAT values near 1.0; ${a.name} has none. For whole numbers the step is always 1.`, { guide: 'sized-types' }); return T.ANY; } + if (ty.k === 'INTEGER' && !ty.bits) { this.error('type', a, `INTEGER has no ${name === 'MAX_VALUE' ? 'largest' : 'lowest'} value: it is a whole number of any size, and a run holds it exactly from -9,007,199,254,740,991 to 9,007,199,254,740,991. Name a sized type, such as INT32.`, { guide: 'sized-types' }); return T.INTEGER; } + if (ty.k === 'INTEGER' && ty.bits === 64 && (name === 'MAX_VALUE' || ty.signed)) { this.error('exact-limit', a, `${name}(${ty.name}) is ${name === 'MAX_VALUE' ? (ty.signed ? '9,223,372,036,854,775,807' : '18,446,744,073,709,551,615') : '-9,223,372,036,854,775,808'}, beyond what this lab holds exactly (from -9,007,199,254,740,991 to 9,007,199,254,740,991). Name a type up to 32 bits, or write the limit you need as a constant.`, { guide: 'sized-types' }); return ty; } + return ty.k === 'FLOAT' && !ty.bits ? T.FLOAT : ty; + } + case 'GCD': case 'LCM': case 'BIT_AND': case 'BIT_OR': case 'BIT_XOR': { + if (!argc(2)) return T.INTEGER; + int(0); int(1); + if (isAny(ts[0]) || isAny(ts[1])) return T.INTEGER; + return this.mix({ ...e, left: e.args[0], right: e.args[1], op: name }, ts[0], ts[1]); + } + case 'BIT_NOT': if (argc(1)) { int(0); return ts[0].k === 'INTEGER' ? ts[0] : T.INTEGER; } return T.INTEGER; + case 'SHIFT_LEFT': case 'SHIFT_RIGHT': + if (argc(2)) { int(0); int(1); return ts[0].k === 'INTEGER' ? ts[0] : T.INTEGER; } + return T.INTEGER; + case 'WRAP': { + if (!argc(2)) return T.ANY; + int(0); + // the arithmetic inside WRAP is exact (INTEGER), so that WRAP(Count + 1, UINT8) wraps instead of stopping at Count + 1 + const exact = (n: Expr | null | undefined): void => { + if (!n || typeof n !== 'object') return; + if ((n.kind === 'Binary' || n.kind === 'Unary' || n.kind === 'Num') && isSizedInt(n.type)) n.type = T.INTEGER; + if (n.kind === 'Binary') { exact(n.left); exact(n.right); } else if (n.kind === 'Unary') exact(n.arg); + }; + exact(e.args[0]); + const a = e.args[1]; + if (a.kind !== 'TypeRef' || !isSizedInt(T[a.name])) { this.error('type', a, 'WRAP takes a sized whole-number type as its second value, as in WRAP(X, UINT8).', { guide: 'sized-types' }); return T.ANY; } + const ty = T[a.name]; + if (ty.bits! > 32) { this.error('wrap-64', a, `WRAP works on types up to 32 bits in this lab, which holds whole numbers exactly only to 2^53 - 1; ${ty.name} wraps at 2^64.`, { guide: 'sized-types' }); return ty; } + return ty; + } + default: return T.ANY; + } + } +} + +/** + * Check a parsed program. `options.libs`: routines brought in with USE, already checked in their own files (UPPER + * name -> routine), and `options.libOrigin`, the USE line that brought each one in. + */ +export function check(parsed: Parsed, options: CheckOptions = {}): { program: Program; diags: Diag[] } { + const c = new Checker(parsed, options); + const program = c.run(); + program.source = parsed.source; + program.lines = parsed.lines; + program.comments = parsed.comments; + const diags = [...parsed.diags, ...c.diags].sort((a, b) => a.line - b.line || a.col - b.col); + return { program, diags }; +} + +export { elementOf, MUTATORS }; diff --git a/src/engines/pseudo/coach.js b/src/engines/pseudo/coach.js new file mode 100644 index 0000000..2634be4 --- /dev/null +++ b/src/engines/pseudo/coach.js @@ -0,0 +1,421 @@ +/** + * pseudo/coach.js + * + * The mistakes the guide warns about, found in a student's own program. + * Everything here runs after parsing, checking, and analysis, and adds + * warnings and notes rather than errors: a program the coach dislikes still + * runs, because running it is often the fastest way to see the problem. + * + * coach(program, analysis) -> [diag] + * + * Every rule has an id, a severity, and a guide anchor, so the lab can + * link each finding to the paragraph that explains it. + */ + +import { diag, sourceOf } from './parse.js'; +import { walkStatements, walkExpr, rootName, statementExprs, childBlocks } from './analyze.js'; +import { namesIn } from './interp.js'; +import { isNumeric } from './lang.js'; + +const up = (s) => String(s).toUpperCase(); +const LOOPS = ['For', 'ForEach', 'While', 'Repeat']; + +/** The expression a simple name was last set to, if it was set exactly once. */ +function definitionsOf(body) { + const count = new Map(); const value = new Map(); + walkStatements(body, (s) => { + if (s.kind === 'Assign' && s.target.kind === 'Name') { + const k = up(s.target.name); + count.set(k, (count.get(k) || 0) + 1); + value.set(k, s.value); + } + }); + const out = new Map(); + for (const [k, n] of count) if (n === 1) out.set(k, value.get(k)); + return out; +} + +/** Is `e` LENGTH(X), or a name set once to LENGTH(X)? Returns X's source. */ +function lengthOf(e, defs) { + if (!e) return null; + if (e.kind === 'CallExpr' && e.builtin === 'LENGTH' && e.args[0]) return sourceOf(e.args[0]); + if (e.kind === 'Name' && defs.has(up(e.name))) return lengthOf(defs.get(up(e.name)), new Map()); + return null; +} + +/** Every X[...] read inside a list of statements, as { target, index, line }. */ +function indexReads(body) { + const out = []; + const visit = (e) => walkExpr(e, (x) => { if (x.kind === 'Index' && x.target.kind === 'Name') out.push({ target: sourceOf(x.target), index: x.indices[0], node: x }); }); + walkStatements(body, (s) => { for (const e of statementExprs(s)) visit(e); }); + return out; +} + +/** For T[I + 2] with counter I, 2; for T[I], 0; for anything else, null. */ +function offsetOf(e, counter) { + if (e.kind === 'Name' && up(e.name) === up(counter)) return 0; + if (e.kind === 'Binary' && e.op === '+' && e.left.kind === 'Name' && up(e.left.name) === up(counter) && e.right.kind === 'Num') return e.right.value; + if (e.kind === 'Binary' && e.op === '-' && e.left.kind === 'Name' && up(e.left.name) === up(counter) && e.right.kind === 'Num') return -e.right.value; + return null; +} + +const mentions = (e, name) => [...namesIn(e || {})].some((n) => up(n) === up(name)); + +export function coach(program, analysis, opts = {}) { + const out = []; + const say = (severity, code, node, message, extra = {}) => out.push(diag(severity, code, node?.line ?? 1, message, { col: node?.col ?? 1, source: 'coach', ...extra })); + + for (const unit of analysis.units) { + const body = unit.body; + const defs = definitionsOf(body); + const where = unit.kind === 'main' ? 'the main program' : unit.name; + + /* ---------- values read before they are set ---------- */ + const reported = new Set(); + const loopsByLine = unit.loops; + for (const u of unit.uses) { + if (u.kind === 'spec' || u.implicit) continue; + const key = `${u.var}@${u.line}`; + if (reported.has(key)) continue; + const unset = u.defs.filter((d) => d.kind === 'unset'); + if (!unset.length) continue; + // Passed BYREF to a procedure that writes the parameter before reading it: an out-parameter, not a read. + if (u.byref) { + const callee = analysis.units.find((x) => up(x.name) === up(u.callee)); + const readsFirst = !callee || callee.uses.some((cu) => cu.var === u.byref && cu.kind !== 'spec' && cu.defs.some((d) => d.kind === 'param')); + if (!readsFirst) continue; + } + const real = u.defs.filter((d) => d.kind !== 'unset'); + reported.add(key); + const afterLoop = unset.find((d) => d.stmt?.kind === 'For'); + if (afterLoop && !real.length) { + say('warning', 'counter-after-loop', { line: u.line, col: u.expr?.col }, `${u.name} is read on line ${u.line}, after the FOR loop on line ${afterLoop.stmt.line} that used it as a counter. The notation does not say what a counter holds after its loop, and the lab gives it no value. Keep the position you need in a variable of your own.`, { guide: 'for-when-you-know-how-many-times' }); + continue; + } + // A value updated inside a loop from its own old value, with nothing before the loop. + const enclosing = loopsByLine.filter((L) => u.line >= L.line && u.line <= (L.endLine ?? L.line)); + const carried = real.length && real.every((d) => enclosing.some((L) => d.line >= L.line && d.line <= (L.endLine ?? L.line))); + if (!real.length || carried) { + const acc = unit.accumulators.find((a) => a.var === u.var && a.how !== 'flag'); + const msg = acc + ? `${u.name} is updated from its own old value on line ${acc.line}, and nothing before the loop gives it a starting value, so the first pass has nothing to add to. Set up is missing: give ${u.name} its starting value before the loop.` + : carried + ? `${u.name} is read on line ${u.line}, and on the first pass of the loop nothing has given it a value yet.` + : `${u.name} is read on line ${u.line}, and no line before it gives ${u.name} a value.`; + say('warning', 'unset-read', { line: u.line, col: u.expr?.col }, msg, { guide: 'declaring' }); + } else { + say('warning', 'maybe-unset', { line: u.line, col: u.expr?.col }, `${u.name} may reach line ${u.line} without a value: some path through ${where} gets here without assigning it. Give it a value on every path, or before the branch.`, { guide: 'declaring' }); + } + } + + /* ---------- names nobody reads ---------- */ + const read = new Set(unit.uses.map((u) => u.var)); + const given = new Set(unit.defs.filter((d) => d.kind !== 'unset' && d.kind !== 'init').map((d) => d.var)); + for (const sym of unit.symbols || []) { + const k = up(sym.name); + if (sym.kind === 'var' && !read.has(k)) { + const assigned = given.has(k); + say('note', assigned ? 'never-read' : 'unused', sym, assigned + ? `${sym.name} is given a value and never read. Either something still needs it, or the line that sets it can go.` + : `${sym.name} is declared and never used.`, { guide: 'declaring' }); + } + if (sym.kind === 'param' && !read.has(k) && sym.mode !== 'BYREF') say('note', 'unused-param', sym, `Parameter ${sym.name} is never used inside ${unit.name}.`); + } + + /* ---------- loops ---------- */ + for (const L of unit.loops) { + const s = L.node; + if (s.kind === 'For') { + const reads = indexReads(s.body); + // The bound against the furthest position the body reads: a loop over + // [0, LENGTH(T) - d) may read T[I + d - 1] and no further. + if (s.start?.kind === 'Num' && s.start.value === 0 && (!s.step || (s.step.kind === 'Num' && s.step.value === 1))) { + let seq = lengthOf(s.stop, defs); let d = 0; + if (!seq && s.stop?.kind === 'Binary' && s.stop.op === '-' && s.stop.right.kind === 'Num') { seq = lengthOf(s.stop.left, defs); d = s.stop.right.value; } + if (seq) { + const offsets = reads.filter((r) => r.target === seq).map((r) => offsetOf(r.index, s.var)).filter((o) => o !== null); + if (offsets.length) { + const far = Math.max(...offsets); + if (d > far && d === 1 && far === 0) { + say('warning', 'stops-early', s.stop, `FOR ${s.var} <- 0 TO ${sourceOf(s.stop)} stops one short of its bound, so ${s.var} never reaches ${sourceOf(s.stop)}, the last position of ${seq}. The loop skips the last position. A loop over every position uses the length itself: TO ${sourceOf(s.stop.left)}.`, { + guide: 'for-when-you-know-how-many-times', fix: { line: s.stop.line, col: s.stop.col, text: sourceOf(s.stop.left), replaceExpr: true }, + }); + } else if (d > far) { + say('warning', 'stops-early', s.stop, `The body reads ${seq} no further than ${s.var}${far ? ` + ${far}` : ''}, so the loop could run to ${far ? `LENGTH(${seq}) - ${far}` : `LENGTH(${seq})`}; stopping at ${sourceOf(s.stop)} skips the last ${d - far === 1 ? 'position' : `${d - far} positions`}.`, { guide: 'text-and-positions' }); + } else if (far > d) { + say('warning', 'overrun', s.stop, `The body reads ${seq}[${s.var} + ${far}], and on the last pass ${s.var} is ${d ? `LENGTH(${seq}) - ${d + 1}` : `LENGTH(${seq}) - 1`}, so it asks for position ${far > d + 1 ? `LENGTH(${seq}) + ${far - d - 1}` : `LENGTH(${seq})`}, which does not exist. A read that reaches ${far} ahead needs the bound LENGTH(${seq}) - ${far}.`, { guide: 'text-and-positions' }); + } + } + } + } + // FOR I <- 1 TO LENGTH(T) + 1, or starting at 1 + if (s.start?.kind === 'Num' && s.start.value === 1) { + const seq = lengthOf(s.stop?.kind === 'Binary' && s.stop.op === '+' ? s.stop.left : s.stop, defs); + const byCounter = (r) => r.index.kind === 'Name' && up(r.index.name) === up(s.var); + // Starting at 1 is deliberate when position 0 is handled another + // way: read before the loop (Max <- Q[0]), or remembered in a + // variable the body compares against (Q[Best], T[I - 1]). + const esc = String(seq || '').replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const zeroBefore = String(program.source || "").split("\n").slice(0, s.line - 1).some((l) => new RegExp(`\\b${esc}\\[\\s*0\\s*\\]`, 'i').test(l.replace(/\/\/.*$/, ''))); + const otherRead = reads.some((r) => r.target === seq && !byCounter(r)); + if (seq && !zeroBefore && !otherRead && reads.some((r) => r.target === seq && byCounter(r))) { + say('warning', 'starts-at-one', s.start, `${s.var} starts at 1, so ${seq}[0], the first position, is never visited. Positions count from zero.`, { guide: 'text-and-positions' }); + } + } + // STEP k with T[I + k - 1] and a bound that is the length + if (s.step?.kind === 'Num' && s.step.value > 1) { + const seq = lengthOf(s.stop, defs); + const reach = reads.filter((r) => r.target === seq && r.index.kind === 'Binary' && r.index.op === '+' && r.index.left.kind === 'Name' && up(r.index.left.name) === up(s.var) && r.index.right.kind === 'Num'); + if (seq && reach.length) { + const far = Math.max(...reach.map((r) => r.index.right.value)); + say('warning', 'step-overrun', s.step, `With STEP ${s.step.value} and a stop of ${sourceOf(s.stop)}, the last pass can start within ${far} ${far === 1 ? 'position' : 'positions'} of the end, and then ${seq}[${s.var} + ${far}] does not exist unless the length is a multiple of ${s.step.value}. ${sourceOf(s.stop)} DIV ${s.step.value} * ${s.step.value} is a bound that always fits.`, { guide: 'step-for-other-increments' }); + } + } + // APPEND to a list whose length is the bound + walkStatements(s.body, (x) => { + if (x.kind === 'BuiltinCall' && x.name === 'APPEND' && lengthOf(s.stop, new Map()) === sourceOf(x.args[0])) { + say('note', 'growing-bound', s.stop, `The bound ${sourceOf(s.stop)} is worked out once, when the loop starts, and ${sourceOf(x.args[0])} grows inside it. The loop runs as many times as the list was long at the start.`, { guide: 'list-when-you-do-not-know-how-many' }); + } + }); + } + if (s.kind === 'While' || s.kind === 'Repeat') { + const tests = L.tests.filter((t) => ![...(program.constants?.keys?.() || [])].includes(t)); + const exits = L.breaks + L.returns; + if (s.cond && !exits && tests.length && !tests.some((t) => L.changes.has(t))) { + say('warning', 'no-progress', s, `The ${s.kind === 'While' ? 'WHILE' : 'REPEAT'} condition ${sourceOf(s.cond)} reads ${tests.map((t) => nameCase(s.cond, t)).join(' and ')}, and nothing in the body changes ${tests.length === 1 ? 'it' : 'any of them'}. ${s.kind === 'While' ? 'Once the loop starts, it never ends.' : 'If the condition starts FALSE, it stays FALSE.'} Name the quantity that moves, and move it.`, { guide: 'while-when-the-condition-is-checked-first' }); + } + if (s.cond && s.cond.kind === 'Bool' && s.cond.value === (s.kind === 'While') && !exits) { + say('warning', 'no-progress', s, `${s.kind === 'While' ? 'WHILE TRUE' : 'UNTIL FALSE'} with no BREAK or RETURN inside never ends.`); + } + let drains = false; + walkStatements(s.body, (x) => { if (x.kind === 'BuiltinCall' && x.name === 'REMOVE') drains = true; }); + walkStatements(s.body, (x) => { + if (x.kind === 'BuiltinCall' && x.name === 'APPEND' && !drains) { + walkExpr(s.cond || {}, (e) => { + if (e.kind === 'CallExpr' && e.builtin === 'LENGTH' && sourceOf(e.args[0]) === sourceOf(x.args[0])) { + say('warning', 'growing-bound', e, `The condition measures ${sourceOf(e.args[0])}, and the body keeps adding to it, so the target moves every pass. Take the length once, into a name, before the loop.`, { guide: 'list-when-you-do-not-know-how-many' }); + } + }); + } + }); + } + if (s.kind === 'ForEach') { + walkStatements(s.body, (x) => { + const r = x.kind === 'BuiltinCall' ? rootName(x.args[0]) : x.kind === 'Assign' ? rootName(x.target) : null; + if (r && s.iterable && sourceOf(s.iterable) === r) say('warning', 'change-while-visiting', x, `${r} changes inside the FOR EACH that is visiting it. The loop visits the values ${r} held when it started; build a second collection instead.`); + }); + } + if (L.breaks >= 3) say('note', 'many-exits', s, `This loop has ${L.breaks} BREAK lines, so it has ${L.breaks} exits and no single place that says when it ends. One exit, placed where its reason is obvious, is easier to follow.`, { guide: 'leaving-a-loop-early' }); + if (L.depth >= 3) say('note', 'deep-nesting', s, `This loop sits ${L.depth + 1} levels deep. Past three levels a reader loses track of which block is which; pulling the inner work out into a FUNCTION of its own is the usual way back.`, { guide: 'nesting' }); + } + + /* ---------- accumulators reset in the wrong loop ---------- */ + wrongReset(unit, say); + + /* ---------- statements ---------- */ + walkStatements(body, (s, parents) => { + // indentation shows structure + const parent = parents[parents.length - 1]; + if (parent && program.lines && !['Function', 'Procedure'].includes(parent.kind)) { + const pl = program.lines[parent.line - 1]; const sl = program.lines[s.line - 1]; + if (pl && sl && sl.indent <= pl.indent && s.line !== parent.line) say('note', 'indent', s, `Line ${s.line} is inside the ${openerWord(parent)} on line ${parent.line}, and is not indented past it. Indent a block's body by four spaces so its structure shows.`, { guide: 'how-to-read-this-guide' }); + } else if (parent && program.lines) { + const pl = program.lines[parent.line - 1]; const sl = program.lines[s.line - 1]; + if (pl && sl && sl.indent <= pl.indent) say('note', 'indent', s, `Line ${s.line} is in the body of ${parent.name}, and is not indented past its first line.`, { guide: 'how-to-read-this-guide' }); + } + // unreachable lines + for (const b of childBlocks(s).concat(parents.length ? [] : [])) unreachable(b, say); + if (s.kind === 'Assign' && s.target.kind === 'Name' && s.value.kind === 'Name' && up(s.value.name) === up(s.target.name)) say('warning', 'self-assign', s, `${sourceOf(s.target)} <- ${sourceOf(s.value)} changes nothing.`); + if (s.kind === 'Assign' && s.target.type?.k === 'FLOAT' && s.value.kind === 'Binary' && s.value.op === 'DIV') say('warning', 'div-into-float', s.value, `${sourceOf(s.target)} is a FLOAT, and DIV throws away the remainder before the value arrives. For a fraction, divide with /.`, { guide: 'arithmetic' }); + if (s.kind === 'Case') caseChecks(s, say); + if ((s.kind === 'If' || s.kind === 'Case') && s.kind === 'If' && s.branches.some((b) => !b.body.length) && !s.elseBody) say('note', 'empty-body', s, 'This IF has an empty body, so it does nothing either way.'); + if (s.kind === 'Output' && unit.kind === 'Function') say('note', 'function-output', s, `${unit.name} is a FUNCTION and also shows output. A function that prints is harder to reason about, since its value stops being the whole story of what calling it did. Return the value and let the caller show it.`, { guide: 'procedure' }); + for (const e of statementExprs(s)) exprChecks(e, say); + // H[K] read with no guard + hashGuards(s, parents, say); + }); + if (unit.kind !== 'main') unreachable(body, say); + else unreachable(body, say); + + if (unit.kind === 'Function' && unit.node && !/^[A-Z0-9_]+$/.test(unit.name)) { + say('note', 'function-name', unit.node, `Function and procedure names in this course are uppercase with underscores, such as ${up(unit.name.replace(/([a-z])([A-Z])/g, '$1_$2'))}, which tells a call apart from a variable at a glance.`, { guide: 'function' }); + } + } + + // one report per line and rule + const seen = new Set(); + return out.filter((d) => { const k = `${d.code}@${d.line}@${d.message}`; if (seen.has(k)) return false; seen.add(k); return true; }); +} + +function nameCase(expr, upperName) { + let found = upperName; + walkExpr(expr, (x) => { if (x.kind === 'Name' && up(x.name) === upperName) found = x.name; }); + return found; +} + +function openerWord(s) { + return { If: 'IF', Case: 'CASE', For: 'FOR', ForEach: 'FOR EACH', While: 'WHILE', Repeat: 'REPEAT' }[s.kind] || s.kind; +} + +function unreachable(body, say) { + for (let i = 0; i < body.length - 1; i++) { + const s = body[i]; + if (['Return', 'Break', 'Continue'].includes(s.kind)) { + say('warning', 'unreachable', body[i + 1], `Line ${body[i + 1].line} can never run: the ${s.kind.toUpperCase()} on line ${s.line} always leaves before reaching it.`, { guide: s.kind === 'Return' ? 'function' : 'leaving-a-loop-early' }); + return; + } + } +} + +/** + * Two ways to put an accumulator's reset in the wrong loop: + * outer: reset before the outer loop, changed inside an inner loop, read + * once per outer pass (the Matched mistake); every later pass + * starts from whatever the previous one left. + * inner: reset inside the loop, updated from itself inside the same loop, + * and read only after the loop; every pass wipes the last. + */ +function wrongReset(unit, say) { + for (const L of unit.loops) { + if (L.depth !== 0) continue; + const outer = L.node; + const inner = unit.loops.filter((m) => m.parent === outer); + // outer + for (const I of inner) { + for (const name of I.changes) { + if (up(name) === up(I.node.var || '')) continue; + const setBefore = findAssignBefore(unit.body, outer, name); + if (!setBefore || !isConstant(setBefore.value)) continue; + const resetInside = outer.body.some((s, idx) => idx < outer.body.indexOf(I.node) && s.kind === 'Assign' && s.target.kind === 'Name' && up(s.target.name) === name); + if (resetInside) continue; + const readAfter = outer.body.slice(outer.body.indexOf(I.node) + 1).some((s) => mentions({ kind: 'Chain', operands: statementExprs(s), ops: [] }, name) || blockMentions(s, name)); + if (!readAfter) continue; + say('warning', 'reset-outside', setBefore, `${setBefore.target.name} is set to ${sourceOf(setBefore.value)} once, before the ${openerWord(outer)} loop on line ${outer.line}, and changed inside the ${openerWord(I.node)} loop on line ${I.node.line}. Every pass of the outer loop after the first starts with whatever the last pass left in it. If ${setBefore.target.name} belongs to one pass, reset it at the top of the outer loop's body.`, { guide: 'nested-loops' }); + } + } + // inner + const resets = outer.body.filter((s) => s.kind === 'Assign' && s.target.kind === 'Name' && isConstant(s.value)); + for (const r of resets) { + const name = up(r.target.name); + const acc = unit.accumulators.find((a) => a.var === name && a.loop === outer && a.how !== 'flag'); + if (!acc || acc.line < r.line) continue; + const usedInside = outer.body.some((s) => s !== acc.stmt && s !== r && s.line > acc.line && (mentions({ kind: 'Chain', operands: statementExprs(s), ops: [] }, name) || blockMentions(s, name))); + if (usedInside) continue; + const after = unit.body.slice(unit.body.indexOf(outer) + 1); + if (!after.some((s) => mentions({ kind: 'Chain', operands: statementExprs(s), ops: [] }, name))) continue; + say('warning', 'reset-inside', r, `${r.target.name} is reset to ${sourceOf(r.value)} at the start of every pass of the loop on line ${outer.line}, and read only after the loop ends, so it ends up holding just the last pass's contribution. If ${r.target.name} collects over the whole loop, set it up before the loop.`, { guide: 'nested-loops' }); + } + } +} + +function blockMentions(s, name) { + let hit = false; + walkStatements([s], (x) => { for (const e of statementExprs(x)) if (mentions(e, name)) hit = true; }); + return hit; +} + +function findAssignBefore(body, loop, name) { + const i = body.indexOf(loop); + for (let k = i - 1; k >= 0; k--) { + const s = body[k]; + if (s.kind === 'Assign' && s.target.kind === 'Name' && up(s.target.name) === name) return s; + } + return null; +} + +const isConstant = (e) => e && ['Num', 'Bool', 'Str', 'Char'].includes(e.kind); + +function caseChecks(s, say) { + if (!s.otherwise) say('note', 'case-no-otherwise', s, `This CASE has no OTHERWISE, so a value that matches no branch does nothing, silently. That is occasionally what you want, and more often the bug you spend an afternoon on.`, { guide: 'case' }); + const ranges = []; + for (const a of s.arms) for (const l of a.labels) if (l.low && l.low.kind === 'Num' && l.high.kind === 'Num') ranges.push({ lo: l.low.value, hi: l.high.value, node: l.low, arm: a }); + ranges.sort((a, b) => a.lo - b.lo); + for (let i = 0; i < ranges.length; i++) { + const r = ranges[i]; + if (r.hi <= r.lo) say('warning', 'case-empty-range', r.node, `${r.lo} TO ${r.hi} covers [${r.lo}, ${r.hi}), which holds nothing: the second value has to be larger.`, { guide: 'case' }); + const nx = ranges[i + 1]; + if (!nx) continue; + if (nx.lo === r.hi + 1) { + say('warning', 'case-gap', r.node, `${r.lo} TO ${r.hi} covers [${r.lo}, ${r.hi}): ranges in a CASE stop one short of their second value, exactly like FOR. ${r.hi} itself${s.subject?.type?.k === 'FLOAT' ? `, and everything between ${r.hi} and ${nx.lo},` : ''} matches neither this branch nor the next one, which starts at ${nx.lo}. Write ${r.lo} TO ${nx.lo} so the two ranges meet.`, { guide: 'case', fix: { line: r.node.line, text: `${r.lo} TO ${nx.lo}` } }); + } else if (nx.lo < r.hi) { + say('warning', 'case-overlap', nx.node, `${nx.lo} TO ${nx.hi} overlaps ${r.lo} TO ${r.hi}. CASE runs only the first branch that matches, so the overlap goes to whichever comes first.`, { guide: 'case' }); + } else if (nx.lo > r.hi) { + say('note', 'case-hole', nx.node, `Nothing covers [${r.hi}, ${nx.lo}), the values between these two ranges.`, { guide: 'case' }); + } + } +} + +function exprChecks(e, say) { + walkExpr(e, (x) => { + if (x.kind === 'Binary' && (x.op === '==' || x.op === '!=')) { + const [a, b] = [x.left, x.right]; + if (a.kind === 'Bool' || b.kind === 'Bool') { + const other = a.kind === 'Bool' ? b : a; + const lit = a.kind === 'Bool' ? a : b; + const positive = (lit.value === true) === (x.op === '=='); + say('note', 'bool-compare', x, `${sourceOf(other)} already holds TRUE or FALSE, so comparing it with ${lit.value ? 'TRUE' : 'FALSE'} asks the same question twice. Write ${positive ? sourceOf(other) : `NOT ${sourceOf(other)}`}.`, { guide: 'if' }); + } + if ((a.type?.k === 'FLOAT' || b.type?.k === 'FLOAT') && isNumeric(a.type) && isNumeric(b.type)) { + say('warning', 'float-equal', x, `${sourceOf(x)} compares FLOAT values with ${x.op}. Two measurements that should be equal often differ in the last digit, so compare the difference against a tolerance: ABS(${sourceOf(a)} - ${sourceOf(b)}) < 0.0001, or IS_CLOSE(${sourceOf(a)}, ${sourceOf(b)}, 1e-9, 0.0) to say the tolerance as a fraction of the values.`, { guide: 'comparison' }); + } + } + if (x.kind === 'Unary' && x.op === 'NOT' && !x.arg.paren && (x.arg.kind === 'Binary' && ['==', '!=', '<', '<=', '>', '>='].includes(x.arg.op))) { + const opp = { '==': '!=', '!=': '==', '<': '>=', '<=': '>', '>': '<=', '>=': '<' }[x.arg.op]; + say('note', 'not-compare', x, `NOT applies to the whole comparison here, and the comparison has a shorter opposite: ${sourceOf(x.arg.left)} ${opp} ${sourceOf(x.arg.right)}.`, { guide: 'compound-conditions' }); + } + if (x.kind === 'Unary' && x.op === 'NOT' && x.arg.paren && x.arg.kind === 'Binary' && ['<', '<=', '>', '>='].includes(x.arg.op)) { + const opp = { '<': '>=', '<=': '>', '>': '<=', '>=': '<' }[x.arg.op]; + say('note', 'not-compare', x, `${sourceOf(x)} has a shorter equivalent without the negation: ${sourceOf(x.arg.left)} ${opp} ${sourceOf(x.arg.right)}.`, { guide: 'compound-conditions' }); + } + if (x.kind === 'Binary' && x.op === 'AND') guardOrder(x, say); + }); +} + +/** In A AND B, the side that keeps a position legal has to come first. */ +function guardOrder(x, say) { + const reads = []; + walkExpr(x.left, (e) => { if (e.kind === 'Index' && e.target.kind === 'Name') reads.push(e); }); + if (!reads.length) return; + let guard = null; + walkExpr(x.right, (e) => { + if ((e.kind === 'Binary' || e.kind === 'Chain') && ['<', '<=', '>', '>='].includes(e.op || e.ops?.[0])) { + for (const r of reads) { + const idx = sourceOf(r.indices[0]); + const txt = sourceOf(e); + if (txt.includes(idx) && (txt.includes(`LENGTH(${sourceOf(r.target)})`) || /\b[NM]\b/.test(txt))) guard = { e, r }; + } + } + }); + if (guard) { + say('warning', 'guard-order', x, `${sourceOf(guard.r)} is read on the left of AND, and the test that keeps ${sourceOf(guard.r.indices[0])} legal, ${sourceOf(guard.e)}, is on the right. The left side runs first, so on the last test the read happens before the guard can stop it. Swap the two sides.`, { guide: 'compound-conditions' }); + } +} + +function hashGuards(s, parents, say) { + for (const e of statementExprs(s)) { + walkExpr(e, (x) => { + if (x.kind !== 'Index' || x.target.type?.k !== 'HASHMAP') return; + const key = sourceOf(x.indices[0]); const map = sourceOf(x.target); + const guarded = parents.some((p) => p.kind === 'If' && p.branches.some((b, i) => i === 0 && b.body.some((y) => containsLine(y, s.line)) && mentionsIn(b.cond, key, map))) + || parents.some((p) => (p.kind === 'ForEach') && sourceOf(p.iterable) === map && p.var.toUpperCase() === key.toUpperCase()); + const inElse = parents.find((p) => p.kind === 'If' && p.elseBody && p.elseBody.some((y) => containsLine(y, s.line)) && p.branches.some((b) => mentionsIn(b.cond, key, map))); + if (inElse) { say('warning', 'missing-key', x, `This line runs when ${key} IN ${map} is FALSE, and it reads ${map}[${key}], which is exactly the key that is missing.`, { guide: 'hashmap-for-lookup-by-key' }); return; } + if (!guarded && s.kind === 'Assign' && sourceOf(s.target) === sourceOf(x)) { + say('warning', 'missing-key', x, `${sourceOf(x)} is read to compute its own new value, and nothing checks that ${key} is already a key of ${map}. The first sighting of each key has no old value: test ${key} IN ${map} first.`, { guide: 'hashmap-for-lookup-by-key' }); + } + }); + } +} + +function mentionsIn(cond, key, map) { + let hit = false; + walkExpr(cond, (x) => { if (x.kind === 'Binary' && x.op === 'IN' && sourceOf(x.left) === key && sourceOf(x.right) === map) hit = true; }); + return hit; +} + +function containsLine(s, line) { + if (s.line === line) return true; + let hit = false; + walkStatements([s], (x) => { if (x.line === line) hit = true; }); + return hit; +} diff --git a/src/engines/pseudo/dom.js b/src/engines/pseudo/dom.js new file mode 100644 index 0000000..30c4982 --- /dev/null +++ b/src/engines/pseudo/dom.js @@ -0,0 +1,35 @@ +/** + * pseudo/dom.js + * + * DOM helpers for the pseudocode widgets: the shared core (el, sv, add, + * clear, commas, plural, ...) plus the helpers that carry this family's CSS + * class prefix (.pl-*, see css/widgets/pseudo-lab.css). + */ + +import { el } from '../shared/dom.js'; + +export * from '../shared/dom.js'; + +export function panel(title, body, extraClass = '', tools = null) { + return el('section', { class: `pl-panel ${extraClass}` }, [ + title || tools ? el('div', { class: 'pl-panel-head' }, [title ? el('h4', { class: 'pl-panel-title', text: title }) : null, tools]) : null, + body, + ]); +} + +export function legend(items, extraClass = '') { + return el('div', { class: `pl-legend ${extraClass}` }, items.map(([cls, label]) => el('span', { class: 'pl-legend-item' }, [el('span', { class: `pl-swatch ${cls}` }), label]))); +} + +/** A small button. */ +export function button(text, onclick, extra = {}) { + return el('button', { type: 'button', class: `pl-btn${extra.primary ? ' pl-btn-primary' : ''}${extra.class ? ` ${extra.class}` : ''}`, text, onclick, title: extra.title, 'aria-pressed': extra.pressed }); +} + +/** Where the guide lives, for links from findings to the paragraph that explains them. */ +let GUIDE = '/programmingplus/reference/pseudocode/'; +export function setGuideUrl(u) { GUIDE = u; } +export function guideLink(anchor, text = 'Read more') { + if (!anchor) return null; + return el('a', { class: 'pl-guide-link', href: `${GUIDE}#${anchor}`, target: '_blank', rel: 'noopener', text }); +} diff --git a/src/engines/pseudo/editor.js b/src/engines/pseudo/editor.js new file mode 100644 index 0000000..25a3e13 --- /dev/null +++ b/src/engines/pseudo/editor.js @@ -0,0 +1,147 @@ +/** + * pseudo/editor.js + * + * A small code editor for pseudocode: a plain textarea laid over a + * highlighted copy of its text, with line numbers and a mark on every line + * the checker or coach has something to say about. Plain DOM, no library. + * + * editor({ value, onChange, onRun, minLines }) -> { + * node, value (get/set), setDiagnostics(diags), setCurrentLine(n), + * focus(), insert(text), goToLine(n) + * } + * + * Keys: Tab indents by four spaces (a selection, line by line), Shift+Tab + * takes four away, Enter keeps the indentation and adds four after a line + * that opens a block, Ctrl+Enter or Cmd+Enter runs. + */ + +import { el, add, clear } from './dom.js'; +import { lineHtml, esc } from './highlight.js'; + +const OPENS = /^\s*(IF\b.*\bTHEN|ELSEIF\b.*\bTHEN|ELSE|FOR\b|WHILE\b|REPEAT|CASE\s+OF\b|FUNCTION\b|PROCEDURE\b|TYPE\s+\w+\s*$|OTHERWISE\s*:\s*$|.*:\s*$)\s*(\/\/.*)?$/; + +export function editor(options = {}) { + const opts = { value: '', minLines: 12, highlight: lineHtml, opens: OPENS, ...options }; + const gutter = el('div', { class: 'pl-ed-gutter', 'aria-hidden': 'true' }); + const hl = el('pre', { class: 'pl-ed-hl', 'aria-hidden': 'true' }); + const ta = el('textarea', { + class: 'pl-ed-ta', spellcheck: 'false', autocapitalize: 'off', autocomplete: 'off', autocorrect: 'off', + 'aria-label': opts.label || 'pseudocode', wrap: 'off', + }); + const body = el('div', { class: 'pl-ed-body' }, [hl, ta]); + const node = el('div', { class: 'pl-editor' }, [gutter, body]); + let diags = []; + let current = null; + ta.value = opts.value; + + function render() { + const lines = ta.value.split('\n'); + while (lines.length < opts.minLines) lines.push(''); + const byLine = new Map(); + for (const d of diags) { + const worst = byLine.get(d.line); + if (!worst || rank(d.severity) < rank(worst.severity)) byLine.set(d.line, d); + } + hl.innerHTML = lines.map((l, i) => { + const d = byLine.get(i + 1); + const cls = `pl-ed-line${d && d.severity !== 'note' ? ` is-${d.severity}` : ''}${current === i + 1 ? ' is-current' : ''}`; + return `${opts.highlight(l) || '​'}`; + }).join(''); + clear(gutter); + lines.forEach((_, i) => { + const d = byLine.get(i + 1); + const all = diags.filter((x) => x.line === i + 1); + add(gutter, el('div', { class: d ? `is-${d.severity}` : '' }, [ + el('span', { class: 'pl-ed-flag', text: d ? (d.severity === 'error' ? '✖' : d.severity === 'warning' ? '▲' : '•') : '', title: all.map((x) => x.message).join('\n\n') }), + el('span', { text: String(i + 1) }), + ])); + }); + } + const rank = (s) => ({ error: 0, warning: 1, note: 2 }[s] ?? 3); + + let timer = null; + const changed = () => { + render(); + clearTimeout(timer); + timer = setTimeout(() => opts.onChange?.(ta.value), opts.debounce ?? 250); + }; + ta.addEventListener('input', changed); + + function replaceRange(a, b, text, selA, selB) { + ta.setRangeText(text, a, b, 'end'); + if (selA != null) { ta.selectionStart = selA; ta.selectionEnd = selB ?? selA; } + changed(); + } + + ta.addEventListener('keydown', (e) => { + const v = ta.value; + const a = ta.selectionStart; const b = ta.selectionEnd; + if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) { e.preventDefault(); opts.onRun?.(ta.value); return; } + if (e.key === 'Tab') { + e.preventDefault(); + const ls = v.lastIndexOf('\n', a - 1) + 1; + const le = v.indexOf('\n', b); const end = le < 0 ? v.length : le; + if (a === b && !e.shiftKey) { replaceRange(a, b, ' ', a + 4); return; } + const block = v.slice(ls, end); + const out = block.split('\n').map((l) => (e.shiftKey ? l.replace(/^ {1,4}/, '') : ` ${l}`)).join('\n'); + replaceRange(ls, end, out, ls, ls + out.length); + return; + } + if (e.key === 'Enter' && !e.shiftKey && !e.altKey) { + e.preventDefault(); + const ls = v.lastIndexOf('\n', a - 1) + 1; + const line = v.slice(ls, a); + let indent = /^\s*/.exec(line)[0]; + if (opts.opens.test(line) && !/^\s*\/\//.test(line)) indent += ' '; + replaceRange(a, b, `\n${indent}`, a + 1 + indent.length); + } + }); + render(); + + function goToLine(n) { + const lines = ta.value.split('\n'); + let pos = 0; + for (let i = 0; i < n - 1 && i < lines.length; i++) pos += lines[i].length + 1; + ta.focus(); + ta.selectionStart = pos; ta.selectionEnd = pos + (lines[n - 1]?.length || 0); + const lh = parseFloat(getComputedStyle(ta).lineHeight) || 20; + node.scrollTop = Math.max(0, (n - 4) * lh); + } + + return { + node, + get value() { return ta.value; }, + set value(v) { ta.value = v; render(); }, + setDiagnostics(d) { diags = d || []; render(); }, + setCurrentLine(n) { current = n; render(); }, + focus() { ta.focus(); }, + goToLine, + insert(text) { + const a = ta.selectionStart; + const v = ta.value; + const ls = v.lastIndexOf('\n', a - 1) + 1; + const indent = /^\s*/.exec(v.slice(ls, a))[0]; + const t = text.split('\n').map((l, i) => (i ? indent + l : l)).join('\n'); + ta.focus(); + replaceRange(a, ta.selectionEnd, t, a + t.length); + }, + textarea: ta, + }; +} + +/** Syntax templates from the guide, for a student who does not remember a shape. */ +export const SNIPPETS = [ + ['IF', 'IF THEN\n \nENDIF'], + ['IF / ELSE', 'IF THEN\n \nELSE\n \nENDIF'], + ['CASE', 'CASE OF \n : \n OTHERWISE : \nENDCASE'], + ['FOR', 'FOR I <- 0 TO \n \nNEXT I'], + ['FOR EACH', 'FOR EACH X IN \n \nNEXT X'], + ['WHILE', 'WHILE \n \nENDWHILE'], + ['REPEAT', 'REPEAT\n \nUNTIL '], + ['FUNCTION', 'FUNCTION ( : ) RETURNS \n \n RETURN \nENDFUNCTION'], + ['PROCEDURE', 'PROCEDURE ( : )\n \nENDPROCEDURE'], + ['INVARIANT', 'INVARIANT '], + ['ENSURE', 'ENSURE RESULT == '], +]; + +export { esc }; diff --git a/src/engines/pseudo/english.js b/src/engines/pseudo/english.js new file mode 100644 index 0000000..4eb5727 --- /dev/null +++ b/src/engines/pseudo/english.js @@ -0,0 +1,3297 @@ +/** + * pseudo/english.js + * + * Plain English steps that run. A student writes the steps of an algorithm + * as ordinary sentences, and this module reads them literally: every + * sentence either means one exact thing, or it gets a question back. A + * phrase the module does not know ("P matches at i") is not an error to + * fix but a question to answer, by defining it in more precise sentences + * ("P matches at i when every letter of P is the same as the letter of T at + * i + its position"). That is the whole method of stepwise refinement: + * each vague phrase is replaced by sharper ones until nothing is left to + * interpret. + * + * readSteps(source, options) -> { + * ok, diags, routines, defs, examples, main, + * pseudo, lineMap, study, the same algorithm in the course's + * pseudocode, and which English line each + * pseudocode line came from + * display, pseudocode name -> the student's name + * callFor(values) a call of the main routine, for run() + * } + * + * A program is a list of top-level sentences: + * + * To find every start of P in T: a routine; P and T are its inputs + * Start with an empty list called Starts. + * For each position i of T: + * If P matches at i, add i to Starts. + * The answer is Starts. + * + * P matches at i when ... a yes-or-no phrase, defined + * The complement of b is 'T' when b is 'A', ..., and 'C' otherwise. + * P is a text. an input's kind, if examples do not settle it + * Example: find every start of "ATT" in "GATTACA" gives [1]. + * + * Nothing here needs a DOM; the widget lives in sharpen.js. + */ + +import { KEYWORDS, BUILTINS, CONTEXTUAL, formatValue, UNSET, chars } from './lang.js'; +import { sourceOf, parseExpression } from './parse.js'; +import { study } from './study.js'; +import { evaluateConstant } from './interp.js'; +import { editDistance as sharedEditDistance } from '../shared/text.js'; + +/* ------------------------------------------------------------------ */ +/* Types */ +/* ------------------------------------------------------------------ */ + +const T = { + INT: { k: 'INTEGER' }, FLOAT: { k: 'FLOAT' }, BOOL: { k: 'BOOL' }, CHAR: { k: 'CHAR' }, + STR: { k: 'STRING' }, LETTER: { k: 'LETTER' }, ANY: { k: 'ANY' }, + list: (of) => ({ k: 'LIST', of: of || T.ANY }), + set: (of) => ({ k: 'SET', of: of || T.ANY }), + map: (key, val) => ({ k: 'HASHMAP', key: key || T.ANY, val: val || T.ANY }), +}; +const isNum = (t) => t && (t.k === 'INTEGER' || t.k === 'FLOAT'); +const isText = (t) => t && (t.k === 'STRING' || t.k === 'CHAR' || t.k === 'LETTER'); +const unknown = (t) => !t || t.k === 'ANY'; + +/** The smallest type both values fit, or null when there is none. */ +export function join(a, b) { + if (unknown(a)) return b || T.ANY; + if (unknown(b)) return a; + if (a.k === b.k) { + if (a.k === 'LIST' || a.k === 'SET') { const of = join(a.of, b.of); return of ? { k: a.k, of } : null; } + if (a.k === 'HASHMAP') { const key = join(a.key, b.key); const val = join(a.val, b.val); return key && val ? T.map(key, val) : null; } + return a; + } + if (isNum(a) && isNum(b)) return T.FLOAT; + if (isText(a) && isText(b)) { + if (a.k === 'STRING' || b.k === 'STRING') return T.STR; + return T.CHAR; + } + return null; +} +const sameT = (a, b) => JSON.stringify(a) === JSON.stringify(b); + +/** A type as the pseudocode writes it. */ +function typeText(t) { + switch (t?.k) { + case 'INTEGER': case 'ANY': case undefined: return 'INTEGER'; + case 'LETTER': return 'CHAR'; + case 'LIST': return `LIST OF ${typeText(t.of)}`; + case 'SET': return `SET OF ${unknown(t.of) ? 'STRING' : typeText(t.of)}`; + case 'HASHMAP': return `HASHMAP OF ${unknown(t.key) ? 'STRING' : typeText(t.key)} : ${typeText(t.val)}`; + default: return t.k; + } +} + +/** A type in words, for messages: "a whole number", "a list of letters". */ +export function kindText(t, plural = false) { + const one = { + INTEGER: ['a whole number', 'whole numbers'], FLOAT: ['a decimal number', 'decimal numbers'], BOOL: ['yes or no', 'yes-or-no values'], + CHAR: ['a letter', 'letters'], LETTER: ['a letter', 'letters'], STRING: ['a text', 'texts'], ANY: ['a value', 'values'], + }[t?.k]; + if (one) return one[plural ? 1 : 0]; + if (t.k === 'LIST') return plural ? `lists of ${kindText(t.of, true)}` : `a list of ${kindText(t.of, true)}`; + if (t.k === 'SET') return plural ? `sets of ${kindText(t.of, true)}` : `a set of ${kindText(t.of, true)}`; + if (t.k === 'HASHMAP') return plural ? 'tallies' : `a tally from ${kindText(t.key, true)} to ${kindText(t.val, true)}`; + return 'a value'; +} + +/** "a text", "list of whole numbers", ... -> a type. Returns null for words it does not know. */ +export function kindFromWords(words) { + const w = words.map((x) => x.toLowerCase()).filter((x) => !['a', 'an', 'the', 'one', 'single'].includes(x)); + const s = w.join(' '); + if (!s) return null; + const m = /^(list|sequence of values|array)\s+of\s+(.+)$/.exec(s); + if (m) return T.list(kindFromWords(m[2].split(' ').map(singular)) || T.ANY); + const ms = /^set\s+of\s+(.+)$/.exec(s); + if (ms) return T.set(kindFromWords(ms[1].split(' ').map(singular)) || T.ANY); + const mt = /^(tally|table|count table|dictionary|map)(\s+of\s+(.+))?$/.exec(s); + if (mt) return T.map(mt[3] ? kindFromWords(mt[3].split(' ').map(singular)) : T.ANY, T.INT); + if (/^(text|string|sequence|dna sequence|dna|rna|read|word|strand|primer|pattern|genome|protein|k-?mer|codon|window)s?$/.test(s)) return T.STR; + if (/^(letter|base|character|char|nucleotide|symbol)s?$/.test(s)) return T.CHAR; + if (/^(whole number|integer|int|count|position|index|length|number|natural number|size|score)s?$/.test(s)) return T.INT; + if (/^(decimal|decimal number|fraction|real number|float|proportion|ratio|percentage|average|mean)s?$/.test(s)) return T.FLOAT; + if (/^(yes or no|yes-or-no|true or false|flag|bool|boolean|truth value)s?$/.test(s)) return T.BOOL; + return null; +} +function singular(w) { return /ss$/.test(w) ? w : w.replace(/ies$/, 'y').replace(/(ch|sh|x)es$/, '$1').replace(/s$/, ''); } + +/* ------------------------------------------------------------------ */ +/* Words */ +/* ------------------------------------------------------------------ */ + +const LETTER_WORDS = ['letter', 'letters', 'base', 'bases', 'character', 'characters', 'nucleotide', 'nucleotides', 'char', 'chars', 'symbol', 'symbols']; +const ITEM_WORDS = ['item', 'items', 'value', 'values', 'element', 'elements', 'entry', 'entries', 'score', 'scores', 'word', 'words', 'number', 'numbers']; +const POSITION_WORDS = ['position', 'positions', 'index', 'indices', 'indexes', 'place', 'places', 'start', 'starts', 'offset', 'offsets']; +const ORDINALS = { first: 1, second: 2, third: 3, fourth: 4, fifth: 5, sixth: 6, seventh: 7, eighth: 8, ninth: 9, tenth: 10 }; +const NUMBER_WORDS = { zero: 0, one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, ten: 10 }; +const ARTICLES = new Set(['the', 'a', 'an']); +const STOP_AFTER_VALUE = new Set(['is', 'are', 'and', 'or', 'then', 'to', 'at', 'of', 'in', 'from', 'by', 'when', 'where', 'with', 'if', 'otherwise', 'gives', 'give', 'up', 'down', 'through', 'until', 'starting', 'plus', 'minus', 'times', 'divided', 'mod', 'equals', 'does', 'differs', 'exceeds', 'has', 'have', 'contains', 'for', 'that', 'which', 'as', 'into', 'called', 'named', 'be', 'becomes', 'goes', 'starts', 'begins', 'still']); + +const STEP_VERBS = ['to', 'if', 'otherwise', 'else', 'for', 'repeat', 'while', 'until', 'start', 'let', 'set', 'make', 'add', 'append', 'put', 'insert', 'subtract', 'increase', 'decrease', 'multiply', 'count', 'tally', 'remove', 'sort', 'swap', 'show', 'print', 'check', 'stop', 'skip', 'the', 'answer', 'return']; + +/** How many single-letter edits turn one word into another, ignoring case; a swap of two neighbors counts as one. */ +const editDistance = (a, b) => sharedEditDistance(a, b, { fold: true, swaps: true }); +/** The known word closest to a misspelled one, if any is close. */ +function closest(word, candidates) { + let best = null; let bd = Infinity; + for (const c of candidates) { const dist = editDistance(word, c); if (dist < bd && dist > 0) { bd = dist; best = c; } } + return best && bd <= (word.length > 5 ? 2 : 1) ? best : null; +} + +/* ------------------------------------------------------------------ */ +/* Tokens and lines */ +/* ------------------------------------------------------------------ */ + +const SYMBOLS = ['<-', '<=', '>=', '!=', '==', '->', '=', '<', '>', '+', '-', '*', '/', '(', ')', '[', ']', '{', '}', ',', ':', '.', ';', '&', '%', '≤', '≥', '≠', '←', '×', '÷']; +const SYMBOL_ALIAS = { '≤': '<=', '≥': '>=', '≠': '!=', '←': '<-', '×': '*', '÷': '/' }; + +/** Tokens of one line: words, numbers, quoted text, and symbols, each with its column. */ +export function tokenize(text, offset = 0) { + const toks = []; + const bad = []; + let i = 0; + while (i < text.length) { + const c = text[i]; + if (/\s/.test(c)) { i++; continue; } + const col = offset + i + 1; + // quoted text: a quote not glued to a word before it + if ('"“”\'‘’`'.includes(c) && !(i > 0 && /[A-Za-z0-9]/.test(text[i - 1]))) { + const closers = c === '"' || c === '“' || c === '”' ? '"”“' : c === '`' ? '`' : '\'’‘'; + let j = i + 1; + while (j < text.length && !closers.includes(text[j])) j++; + if (j >= text.length) { bad.push({ col, end: offset + text.length + 1, message: `The text that starts here has no closing quote.` }); toks.push({ t: 's', v: text.slice(i + 1), q: c, col, end: offset + text.length + 1 }); break; } + toks.push({ t: 's', v: text.slice(i + 1, j), q: closers[0] === '"' ? '"' : '\'', col, end: offset + j + 2 }); + i = j + 1; + continue; + } + const num = /^\d+(?:\.\d+)?(?:[eE][-+]?\d+)?/.exec(text.slice(i)); + if (num) { toks.push({ t: 'n', v: num[0], float: /[.eE]/.test(num[0]), col, end: col + num[0].length }); i += num[0].length; continue; } + const word = /^[A-Za-z_][A-Za-z0-9_]*(?:['’][A-Za-z]+)?/.exec(text.slice(i)); + if (word) { + let w = word[0]; + const ap = /^(.*?)(['’])([A-Za-z]+)$/.exec(w); + if (ap && /^t$/i.test(ap[3]) && /n$/i.test(ap[1])) { + // isn't -> is not; can't -> can not + const base = /^can$/i.test(ap[1]) ? 'can' : ap[1].slice(0, -1); + toks.push({ t: 'w', v: base, lc: base.toLowerCase(), col, end: col + base.length }); + toks.push({ t: 'w', v: 'not', lc: 'not', col: col + base.length, end: col + w.length }); + } else if (ap && /^s$/i.test(ap[3])) { + toks.push({ t: 'w', v: ap[1], lc: ap[1].toLowerCase(), col, end: col + ap[1].length }); + toks.push({ t: 'y', v: "'s", col: col + ap[1].length, end: col + w.length }); + } else { + if (ap) w = ap[1]; + toks.push({ t: 'w', v: w, lc: w.toLowerCase(), col, end: col + word[0].length }); + } + i += word[0].length; + continue; + } + const sym = SYMBOLS.find((s) => text.startsWith(s, i)); + if (sym) { toks.push({ t: 'y', v: SYMBOL_ALIAS[sym] || sym, col, end: col + sym.length }); i += sym.length; continue; } + bad.push({ col, end: col + 1, message: `The character ${c} means nothing here.` }); + i++; + } + return { toks, bad }; +} + +/** Lines, with their indentation, list markers and comments removed, and sentences split. */ +function readLines(source) { + const raw = String(source).replace(/\r\n?/g, '\n').split('\n'); + const out = []; + raw.forEach((text, k) => { + let body = text.replace(/\t/g, ' '); + const indent = /^ */.exec(body)[0].length; + // a // comment, outside quotes + let inQ = null; + for (let i = 0; i < body.length; i++) { + const c = body[i]; + if (inQ) { if (c === inQ || (inQ === '"' && c === '”')) inQ = null; continue; } + if ((c === '"' || c === '“')) { inQ = '"'; continue; } + if (c === '/' && body[i + 1] === '/') { body = body.slice(0, i); break; } + } + let start = indent; + const marker = /^(\d+[.)]|[-*•]|[a-z][.)])\s+/.exec(body.slice(indent)); + if (marker) start += marker[0].length; + const content = body.slice(start).trimEnd(); + if (!content.trim() || /^(note|n\.b\.)\s*:/i.test(content.trim())) return; + out.push({ line: k + 1, indent, text: content, offset: start, full: text }); + }); + return out; +} + +/* ------------------------------------------------------------------ */ +/* Expression nodes */ +/* ------------------------------------------------------------------ */ +// Expression nodes are the pseudocode's own AST shapes, so sourceOf() can +// print them. `text` keeps the student's words for narration. + +const N = { + num: (v, float) => ({ kind: 'Num', value: Number(v), float: Boolean(float) }), + str: (v, q) => ({ kind: 'Str', value: v, quote: q }), + bool: (v) => ({ kind: 'Bool', value: v }), + name: (entry) => ({ kind: 'Name', name: entry.pseudo, entry }), + index: (target, i) => ({ kind: 'Index', target, indices: [i] }), + call: (name, args, extra = {}) => ({ kind: 'CallExpr', name, args, ...extra }), + bin: (op, left, right) => ({ kind: 'Binary', op, left, right }), + un: (op, arg) => ({ kind: 'Unary', op, arg }), + len: (x) => ({ kind: 'CallExpr', name: 'LENGTH', args: [x], builtin: 'LENGTH' }), +}; +const add1 = (e) => (e.kind === 'Num' && !e.float ? N.num(e.value + 1) : e.kind === 'Binary' && e.op === '-' && e.right.kind === 'Num' && e.right.value === 1 ? e.left : N.bin('+', e, N.num(1))); +const sub1 = (e) => (e.kind === 'Num' && !e.float ? N.num(e.value - 1) : e.kind === 'Binary' && e.op === '+' && e.right.kind === 'Num' && e.right.value === 1 ? e.left : N.bin('-', e, N.num(1))); + +/* ------------------------------------------------------------------ */ +/* The reader */ +/* ------------------------------------------------------------------ */ + +/** + * A reading that does not fit, thrown to back out of a try. It carries nothing, so one instance serves every throw: + * building an Error captures a stack trace, and a reader that tries each routine's words at each sentence throws + * thousands of them. + */ +class Stop extends Error { } +const STOP = new Stop(); +/** A sentence nested past MAX_NESTING: said once, and not retried another way (tryParse lets it through). */ +class TooDeep extends Stop { } + +/** + * How deep steps and values may nest (brackets, "not", "minus", quantifiers, steps inside steps), and how many + * words and symbols one sentence may have. Every stage after the reader (the types, the pseudocode it writes, + * the narration) walks a sentence's tree by calling itself, so without these a link holding thousands of + * brackets would end in the engine's own stack error. The pseudocode has the same nesting limit + * (src/engines/pseudo/parse.js), and a sentence's tree can be no taller than its words (docs/limits.md). + */ +const MAX_NESTING = 200; +const MAX_SENTENCE = 1000; +/** Every value in a sentence is tried against every routine's and phrase's first line, so a program is read with at most this many. */ +const MAX_HEADS = 200; +/** The longest pseudocode the steps may become: it is checked and run as pseudocode, whose checks grow faster than its length. */ +const MAX_PSEUDO_LINES = 2000; + +/** + * A phrase pattern ("the? length|size of") as its words, each optional or not, with its alternatives: split once and + * kept, since every value of every sentence is tried against dozens of patterns. + */ +const PATTERNS = new Map(); +function wordsOf(pattern) { + let w = PATTERNS.get(pattern); + if (!w) { + w = pattern.split(' ').map((p) => { const opt = p.endsWith('?') && p.length > 1; return { opt, alts: (opt ? p.slice(0, -1) : p).split('|') }; }); + PATTERNS.set(pattern, w); + } + return w; +} + +class Reader { + constructor(source, options) { + this.source = String(source); + this.options = options; + this.lines = readLines(source); + this.rawLines = this.source.replace(/\r\n?/g, '\n').split('\n'); + this.diags = []; + this.routines = []; + this.defs = []; + this.examples = []; + this.kinds = new Map(); // input name (lowercase) -> type, from "P is a text." + this.context = new Map(); // names of every routine's inputs, visible in definitions + this.fresh = 0; + this.fnNames = new Map(); // FUNCTION names -> the routine or phrase that owns it + this.allVars = new Set(); + this.space = new Set(); // variable names of the routine being read + } + + depth = 0; // steps and values open around the one being read, across the sentences read inside one another + + diag(severity, line, col, endCol, message, extra = {}) { + const d = { severity, line, col: col ?? 1, endCol: endCol ?? null, message, source: 'english', ...extra }; + this.diags.push(d); + return d; + } + + /** + * A pseudocode name for a student's name: words joined by _, never a + * keyword. Function names are global; a variable's name only has to be + * unique within its routine, and a variable keeps its own name even when + * a function wanted it (the function is renamed, since students read + * variable names far more often). + */ + pseudoName(words, kind = 'var', space = this.space) { + let base = words.join('_').replace(/[^A-Za-z0-9_]/g, ''); + if (!/^[A-Za-z_]/.test(base)) base = `v_${base}`; + if (kind === 'fn') base = base.toUpperCase(); + const U = base.toUpperCase(); + if (KEYWORDS.has(U) || BUILTINS[U] || CONTEXTUAL[U] || ['EACH', 'RESULT', 'OLD', 'SUM', 'MAIN', 'PASS'].includes(U)) base = `${base}_`; + const taken = (n) => (kind === 'fn' ? this.fnNames.has(n.toUpperCase()) || this.allVars.has(n.toUpperCase()) : space.has(n.toUpperCase())); + let name = base; let k = 2; + while (taken(name)) name = `${base}_${k++}`; + if (kind === 'fn') this.fnNames.set(name.toUpperCase(), null); + else { + space.add(name.toUpperCase()); + this.allVars.add(name.toUpperCase()); + const owner = this.fnNames.get(name.toUpperCase()); + if (this.fnNames.has(name.toUpperCase())) { + this.fnNames.delete(name.toUpperCase()); + if (owner) { owner.pseudo = this.pseudoName([`${name}_of`], 'fn'); this.fnNames.set(owner.pseudo.toUpperCase(), owner); } + } + } + return name; + } + claimFn(owner) { this.fnNames.set(owner.pseudo.toUpperCase(), owner); } + freshName(hint) { return this.pseudoName([`${hint}${++this.fresh}`]); } + + /* ---------- structure ---------- */ + + /** Group lines into sentences with children, by indentation. */ + structure() { + // a program longer than its pseudocode may be is not read past that line: reading it would take seconds for nothing + if (this.lines.length > MAX_PSEUDO_LINES) { + const cut = this.lines[MAX_PSEUDO_LINES]; + this.diag('error', cut.line, 1, null, `These steps are more than ${MAX_PSEUDO_LINES.toLocaleString('en-US')} lines long, which is more than this tool reads. Keep the steps to one algorithm at a time.`, { code: 'too-long' }); + this.lines = this.lines.slice(0, MAX_PSEUDO_LINES); + } + const items = []; + const stack = [{ indent: -1, children: items }]; + let prev = null; + for (const L of this.lines) { + const { toks, bad } = tokenize(L.text, L.offset); + for (const b of bad) this.diag('error', L.line, b.col, b.end, b.message); + // a line indented under one that does not open a block continues it + if (prev && L.indent > prev.indent && !prev.opens && !startsBlock(prev.toks)) { + // one at a time: push(...toks) passes every token as an argument, past the engine's limit on a long line + for (const t of toks) { if (prev.toks.length >= MAX_SENTENCE) { this.tooLong(prev, L.line); break; } prev.toks.push(t); } + prev.opens = lastIs(prev.toks, ':'); + continue; + } + while (stack.length > 1 && stack[stack.length - 1].indent >= L.indent) stack.pop(); + // sentences on one line: split at a full stop followed by more words + const parts = splitSentences(toks); + let node = null; + for (const p of parts) { + node = { line: L.line, indent: L.indent, toks: p, children: [], opens: lastIs(p, ':') }; + if (p.length > MAX_SENTENCE) { node.toks = p.slice(0, MAX_SENTENCE); this.tooLong(node, L.line); } + stack[stack.length - 1].children.push(node); + } + if (node) { stack.push({ indent: L.indent, children: node.children }); prev = node; } + } + return items; + } + + /** A sentence past MAX_SENTENCE words and symbols: one error, and only its first MAX_SENTENCE are read. */ + tooLong(node, line) { + if (node.tooLong) return; + node.tooLong = true; + this.diag('error', line, node.toks[0]?.col ?? 1, null, `This sentence has more than ${MAX_SENTENCE.toLocaleString('en-US')} words and symbols, which is more than this tool reads. Split it into shorter steps.`, { code: 'too-long' }); + } + + /* ---------- the program ---------- */ + + read() { + const items = this.structure(); + // 1. heads first, so a phrase can be used before the sentence defining it + const tops = items.map((it) => ({ it, kind: topKind(it) })); + const heads = tops.filter((t) => t.kind === 'routine' || t.kind === 'predicate' || t.kind === 'value'); + if (heads.length > MAX_HEADS) { + const first = heads[MAX_HEADS].it; + this.diag('error', first.line, first.toks[0]?.col ?? 1, null, `This program has more than ${MAX_HEADS} routines and phrases, which is more than this tool reads. The ones from here on are not read.`, { code: 'too-long' }); + for (const t of heads.slice(MAX_HEADS)) t.kind = 'unread'; + } + // what each input is, wherever the sentence saying so sits + for (const { it, kind } of tops) if (kind === 'kind') this.kindLine(it); + for (const { it, kind } of tops) { + if (kind === 'routine') this.routineHead(it); + else if (kind === 'predicate' || kind === 'value') this.defHead(it, kind); + } + // 2. bodies + for (const { it, kind } of tops) { + try { + if (kind === 'routine') this.routineBody(it); + else if (kind === 'predicate' || kind === 'value') this.defBody(it); + else if (kind === 'example') this.example(it); + else if (kind === 'kind' || kind === 'unread') { /* done, or past MAX_HEADS */ } else this.topLevelProblem(it); + } catch (err) { if (!(err instanceof Stop)) throw err; } + } + return this; + } + + topLevelProblem(it) { + const t = it.toks; + const first = t[0]; + if (!first) return; + const text = sentenceText(t); + if (it.indent > 0 && !this.routines.length) { + this.diag('error', it.line, first.col, lastEnd(t), `This step is indented, and nothing above it opens a block of steps. Start the algorithm with a line such as "To find every start of P in T:" and indent the steps under it.`, { code: 'no-routine' }); + return; + } + this.diag('error', it.line, first.col, lastEnd(t), `"${text}" sits outside any routine. At the top level, a sentence starts a routine ("To count the G bases in T:"), defines a phrase ("P matches at i when ..."), says an input's kind ("T is a text."), or gives an example ("Example: count the G bases in "GAG" gives 2.").`, { code: 'top-level' }); + } + + kindLine(it) { + const t = trimEnd(it.toks); + const k = t.findIndex((x) => x.lc === 'is' || x.lc === 'are'); + const names = t.slice(0, k).filter((x) => x.t === 'w' && x.lc !== 'and').map((x) => x.v); + const kind = kindFromWords(t.slice(k + 1).filter((x) => x.t === 'w').map((x) => x.v)); + if (!kind) { this.diag('error', it.line, t[k + 1]?.col, lastEnd(t), `"${sentenceText(t.slice(k + 1))}" is not a kind this tool knows. Say "a text", "a letter", "a whole number", "a decimal number", "yes or no", or "a list of whole numbers".`, { code: 'kind' }); return; } + for (const n of names) this.kinds.set(n.toLowerCase(), kind); + } + + /** "To find every start of P in T:" -> a routine with inputs P and T. */ + routineHead(it) { + const t = trimEnd(it.toks).slice(1); + const used = usedWords([]); + const walk = (items) => { for (const c of items) { usedWords(c.toks, used); walk(c.children); } }; + walk(it.children); + const head = this.head(t, it.line, true, used); + if (head.slots.length) this.diag('note', it.line, head.slots[0].col, lastEnd(t), `Reading ${head.slots.map((x) => x.v).join(head.slots.length === 2 ? ' and ' : ', ')} as the ${head.slots.length === 1 ? 'input' : 'inputs'} of this routine: ${head.slots.length === 1 ? 'the name in its first line that' : 'the names in its first line that'} the steps use.`, { code: 'inputs' }); + const r = { kind: 'routine', line: it.line, head, it, body: [], answers: 0, scope: null, params: [], ret: T.ANY, calls: [], space: new Set() }; + r.words = head.words; + r.pseudo = this.pseudoName(head.words.length ? head.words : ['main'], 'fn'); + this.claimFn(r); + r.params = head.slots.map((s) => ({ display: s.v, key: s.v.toLowerCase(), words: [s.v.toLowerCase()], pseudo: this.pseudoName([s.v], 'var', r.space), kind: 'param', line: it.line, type: this.kinds.get(s.v.toLowerCase()) || T.ANY })); + for (const p of r.params) if (!this.context.has(p.key)) this.context.set(p.key, p); + this.routines.push(r); + if (!it.opens) this.diag('warning', it.line, lastEnd(it.toks), null, 'A routine line ends with a colon, and its steps go on the lines under it, indented.', { code: 'colon' }); + if (!it.children.length) this.diag('error', it.line, it.toks[0].col, lastEnd(it.toks), `"${sentenceText(it.toks)}" has no steps under it. Indent the steps on the lines below it.`, { code: 'no-steps' }); + } + + /** "P matches at i when ..." / "The complement of b is ..." */ + defHead(it, kind) { + const t = trimEnd(it.toks); + let split; + if (kind === 'predicate') split = t.findIndex((x) => x.t === 'w' && ['when', 'means', 'iff'].includes(x.lc)); + else split = t.findIndex((x, i) => x.t === 'w' && (x.lc === 'is' || x.lc === 'are') && i > 0); + const headToks = t.slice(0, split); + const used = usedWords(t.slice(split + 1)); + const head = this.head(headToks, it.line, false, used); + const space = new Set(); + const d = { + kind, line: it.line, head, it, split, words: head.words, space, + pseudo: this.pseudoName(head.words.length ? head.words : ['phrase'], 'fn'), + params: head.slots.map((s) => ({ display: s.v, key: s.v.toLowerCase(), words: [s.v.toLowerCase()], pseudo: this.pseudoName([s.v], 'var', space), kind: 'param', line: it.line, type: this.kinds.get(s.v.toLowerCase()) || T.ANY })), + implicit: [], ret: kind === 'predicate' ? T.BOOL : T.ANY, calls: [], + }; + this.claimFn(d); + this.defs.push(d); + } + + /** + * A head's words and slots. A slot is a single letter, a Capitalized name + * after the first word, or the name of a known input. + */ + head(toks, line, isRoutine, used = null) { + const elems = []; + const slots = []; + const words = []; + toks.forEach((x, i) => { + if (x.t !== 'w') { if (x.t !== 'y' || x.v !== "'s") elems.push({ lit: x.v }); return; } + const lead = i === 0 && ARTICLES.has(x.lc); + if (lead) return; + // A word stands for an input when it looks like a name (one letter, or + // Capitalized after the first word) and the steps use it as one. The + // G and C of "count the G and C bases in T" are just words. + // One letter counts only in the same case ("T" is not the "t" of a + // sentence), and a word the steps give a value to is theirs, not an input. + const single = /^[A-Za-z]$/.test(x.v); + const capital = i > 0 && /^[A-Z][a-z0-9]+$/.test(x.v); + const reads = !used || (single ? used.exact.has(x.v) : used.lc.has(x.lc)); + const own = used && used.assigned.has(x.lc); + const slot = ((single && x.v !== 'a') || capital) ? reads && !own : (!isRoutine && this.context.has(x.lc) && reads && !own); + if (slot) { elems.push({ slot: x.v }); slots.push(x); } else { elems.push({ lit: x.lc }); if (!['of', 'in', 'at', 'the', 'a', 'an', 'to', 'and', 'for', 'on', 'from', 'with', 'by'].includes(x.lc)) words.push(x.lc); } + }); + const dup = slots.find((s, i) => slots.findIndex((z) => z.v.toLowerCase() === s.v.toLowerCase()) !== i); + if (dup) this.diag('error', line, dup.col, dup.end, `${dup.v} appears twice in "${sentenceText(toks)}". Each input needs its own name.`, { code: 'dup-slot' }); + if (!elems.some((e) => e.lit)) this.diag('error', line, toks[0]?.col, lastEnd(toks), `"${sentenceText(toks)}" is only names. Give it words that say what it does, such as "find every start of P in T".`, { code: 'head' }); + return { elems, slots, words, text: sentenceText(toks) }; + } + + routineBody(it) { + const r = this.routines.find((x) => x.it === it); + this.space = r.space; + r.scope = new Scope(null); + for (const p of r.params) r.scope.add(p); + const ctx = { routine: r, owner: r, loops: 0, scope: r.scope }; + r.body = this.block(it.children, ctx); + r.isFunction = r.answers > 0; + } + + defBody(it) { + const d = this.defs.find((x) => x.it === it); + this.space = d.space; + const scope = new Scope(null); + for (const p of d.params) scope.add(p); + // the inputs of the routines are visible, as extra inputs this phrase takes + scope.implicitFrom = this.context; + scope.onImplicit = (entry) => { + if (!d.implicit.some((x) => x.key === entry.key)) d.implicit.push({ ...entry, pseudo: this.pseudoName([entry.display], 'var', d.space), kind: 'param', outer: entry }); + return d.implicit.find((x) => x.key === entry.key); + }; + const ctx = { routine: null, owner: d, loops: 0, scope }; + const t = trimEnd(it.toks); + const rest = t.slice(d.split + 1); + if (!rest.length) { this.diag('error', it.line, lastEnd(t), null, `Say what "${d.head.text}" means after "${t[d.split].v}".`, { code: 'def-empty' }); return; } + const s = new Sentence(rest, this, ctx, it.line); + try { + if (d.kind === 'predicate') { + d.cond = s.condition(); + s.end(); + } else { + d.pieces = s.pieces(); + s.end(); + } + } catch (err) { if (!(err instanceof Stop)) throw err; } + } + + example(it) { + const t = trimEnd(it.toks); + let k = 0; + if (t[0]?.lc === 'example' || t[0]?.lc === 'examples') k = 1; + if (t[0]?.lc === 'for' && t[1]?.lc === 'example') k = 2; + if (t[k]?.v === ':' || t[k]?.v === ',') k++; + const body = t.slice(k); + const g = body.findIndex((x, i) => (x.t === 'w' && ['gives', 'give', 'returns', 'shows', 'prints', 'displays'].includes(x.lc)) || x.v === '->' || (x.lc === 'should' && ['give', 'show'].includes(body[i + 1]?.lc))); + const shows = g >= 0 && (['shows', 'prints', 'displays'].includes(body[g].lc) || (body[g].lc === 'should' && body[g + 1]?.lc === 'show')); + if (g < 0) { this.diag('error', it.line, t[0].col, lastEnd(t), 'An example says what a call gives: "Example: count the G bases in "GAG" gives 2."', { code: 'example' }); return; } + const callToks = body.slice(0, g); + const valToks = body.slice(g + (body[g].lc === 'should' ? 2 : 1)); + const ctx = { routine: null, owner: null, loops: 0, scope: new Scope(null), constant: true }; + const s = new Sentence(callToks, this, ctx, it.line); + const call = s.routineCall(); + if (!call) { this.diag('error', it.line, callToks[0]?.col, lastEnd(callToks), `"${sentenceText(callToks)}" does not match any routine's first line. An example uses the same words, with values in place of the inputs.`, { code: 'example-call' }); return; } + const vs = new Sentence(valToks, this, ctx, it.line); + let expect; + try { expect = vs.value(); vs.end(); } catch (err) { if (err instanceof Stop) return; throw err; } + this.examples.push({ line: it.line, routine: call.routine, args: call.args, expect: shows ? null : expect, shows: shows ? expect : null, text: sentenceText(body) }); + } + + /* ---------- steps ---------- */ + + block(items, ctx) { + const out = []; + let prevIf = null; + for (const it of items) { + let st = null; + try { + st = this.statement(it, ctx, prevIf); + } catch (err) { if (!(err instanceof Stop)) throw err; st = null; } + if (st && st.kind === 'Otherwise') { prevIf = st.attachTo; continue; } + const ended = out.length && ['Return', 'Break', 'Continue'].includes(out[out.length - 1].kind) ? out[out.length - 1] : null; + if (st && ended && !ended.warnedAfter) { + ended.warnedAfter = true; + const why = ended.kind === 'Return' ? (ended.value ? 'hands back the answer and ends the routine' : 'ends the routine') : ended.kind === 'Break' ? 'stops the loop' : 'skips to the next pass'; + this.diag('error', it.line, it.toks[0]?.col, null, `This step never runs: the step before it (line ${ended.line}) ${why}. Move it before that step, or put the answer at the end.`, { code: 'unreachable' }); + } + if (st) out.push(st); + prevIf = st && st.kind === 'If' ? st : null; + } + return out; + } + + statement(it, ctx, prevIf) { + const s = new Sentence(trimEnd(it.toks), this, ctx, it.line); + const st = s.statement(it, prevIf); + if (st) st.line = it.line; + return st; + } +} + +/** The words a body uses, in both cases, and the ones its steps give values to. */ +function usedWords(toks, into = null) { + const u = into || { lc: new Set(), exact: new Set(), assigned: new Set() }; + toks.forEach((x, k) => { + if (x.t !== 'w') return; + // "the" and "a" are articles, never inputs + if (!(ARTICLES.has(x.lc) && x.v !== 'A')) { u.lc.add(x.lc); u.exact.add(x.v); } + const prev = toks[k - 1]?.lc; const prev2 = toks[k - 2]?.lc; + if ((prev === 'with' && prev2 === 'start') || ['let', 'called', 'named'].includes(prev) || (prev === 'set' && k === 1) || (prev === 'start' && k === 1)) u.assigned.add(x.lc); + if (ARTICLES.has(prev) && (['with', 'let', 'called', 'named'].includes(prev2) || (prev2 === 'set' && k === 2))) u.assigned.add(x.lc); + }); + return u; +} + +/* ------------------------------------------------------------------ */ +/* Scopes */ +/* ------------------------------------------------------------------ */ + +class Scope { + constructor(parent) { this.parent = parent; this.names = new Map(); this.implicitFrom = null; this.onImplicit = null; } + add(entry) { this.names.set(entry.key, entry); return entry; } + get(key) { + if (this.names.has(key)) return this.names.get(key); + if (this.parent) return this.parent.get(key); + return null; + } + all() { const m = new Map(this.parent ? this.parent.all() : []); for (const [k, v] of this.names) m.set(k, v); return m; } + canImplicit(key) { return Boolean((this.implicitFrom && this.implicitFrom.has(key)) || (this.parent && this.parent.canImplicit(key))); } + implicit(key) { + if (this.implicitFrom && this.implicitFrom.has(key) && this.onImplicit) return this.onImplicit(this.implicitFrom.get(key)); + return this.parent ? this.parent.implicit(key) : null; + } +} + +/* ------------------------------------------------------------------ */ +/* Sentence helpers */ +/* ------------------------------------------------------------------ */ + +function lastIs(toks, v) { return toks.length > 0 && toks[toks.length - 1].v === v; } +function trimEnd(toks) { let t = toks; while (t.length && (t[t.length - 1].v === '.' || t[t.length - 1].v === ':' || t[t.length - 1].v === ';')) t = t.slice(0, -1); return t; } +function lastEnd(toks) { return toks.length ? toks[toks.length - 1].end : null; } +function sentenceText(toks) { + let s = ''; + toks.forEach((x, i) => { + const v = x.t === 's' ? `${x.q}${x.v}${x.q}` : x.v; + const glue = i === 0 || [',', '.', ':', ';', ')', ']', "'s"].includes(v) || ['(', '['].includes(toks[i - 1]?.v); + s += (glue ? '' : ' ') + v; + }); + return s; +} +function splitSentences(toks) { + const parts = []; + let cur = []; + toks.forEach((x, i) => { + cur.push(x); + if ((x.v === '.' || x.v === ';') && x.t === 'y' && i < toks.length - 1) { parts.push(cur); cur = []; } + }); + if (cur.length) parts.push(cur); + return parts; +} +function startsBlock(toks) { + const w = toks[0]?.lc; + return ['to', 'if', 'for', 'while', 'repeat', 'until', 'otherwise', 'else', 'keep', 'do'].includes(w); +} +function topKind(it) { + const t = trimEnd(it.toks); + const w0 = t[0]?.lc; + if (w0 === 'to' && t.length > 1) return 'routine'; + if (w0 === 'example' || w0 === 'examples' || (w0 === 'for' && t[1]?.lc === 'example')) return 'example'; + const when = t.findIndex((x) => x.t === 'w' && ['when', 'means', 'iff'].includes(x.lc)); + const is = t.findIndex((x, i) => x.t === 'w' && (x.lc === 'is' || x.lc === 'are') && i > 0); + if (is > 0 && t.slice(0, is).every((x) => x.t === 'w' && (x.lc === 'and' || /^[A-Za-z][A-Za-z0-9_]*$/.test(x.v))) && ['a', 'an'].includes(t[is + 1]?.lc) && kindFromWords(t.slice(is + 1).filter((x) => x.t === 'w').map((x) => x.v))) return 'kind'; + if (when > 0 && (is < 0 || when < is || !ARTICLES.has(w0))) return 'predicate'; + if (is > 0 && ARTICLES.has(w0)) return 'value'; + if (when > 0) return 'predicate'; + if (is > 0) return 'value'; + return 'other'; +} + +export const COMPARISONS = [ + ['is not equal to', '!='], ['is not the same as', '!='], ['are not the same as', '!='], ['is different from', '!='], ['are different from', '!='], ['does not equal', '!='], ['differs from', '!='], ['is not', '!='], ['are not', '!='], ['!=', '!='], + ['is less than or equal to', '<='], ['is at most', '<='], ['is no more than', '<='], ['is not more than', '<='], ['are at most', '<='], ['<=', '<='], + ['is greater than or equal to', '>='], ['is at least', '>='], ['is no less than', '>='], ['is not less than', '>='], ['are at least', '>='], ['>=', '>='], + ['is less than', '<'], ['is smaller than', '<'], ['is fewer than', '<'], ['is below', '<'], ['is under', '<'], ['are less than', '<'], ['<', '<'], + ['is greater than', '>'], ['is more than', '>'], ['is larger than', '>'], ['is bigger than', '>'], ['is above', '>'], ['is over', '>'], ['exceeds', '>'], ['are greater than', '>'], ['are more than', '>'], ['>', '>'], + ['is equal to', '=='], ['is the same as', '=='], ['are the same as', '=='], ['equals', '=='], ['is', '=='], ['are', '=='], ['==', '=='], ['=', '=='], +]; +export const IN_PHRASES = [ + ['is|are not one of', 'notin'], ['is|are not in', 'notin'], ['is|are not a key of', 'notin'], ['is|are not a member of', 'notin'], ['is|are not among', 'notin'], + ['is|are one of', 'in'], ['is|are in', 'in'], ['is|are a key of', 'in'], ['is|are a member of', 'in'], ['is|are among', 'in'], +]; + +/* ------------------------------------------------------------------ */ +/* One sentence */ +/* ------------------------------------------------------------------ */ + +class Sentence { + constructor(toks, rd, ctx, line) { + this.toks = toks; this.rd = rd; this.ctx = ctx; this.line = line; this.i = 0; + } + + /* ---------- token helpers ---------- */ + tok(k = 0) { return this.toks[this.i + k]; } + word(k = 0) { const t = this.toks[this.i + k]; return t ? (t.t === 'w' ? t.lc : t.t === 'y' ? t.v : null) : null; } + atEnd() { return this.i >= this.toks.length; } + /** Does the phrase match at position k? Words separated by spaces; a|b alternatives; x? optional. */ + match(pattern, k = this.i) { + // A string pattern is words, each with | alternatives and ? for optional: + // "the? length|size of". An array is a list of whole phrases. + if (Array.isArray(pattern)) { + // whole-phrase alternatives: the longest one that matches + let best = -1; + for (const alt of pattern) { const n = this.matchWords(alt, k); if (n > best) best = n; } + return best; + } + return this.matchWords(pattern, k); + } + matchWords(pattern, k) { + let j = k; + for (const { opt, alts } of wordsOf(pattern)) { + const t = this.toks[j]; + const v = t ? (t.t === 'w' ? t.lc : t.t === 'y' ? t.v : null) : null; + if (v != null && alts.includes(v)) { j++; continue; } + if (opt) continue; + return -1; + } + return j - k; + } + take(pattern) { const n = this.match(pattern); if (n < 0) return false; this.i += n; return true; } + expect(pattern, what) { + if (this.take(pattern)) return; + const t = this.tok() || this.toks[this.toks.length - 1]; + this.fail(t, `Expected ${what || `"${pattern.replace(/\|/g, '" or "').replace(/\?/g, '')}"`} ${this.tok() ? `here, and found "${this.rest()}"` : 'at the end of this sentence'}.`); + } + rest(from = this.i) { return sentenceText(this.toks.slice(from)); } + text(a, b) { return sentenceText(this.toks.slice(a, b)); } + colOf(k) { return this.toks[k]?.col ?? lastEnd(this.toks) ?? 1; } + endOf(k) { return this.toks[k - 1]?.end ?? null; } + fail(tok, message, extra = {}) { + const t = tok || this.toks[this.toks.length - 1]; + this.rd.diag('error', this.line, t?.col, t?.end, message, extra); + throw STOP; + } + failSpan(a, b, message, extra = {}) { + this.rd.diag('error', this.line, this.colOf(a), this.endOf(b), message, extra); + throw STOP; + } + end() { + if (!this.atEnd()) { + const t = this.tok(); + this.fail(t, `The sentence should end before "${this.rest()}". ${hintFor(this.rest())}`.trim(), { code: 'extra' }); + } + } + mark(node, a) { node.text = this.text(a, this.i); node.span = [this.colOf(a), this.endOf(this.i)]; return node; } + /** Read one more level of nesting; past MAX_NESTING the sentence is not read, with one too-deep error. */ + nested(read) { + if (this.rd.depth >= MAX_NESTING) { + const t = this.tok() || this.toks[this.toks.length - 1]; + this.rd.diag('error', this.line, t?.col, t?.end, `This is nested more than ${MAX_NESTING} levels deep, which is more than this tool reads. Give the inner parts names of their own, in steps before this one.`, { code: 'too-deep' }); + throw new TooDeep(); + } + this.rd.depth++; + try { return read(); } finally { this.rd.depth--; } + } + /** Note that a name was used as a text or a list, for inputs no example has typed yet. */ + hint(x, family) { if (x && x.kind === 'Name' && x.entry) (x.entry.hints || (x.entry.hints = new Set())).add(family); return x; } + + /* ---------- names ---------- */ + + /** The longest known name at k, with an optional "the" before it. */ + nameAt(k = this.i) { + const all = this.ctx.scope.all(); + let best = null; + for (const lead of [0, 1]) { + if (lead === 1 && !['the', 'its', 'this'].includes(this.toks[k]?.lc)) continue; + const s = k + lead; + for (const [key, entry] of all) { + const ws = key.split(' '); + let ok = true; + for (let j = 0; j < ws.length; j++) { const t = this.toks[s + j]; if (!t || t.t !== 'w' || t.lc !== ws[j]) { ok = false; break; } } + if (ok && (!best || lead + ws.length > best.n)) best = { entry, n: lead + ws.length }; + } + // a routine input seen from inside a definition + if (!best && this.toks[s]?.t === 'w' && this.ctx.scope.canImplicit(this.toks[s].lc)) { + const e = this.ctx.scope.implicit(this.toks[s].lc); + if (e) best = { entry: e, n: lead + 1 }; + } + } + return best; + } + + /** Words naming a new variable, up to one of the stop words. */ + newNameWords(stops) { + const a = this.i; + const ws = []; + while (!this.atEnd() && !stops.includes(this.word())) { + const t = this.tok(); + if (t.t !== 'w') this.fail(t, `A name is made of words; "${t.t === 's' ? `${t.q}${t.v}${t.q}` : t.v}" cannot be part of one.`); + ws.push(t); + this.i++; + } + while (ws.length > 1 && ARTICLES.has(ws[0].lc)) ws.shift(); + if (!ws.length) this.fail(this.toks[a] || this.tok(), 'A name is missing here.'); + return ws; + } + + /** Find an existing variable by its words, or make one. */ + variable(ws, line, { create = true } = {}) { + const key = ws.map((w) => w.lc).join(' '); + const found = this.ctx.scope.get(key); + if (found) return found; + if (!create) return null; + const entry = { key, words: ws.map((w) => w.lc), display: ws.map((w) => w.v).join(' '), pseudo: this.rd.pseudoName(ws.map((w) => w.v)), kind: 'var', line, type: T.ANY }; + return this.ctx.scope.add(entry); + } + + /* ---------- statements ---------- */ + + statement(it, prevIf) { return this.nested(() => this.statementInner(it, prevIf)); } + statementInner(it, prevIf) { + const a = this.i; + const w = this.word(); + const ctx = this.ctx; + const r = ctx.routine; + const children = it.children; + const blockOf = (loopish, loopVar = null) => { + if (!children.length) this.fail(this.toks[this.toks.length - 1], 'The steps this line controls go on the lines under it, indented further.', { code: 'no-block' }); + const inner = { ...ctx, loops: ctx.loops + (loopish ? 1 : 0), scope: new Scope(ctx.scope), loopVars: loopVar ? [...(ctx.loopVars || []), loopVar] : ctx.loopVars }; + return this.rd.block(children, inner); + }; + const noChildren = () => { if (children.length) this.rd.diag('error', children[0].line, children[0].toks[0]?.col, null, 'These lines are indented under a step that does not control other steps. Only lines ending in a colon (If, For each, Repeat, Otherwise) have steps under them.', { code: 'stray-indent' }); }; + + // If ... / Otherwise ... + if (w === 'if') { + this.i++; + const cond = this.test(); + const inline = this.inlineAfter(); + const body = inline ? [inline] : blockOf(false); + if (inline) noChildren(); + return { kind: 'If', branches: [{ cond, body, line: this.line }], elseBody: null }; + } + if (w === 'otherwise' || w === 'else' || (w === 'if' && false)) { + this.i++; + this.take(','); + if (!prevIf) this.fail(this.toks[a], '"Otherwise" has to follow an "If" step at the same indentation.', { code: 'otherwise' }); + if (prevIf.elseBody) this.fail(this.toks[a], 'This "If" already has its "Otherwise". To test another condition first, write "Otherwise, if ..." before the last "Otherwise".', { code: 'otherwise' }); + if (this.take('if')) { + const cond = this.test(); + const inline = this.inlineAfter(); + const body = inline ? [inline] : blockOf(false); + prevIf.branches.push({ cond, body, line: this.line }); + return { kind: 'Otherwise', attachTo: prevIf }; + } + this.take('then'); + this.take(','); + if (this.atEnd() || this.match(':') >= 0) { prevIf.elseBody = blockOf(false); prevIf.elseLine = this.line; return { kind: 'Otherwise', attachTo: prevIf }; } + const inner = this.statement({ children: [] }, null); + inner.line = this.line; + noChildren(); + prevIf.elseBody = [inner]; prevIf.elseLine = this.line; + return { kind: 'Otherwise', attachTo: prevIf }; + } + + // loops + if (w === 'for' && this.match('for each|every|all') >= 0) { const loop = this.forLoop(); loop.body = loop.inline ? [loop.inline] : blockOf(true, loop.var); if (loop.inline) noChildren(); return loop; } + if (w === 'for' && this.match('for example') < 0) { this.i++; const loop = this.forLoop(true); loop.body = loop.inline ? [loop.inline] : blockOf(true, loop.var); if (loop.inline) noChildren(); return loop; } + if (this.take('repeat|keep|do')) { + this.take('going|doing|repeating|on'); + this.take(['the following', 'these steps', 'this', 'steps']); + if (this.take(['while', 'as long as'])) { const cond = this.test(); this.take(':'); return { kind: 'While', cond, body: blockOf(true) }; } + if (this.take('until')) { const cond = this.test(); this.take(':'); return { kind: 'While', cond: N.un('NOT', cond), until: cond, body: blockOf(true) }; } + if (this.match('for each|every|all') >= 0) { const loop = this.forLoop(); loop.body = loop.inline ? [loop.inline] : blockOf(true, loop.var); if (loop.inline) noChildren(); return loop; } + const times = ['times', 'time'].includes(this.word(1)) ? this.tryParse(() => { const n = this.unary(); this.expect('times|time', '"times"'); return n; }) : null; + if (times) { const v = this.rd.freshName('pass'); return { kind: 'For', var: { pseudo: v, display: 'pass', key: '#pass', type: T.INT, kind: 'loop' }, lo: N.num(0), hi: times, step: 1, body: blockOf(true), times: true }; } + this.fail(this.tok() || this.toks[a], 'Say how long to repeat: "Repeat while ...:", "Repeat until ...:", or "Repeat 3 times:".', { code: 'repeat' }); + } + if (w === 'while' || (w === 'as' && this.match('as long as') >= 0)) { this.i += w === 'while' ? 1 : 3; const cond = this.test(); this.take(':'); return { kind: 'While', cond, body: blockOf(true) }; } + if (w === 'until') { this.i++; const cond = this.test(); this.take(':'); return { kind: 'While', cond: N.un('NOT', cond), until: cond, body: blockOf(true) }; } + + noChildren(); + + // answers and stopping + if (this.take(['the answer is', 'the result is', 'the answer will be', 'the answer becomes'])) return this.answer(a, { early: false }); + if (this.take(['stop and answer', 'stop and give', 'stop and return', 'stop with the answer', 'stop with', 'return', 'answer', 'give back', 'hand back', 'stop and report'])) { this.take('back?'); return this.answer(a, { early: true }); } + if (this.take('report')) { + this.rd.diag('error', this.line, this.colOf(a), this.endOf(this.i), '"Report" could mean two things: hand the value back as the answer, or show it on the screen. Say which.', { + code: 'report', question: { + prompt: 'Which does "report" mean here?', options: [ + { label: 'Hand it back as the answer', replace: { col: this.colOf(a), endCol: this.endOf(a + 1), text: 'Stop and answer' } }, + { label: 'Show it on the screen', replace: { col: this.colOf(a), endCol: this.endOf(a + 1), text: 'Show' } }, + ] + }, + }); + throw STOP; + } + if (this.take('stop|break|leave|exit|quit')) { + const loopWords = this.take(['the loop', 'this loop', 'looking', 'searching', 'scanning', 'repeating', 'going', 'the search', 'the scan', 'early', 'now', 'here', 'there']); + this.take(['the loop', 'now']); + if (!this.atEnd()) this.fail(this.tok(), `After "stop", say what stops: "Stop the loop." or "Stop and answer ${''}."`); + if (ctx.loops > 0) { + if (!loopWords) { + const col = this.colOf(a); const endCol = this.endOf(this.i); + this.rd.diag('error', this.line, col, endCol, '"Stop" could stop only the loop, so the steps after it still run, or stop the whole routine with its answer.', { + code: 'stop', question: { + prompt: 'What stops?', options: [ + { label: 'Only the loop: "Stop the loop."', replace: { col, endCol, text: 'Stop the loop' } }, + { label: 'The whole routine: "Stop and answer ..."', replace: { col, endCol, text: 'Stop and answer ' } }, + ] + }, + }); + throw STOP; + } + return { kind: 'Break' }; + } + if (r && !r.isFunctionHint) return { kind: 'Return', value: null }; + this.fail(this.toks[a], 'There is no loop here to stop.'); + } + if (this.take(['skip to', 'move on to', 'go on to', 'continue with', 'go to', 'jump to'])) { + this.take('the next'); + while (!this.atEnd()) this.i++; + if (ctx.loops === 0) this.fail(this.toks[a], 'There is no loop here, so there is no next pass to move on to.'); + return { kind: 'Continue' }; + } + if (this.take(['continue', 'skip it', 'skip this one', 'skip this', 'skip'])) { this.end(); if (ctx.loops === 0) this.fail(this.toks[a], 'There is no loop here to continue.'); return { kind: 'Continue' }; } + + // making and changing values + if (this.take('let')) { + const ws = this.newNameWords(['be', '=', 'equal', 'become', '<-']); + this.expect('be|equal|become|=|<-', '"be"'); + this.take(['to', 'equal to']); + return this.assignNew(ws, a); + } + if (this.take('set|change|update|reset')) { + const stopAt = this.findWord('to', this.i); + if (stopAt < 0) this.fail(this.tok() || this.toks[a], `Say what it becomes: "Set Total to 0."`); + // "Set" may give a new name its first value; "Change" and "Update" need one already there + const target = this.target(this.i, stopAt, { create: this.toks[a].lc === 'set' }); + this.i = stopAt + 1; + return this.assignTarget(target, a); + } + if (this.take('start|begin|initialize|initialise')) { + this.take(['with', 'by setting', 'by making', 'off with']); + if (this.match('an|a? empty') >= 0 || this.match('a|an? new') >= 0) return this.emptyThing(a); + const ws = this.newNameWords(['at', 'as', 'of', '=', 'being', 'set', 'equal', 'from', 'on']); + this.expect('at|as|of|=|being|set|equal|from|on', '"at"'); + this.take(['to', 'equal to']); + if (this.match('an|a? empty') >= 0) { const e = this.emptyKind(); return this.declareEmpty(ws, e, a); } + return this.assignNew(ws, a); + } + if (this.take('make|create')) { + if (this.match('an|a? empty') >= 0 || this.match('a|an? new') >= 0) return this.emptyThing(a); + const ws = this.newNameWords(['an', 'a', 'equal', 'be', 'into']); + if (this.match('an|a? empty') >= 0) { const e = this.emptyKind(); return this.declareEmpty(ws, e, a); } + this.take(['equal to', 'be', 'into']); + return this.assignNew(ws, a); + } + if (this.take('add|append|put|insert|include|push')) return this.addStatement(a, w); + if (this.take(['subtract', 'take away', 'take'])) { + const from = this.findWord('from', this.i); + if (from < 0) this.fail(this.tok() || this.toks[a], 'Say what to subtract it from: "Subtract 1 from Count."'); + const amount = this.sub(this.i, from, (z) => z.sum()); + this.i = from + 1; + const target = this.target(this.i, this.toks.length); this.i = this.toks.length; + return this.changeBy(target, '-', amount, a); + } + if (this.take('increase|increment|raise|decrease|decrement|lower|reduce|multiply|double|halve')) { + const verb = this.toks[a].lc; + if (verb === 'double' || verb === 'halve') { const target = this.target(this.i, this.toks.length); this.i = this.toks.length; return this.changeBy(target, verb === 'double' ? '*' : '/', N.num(2), a); } + const by = this.findWord('by', this.i); + if (by < 0 && (verb === 'increment' || verb === 'increase')) { const target = this.target(this.i, this.toks.length); this.i = this.toks.length; this.rd.diag('note', this.line, this.colOf(a), this.endOf(this.toks.length), `Reading "${verb}" with no amount as "${verb} by 1".`, { code: 'by-one' }); return this.changeBy(target, '+', N.num(1), a); } + if (by < 0) this.fail(this.tok() || this.toks[a], `Say by how much: "${verb[0].toUpperCase()}${verb.slice(1)} Count by 1."`); + const target = this.target(this.i, by); + this.i = by + 1; + const amount = this.sum(); + this.end(); + const op = ['increase', 'increment', 'raise'].includes(verb) ? '+' : verb === 'multiply' ? '*' : '-'; + return this.changeBy(target, op, amount, a); + } + if (this.match('count|tally|record|note') >= 0 && this.findLastWord('in', this.i + 1) > 0 && this.word(1) !== 'is') { + this.i++; + const inAt = this.findLastWord('in', this.i); + const keyStart = this.i; + this.take(['one more', 'another']); + const key = this.sub(this.i, inAt, (z) => z.sum()); + this.i = inAt + 1; + const target = this.target(this.i, this.toks.length); this.i = this.toks.length; + if (target.kind !== 'Name') this.fail(this.toks[inAt + 1], 'A tally is counted by name: "Count the word in Tally."'); + void keyStart; + return { kind: 'Tally', map: target, key }; + } + if (this.take('remove|delete|drop')) { + const from = this.findLastWord('from', this.i); + if (from < 0) this.fail(this.tok() || this.toks[a], 'Say what to remove it from: "Remove the last item from L."'); + const target = this.target(from + 1, this.toks.length); + let what; + const b = this.i; + if (this.take(['the? last item', 'the? last value', 'the? last letter', 'the? last entry'])) what = { last: true }; + else if (this.take(['the? first item', 'the? first value', 'the? first entry'])) what = { first: true }; + else if (this.take(['item', 'value', 'entry', 'position', 'the item at', 'the value at', 'the item at position'])) what = { at: this.sub(this.i, from, (z) => z.sum()) }; + else what = { value: this.sub(this.i, from, (z) => z.sum()) }; + void b; + this.i = this.toks.length; + return { kind: 'Remove', target, what }; + } + if (this.take('sort')) { const target = this.target(this.i, this.toks.length); this.i = this.toks.length; return { kind: 'Sort', target }; } + if (this.take('swap|exchange')) { + const and = this.findWord('and', this.i); + if (and < 0) this.fail(this.tok() || this.toks[a], 'Say what to swap: "Swap X and Y."'); + const x = this.target(this.i, and); const y = this.target(and + 1, this.toks.length); this.i = this.toks.length; + return { kind: 'Swap', a: x, b: y }; + } + if (this.take('show|print|display|output|write|say')) { + const values = []; + do { values.push(this.value()); } while (this.take([',', 'then', 'and then', 'followed by'])); + this.end(); + return { kind: 'Output', values }; + } + if (this.take(['check that', 'make sure that', 'make sure', 'check', 'assert'])) { + const cond = this.test(); this.end(); + return { kind: 'Assert', cond }; + } + + // Count += 1 and Count++ + const pe = this.toks.findIndex((x, k) => x.t === 'y' && ['+', '-'].includes(x.v) && this.toks[k + 1] && (this.toks[k + 1].v === '=' || this.toks[k + 1].v === x.v) && this.toks[k + 1].col === x.end); + if (pe > this.i) { + const name = this.text(this.i, pe); const amount = this.text(pe + 2, this.toks.length) || '1'; + const plus = this.toks[pe].v === '+'; + const text = plus ? `Add ${amount} to ${name}` : `Subtract ${amount} from ${name}`; + this.rd.diag('error', this.line, this.colOf(a), this.endOf(this.toks.length), `"${this.text(a, this.toks.length)}" is written the way a programming language writes it. In plain English: "${text}."`, { code: 'code-syntax', question: { prompt: 'Write it in English?', options: [{ label: text, replace: { col: this.colOf(a), endCol: this.endOf(this.toks.length), text } }] } }); + throw STOP; + } + const whenAt = this.findWord('when', this.i + 1); + if (!STEP_VERBS.includes(w) && whenAt > 0 && !this.toks.slice(this.i, whenAt).some((x) => ['is', 'are', '=', 'becomes'].includes(x.t === 'w' ? x.lc : x.v))) { + this.rd.diag('error', this.line, this.colOf(a), this.endOf(this.toks.length), `"${this.text(a, this.toks.length)}" reads like a definition. A definition is not a step: it goes on a line of its own at the left edge, below the routine, not indented under it.`, { code: 'def-indented' }); + throw STOP; + } + // a routine used as a step + const call = this.tryParse(() => { const c = this.routineCall(); if (!c) throw STOP; this.end(); return c; }); + if (call) return { kind: 'CallStep', ...call }; + + // "X becomes E", "X = E", "X <- E", "X goes up by E", and the ambiguous "X is E" + const nm = this.nameAt(); + if (nm || this.findWord(['becomes', 'is'], this.i) > 0 || this.findSym(['=', '<-'], this.i) > 0) { + const stop = ['becomes', '=', '<-', 'is', 'goes', 'starts', 'begins']; + let k = this.i; + while (k < this.toks.length && !stop.includes(this.toks[k].t === 'w' ? this.toks[k].lc : this.toks[k].v)) k++; + const verb = this.toks[k]?.t === 'w' ? this.toks[k].lc : this.toks[k]?.v; + if (verb === 'is' && k > this.i) { + const name = sentenceText(this.toks.slice(a, k)); + const valueText = this.rest(k + 1); + this.rd.diag('error', this.line, this.colOf(a), this.endOf(this.toks.length), `"${name} is ${valueText}" could be a step that changes ${name}, or a claim that it already holds. As a step, say which.`, { + code: 'is-step', question: { + prompt: `Does this step change ${name}, or check it?`, options: [ + { label: `Change it: "Set ${name} to ${valueText}."`, replace: { col: this.colOf(a), endCol: this.endOf(this.toks.length), text: `Set ${name} to ${valueText}` } }, + { label: `Check it: "Check that ${name} is ${valueText}."`, replace: { col: this.colOf(a), endCol: this.endOf(this.toks.length), text: `Check that ${name} is ${valueText}` } }, + ] + }, + }); + throw STOP; + } + if (verb && k > this.i) { + const target = this.target(this.i, k, { create: ['becomes', '=', '<-', 'starts', 'begins'].includes(verb) }); + this.i = k + 1; + if (verb === 'goes') { + const up = this.take('up'); const down = !up && this.take('down'); + if (!up && !down) this.fail(this.tok(), 'Say "goes up by" or "goes down by".'); + this.expect('by', '"by"'); + const amount = this.sum(); this.end(); + return this.changeBy(target, up ? '+' : '-', amount, a); + } + if (verb === 'starts' || verb === 'begins') { this.take('at|as|with|from'); return this.assignTarget(target, a); } + if (verb === '=' || verb === '<-') { + this.rd.diag('note', this.line, this.toks[k].col, this.toks[k].end, `Reading "${verb}" as "becomes". In plain English: "Set ${sentenceText(this.toks.slice(a, k))} to ...".`, { code: 'symbol-assign' }); + return this.assignTarget(target, a); + } + if (verb === 'becomes') return this.assignTarget(target, a); + // "X is E" as a step: a statement, or a test? + const valueText = this.rest(); + const name = sentenceText(this.toks.slice(a, k)); + this.rd.diag('error', this.line, this.colOf(a), this.endOf(this.toks.length), `"${name} is ${valueText}" could be a step that changes ${name}, or a claim that it already holds. As a step, say which.`, { + code: 'is-step', question: { + prompt: `Does this step change ${name}, or check it?`, options: [ + { label: `Change it: "Set ${name} to ${valueText}."`, replace: { col: this.colOf(a), endCol: this.endOf(this.toks.length), text: `Set ${name} to ${valueText}` } }, + { label: `Check it: "Check that ${name} is ${valueText}."`, replace: { col: this.colOf(a), endCol: this.endOf(this.toks.length), text: `Check that ${name} is ${valueText}` } }, + ] + }, + }); + throw STOP; + } + } + + this.unknownStatement(a); + return null; + } + + /** After "If C": an inline step ("If C, add 1 to X.") or nothing (a block follows). */ + inlineAfter() { + this.take(','); + this.take('then'); + this.take(','); + if (this.word() === ':' && this.i < this.toks.length - 1) this.i++; + if (this.atEnd() || (this.word() === ':' && this.i === this.toks.length - 1)) { this.take(':'); return null; } + const rest = new Sentence(this.toks.slice(this.i), this.rd, this.ctx, this.line); + this.i = this.toks.length; + const st = rest.statement({ children: [] }, null); + if (st) st.line = this.line; + return st; + } + + answer(a, { early }) { + const r = this.ctx.routine; + if (!r) this.fail(this.toks[a], 'An answer belongs inside a routine.'); + const value = this.value(); + this.end(); + r.answers++; + if (!early && this.ctx.loops > 0) { + this.rd.diag('error', this.line, this.colOf(a), this.endOf(this.toks.length), `Inside a loop, "the answer is ${sentenceText(this.toks.slice(this.toks.findIndex((x) => x.lc === 'is') + 1))}" could mean stop now with that answer, or remember it and keep looking.`, { + code: 'answer-in-loop', question: { + prompt: 'Stop right away, or remember it and keep going?', options: [ + { label: 'Stop right away with this answer', replace: { col: this.colOf(a), endCol: this.endOf(this.toks.findIndex((x) => x.lc === 'is') + 1), text: 'Stop and answer' } }, + { label: 'Remember it as the best so far, and keep going', replace: { col: this.colOf(a), endCol: this.endOf(this.toks.findIndex((x) => x.lc === 'is') + 1), text: 'Set the best so far to' } }, + ] + }, + }); + throw STOP; + } + return { kind: 'Return', value }; + } + + /** "an empty list called Starts" after Start with / Make. */ + emptyThing(a) { + const e = this.emptyKind(); + if (!this.take('called|named|as|,')) { + if (this.atEnd()) this.fail(this.toks[a], `Give it a name: "Start with ${e.words} called Starts."`, { code: 'unnamed' }); + } + this.take('called|named'); + const ws = this.newNameWords([]); + return this.declareEmpty(ws, e, a); + } + emptyKind() { + this.take('an|a'); + this.take('empty|new'); + this.take('empty'); + const b = this.i; + const w = this.word(); + let type; + if (this.take(['list', 'array', 'sequence of values'])) type = T.list(this.ofKind()); + else if (this.take('set|collection')) type = T.set(this.ofKind()); + else if (this.take(['tally', 'table', 'count table', 'dictionary', 'map', 'counter', 'record of counts', 'count'])) { const k = this.ofKind(); type = T.map(k, T.INT); } + else if (this.take(['text', 'string', 'sequence', 'word', 'strand', 'piece of text'])) type = T.STR; + else this.fail(this.tok(), `"${w || 'this'}" is not something that starts empty. Say "an empty list", "an empty set", "an empty tally", or "empty text".`, { code: 'empty-kind' }); + return { type, words: this.text(b - 2 < 0 ? 0 : b - 2, this.i) }; + } + ofKind() { + if (!this.take('of|for')) return T.ANY; + const ws = []; + while (!this.atEnd() && !['called', 'named', ','].includes(this.word())) { ws.push(this.tok().v); this.i++; } + return kindFromWords(ws.map(singular)) || T.ANY; + } + declareEmpty(ws, e, a) { + const v = this.variable(ws, this.line); + void a; + return { kind: 'Empty', target: N.name(v), type: e.type }; + } + + assignNew(ws, a) { + const v = this.variable(ws, this.line); + return this.finishAssign(N.name(v), a); + } + assignTarget(target, a) { return this.finishAssign(target, a); } + finishAssign(target, a) { + if (this.match('an|a? empty') >= 0) { + if (target.kind !== 'Name') this.fail(this.toks[a], 'Only a name can start empty.'); + const e = this.emptyKind(); this.end(); + return { kind: 'Empty', target, type: e.type }; + } + const value = this.value(); + this.end(); + return { kind: 'Assign', target, value }; + } + + /** Add E to X / Put E at the end of X / Append E to X. */ + addStatement(a, verb) { + let to = this.findLastWord(verb === 'put' || verb === 'insert' ? ['at', 'in', 'into', 'on'] : ['to', 'onto', 'into', 'in'], this.i); + if (to < 0) this.fail(this.tok() || this.toks[a], verb === 'put' ? 'Say where it goes: "Put i at the end of Starts."' : `Say where it goes: "${verb[0].toUpperCase()}${verb.slice(1)} 1 to Total."`); + // "at the end of", "to the end of", "to the front of" + let place = 'plain'; + let k = to + 1; + const after = new Sentence(this.toks, this.rd, this.ctx, this.line); after.i = k; + if (after.take('the? end|back|bottom of')) place = 'end'; + else if (after.take('the? front|start|beginning|head|top of')) place = 'front'; + else if (after.take('position')) { + const at = this.findWord('of', after.i); + if (at < 0) this.fail(this.toks[after.i], 'Say where: "Insert X at position 2 of L."'); + const pos = this.sub(after.i, at, (z) => z.sum()); + const target = this.target(at + 1, this.toks.length); + const value = this.sub(this.i, to, (z) => z.value()); + this.i = this.toks.length; + return { kind: 'Insert', target, pos, value }; + } + const target = this.target(after.i, this.toks.length); + const value = this.sub(this.i, to, (z) => z.value()); + this.i = this.toks.length; + if (target.kind === 'MapEntry') return { kind: 'AddTo', target, value, place, verb }; + return { kind: 'AddTo', target, value, place, verb, toCol: this.toks[to].col, endCol: lastEnd(this.toks), targetText: sentenceText(this.toks.slice(to + 1)) }; + } + + changeBy(target, op, amount, a) { + void a; + return { kind: 'Change', target, op, amount }; + } + + /** The thing a step changes: a name, an item of a list, or a tally's entry. */ + target(from, to, { create = false } = {}) { + const s = new Sentence(this.toks.slice(from, to), this.rd, this.ctx, this.line); + if (!s.toks.length) this.fail(this.toks[from - 1] || this.toks[0], 'Say what changes here.'); + // the count for K in X / X's count for K / the entry for K in X + if (s.match('the? count|entry|value|tally|number for|of') >= 0 && s.findLastWord('in', 0) > 0) { + s.take('the? count|entry|value|tally|number for|of'); + const inAt = s.findLastWord('in', s.i); + const key = s.sub(s.i, inAt, (z) => z.sum()); + s.i = inAt + 1; + const map = s.primary(); + s.end(); + return { kind: 'MapEntry', map, key }; + } + // item i of L, the letter of R at i: something with a position + const nm = s.nameAt(); + if (nm && nm.n === s.toks.length) return N.name(nm.entry); + if (!nm) { + const ph = s.tryParse(() => { const e = s.primary(); s.end(); return e; }); + if (ph && ph.kind === 'Index') return ph; + if (ph && ph.kind === 'Name') return ph; + if (create && s.toks.every((t) => t.t === 'w')) { + const ws = s.toks.slice(); + while (ws.length > 1 && ARTICLES.has(ws[0].lc)) ws.shift(); + return N.name(this.variable(ws, this.line)); + } + const words = sentenceText(s.toks); + if (s.toks.every((t) => t.t === 'w')) { + const ws = s.toks.slice(); + while (ws.length > 1 && ARTICLES.has(ws[0].lc)) ws.shift(); + const known = [...this.ctx.scope.all().values()].filter((e) => e.kind !== 'bound').map((e) => e.display); + const guess = closest(words, known); + if (guess) { + this.rd.diag('error', this.line, this.colOf(from), this.endOf(to), `Nothing is called "${words}". Did you mean ${guess}?`, { code: 'misspelled', question: { prompt: `Use ${guess}?`, options: [{ label: `Replace with ${guess}`, replace: { col: this.colOf(from), endCol: this.endOf(to), text: guess } }] } }); + throw STOP; + } + const v = this.variable(ws, this.line); + v.reported = true; + this.rd.diag('error', this.line, this.colOf(from), this.endOf(to), `${words} is changed here before any step gives it a value. Set it up first, before the loop: "Start with ${words.replace(/^the /i, '')} at 0."`, { code: 'used-before-set', setup: words, fix: null }); + return N.name(v); + } + this.failSpan(from, to, `Nothing is called "${words}" yet. Give it a value first, before this step: "Start with ${words.replace(/^the /i, '')} at 0."`, { code: 'unknown-name' }); + } + const e = s.tryParse(() => { const x = s.primary(); s.end(); return x; }); + if (e) return e; + this.failSpan(from, to, `"${sentenceText(s.toks)}" is not something a step can change. Name a variable, or an item of a list: "item i of L".`); + return null; + } + + routineCall() { + for (const r of this.heads('routines')(this.word())) { + const got = this.tryParse(() => this.templateAt(r.head.elems, r, true)); + if (got) return { routine: r, args: got }; + } + return null; + } + + unknownStatement(a) { + const text = this.rest(a); + const lower = text.toLowerCase(); + const hints = [ + [/^(go|walk|run|move|step|loop|iterate|scan)\s+(through|over|along|across)\b|^(look at|visit)\s+(each|every|all)\b/, '"Go through" does not say which positions, in what order, or what to call each one. Say it as a loop: "For each position i of T:" or "For each letter b of T:".'], + [/^(look for|search for|find|locate|seek)\b/, '"Find" says what you want, not how to get it. Split it into steps: a loop over the places it could be, a test at each one, and what to do when the test passes. Or name the phrase and define it on its own line: "To find ... :".'], + [/^(check|test|see|determine|decide|figure out|work out)\b( if| whether)?/, '"Check" needs a condition and an action. Write it as "If , ." and say what happens when the condition holds.'], + [/^(compare)\b/, '"Compare" does not say what is compared with what, or what happens after. Write the comparison as a condition: "If the letter of P at j is not the letter of T at i + j, ...".'], + [/^(keep track of|remember|store|save|track)\b/, 'Give the thing a name and a starting value before the loop: "Start with Count at 0.", then change it inside the loop: "Add 1 to Count."'], + [/^(process|handle|deal with|analy[sz]e|clean|fix|filter|update|go on)\b/, `"${text.split(' ')[0]}" hides the real steps. Say exactly what happens to each value: which test, and which change.`], + [/^(then|next|after that|afterwards|finally|first|second)\b/, `Leave out "${text.split(' ')[0]}": the steps run in the order they are written. Start the sentence with what it does.`], + [/^(return|output|give)\b/, 'Say "Stop and answer X." to hand back an answer, or "Show X." to put it on the screen.'], + [/^(increment|decrement)\b/, 'Say "Increase Count by 1." or "Add 1 to Count."'], + [/^(declare|dim|var|int|string)\b/, 'Nothing needs declaring here. Give a name its first value: "Start with Count at 0."'], + [/^(else|elif|else if)\b/, 'Write "Otherwise:" or "Otherwise, if ...:".'], + [/^(end|endif|endfor|next|done)\b/, 'Steps under a line end where the indentation ends; no closing word is needed.'], + ]; + for (const [re, msg] of hints) if (re.test(lower)) this.failSpan(a, this.toks.length, `The tool cannot run "${text}" yet. ${msg}`, { code: 'vague-step' }); + const first = this.toks[a]?.t === 'w' ? this.toks[a].lc : ''; + const near = first && closest(first, STEP_VERBS.filter((v) => v.length > 2)); + if (near) this.failSpan(a, this.toks.length, `The tool cannot read "${text}". Did you mean "${near[0].toUpperCase()}${near.slice(1)}" for "${this.toks[a].v}"?`, { code: 'misspelled' }); + this.failSpan(a, this.toks.length, `The tool cannot read "${text}" as a step. ${hintFor(text)}`, { code: 'unknown-step' }); + } + + /* ---------- loops ---------- */ + + forLoop(noEach = false) { + const a = this.i; + if (!noEach) { this.take('for'); this.take('each|every|all'); } + // every third position i of T + let step = null; + if (ORDINALS[this.word()] && this.word() !== 'first' && ['position', 'positions', 'index', 'place', 'letter', 'item', 'start', 'starting', 'base', 'value'].includes(this.word(1))) { + if (!['position', 'positions', 'index', 'place', 'start', 'starting'].includes(this.word(1))) this.fail(this.tok(), `"Every ${this.word()} ${this.word(1)}" leaves open where the counting starts. Say it with positions: "For every ${this.word()} position i of T:", and read the letter there as "the letter of T at i".`, { code: 'ordinal-letters' }); + step = ORDINALS[this.word()]; this.i++; + } + const kindWord = this.word(); + let mode = null; + if (POSITION_WORDS.includes(kindWord)) mode = 'positions'; + else if (LETTER_WORDS.includes(kindWord)) mode = 'letters'; + else if (ITEM_WORDS.includes(kindWord)) mode = 'items'; + else if (['key', 'keys'].includes(kindWord)) mode = 'keys'; + else if (['member', 'members'].includes(kindWord)) mode = 'members'; + else if (['codon', 'codons'].includes(kindWord)) mode = 'codons'; + if (mode) this.i++; + // the variable's name: one word, unless the next word begins the range + let varTok = null; + const w = this.word(); + if (this.tok()?.t === 'w' && !['of', 'in', 'from', 'where', 'with', 'that', 'starting', 'going', 'counting', 'on', 'across', 'along', 'within', 'between', 'up', 'down'].includes(w)) { varTok = this.tok(); this.i++; } + if (varTok && !mode && closest(varTok.lc, ['each', 'every']) && !['from', 'of', 'in'].includes(this.word(-1))) this.fail(varTok, `Did you mean "For ${closest(varTok.lc, ['each', 'every'])}"?`, { code: 'misspelled' }); + if (!varTok && !mode) this.fail(this.tok() || this.toks[a], 'Name what the loop visits: "For each position i of T:" or "For each letter b of T:".', { code: 'loop-name' }); + const autoName = !varTok; + const display = varTok ? varTok.v : singular(kindWord); + if (mode === 'codons') { + this.fail(this.toks[a], '"Each codon" leaves open where the first codon starts and what happens to leftover letters at the end. Say it with positions: "For every third position i of T, from 0 up to but not including the length of T minus 2:", and read the codon as "the 3 letters of T starting at i".', { code: 'codon-loop' }); + } + const loopScope = this.ctx.scope; + const makeVar = (type) => { + const key = display.toLowerCase(); + const still = (this.ctx.loopVars || []).find((v) => v.key === key); + if (still) this.fail(varTok || this.toks[a], `${display} already counts the loop on line ${still.line}, which is still running here. Give this loop a name of its own: "For each position k of ...".`, { code: 'loop-reuse' }); + const existing = loopScope.get(key); + if (existing && existing.kind !== 'loop') this.fail(varTok || this.toks[a], `${display} already names something else (line ${existing.line}). A loop needs a fresh name for what it visits.`, { code: 'loop-shadow' }); + const entry = { key, words: [key], display, pseudo: existing?.pseudo || this.rd.pseudoName([display]), kind: 'loop', line: this.line, type }; + loopScope.add(entry); + return entry; + }; + if (autoName) this.rd.diag('note', this.line, this.colOf(a), this.endOf(this.i), `Calling what this loop visits "the ${display}". A name of your own ("For each ${kindWord} ${display[0]} of ...") reads more precisely, especially with two loops.`, { code: 'auto-name' }); + + let loop; + if (mode === 'letters' || mode === 'items' || mode === 'keys' || mode === 'members' || (mode === null && this.match('of|in') >= 0)) { + if (!this.take('of|in|from|across|along|within')) this.fail(this.tok() || this.toks[a], `Say what it walks through: "For each ${kindWord || 'letter'} ${display} of T:".`); + const b = this.i; + const coll = this.primary(); + this.mark(coll, b); + this.hint(coll, mode === 'letters' ? 'text' : mode === 'items' ? 'list' : 'sized'); + if (mode === 'positions') loop = { kind: 'For', lo: N.num(0), hi: N.len(coll), step: 1 }; + else loop = { kind: 'ForEach', coll, mode }; + } else { + // positions: of T, or from A up to ... + const range = this.range(a); + loop = { kind: 'For', lo: range.lo, hi: range.hi, step: range.step, down: range.down }; + } + if (loop.kind === 'For' && step) { if (loop.step !== 1) this.fail(this.toks[a], 'Say the step once.'); loop.step = step; } + if (loop.kind === 'For') { + const extra = this.stepPhrase(); + if (extra) loop.step = loop.down ? -Math.abs(extra) : extra; + loop.var = makeVar(T.INT); + } else { + loop.var = makeVar(T.ANY); + if (loop.mode === 'positions') loop.var.type = T.INT; + } + // for each position i of T where C: + if (this.take(['where', 'at which', 'such that', 'with', 'for which', 'that', 'if', 'whenever'])) { + loop.filter = this.test(); + } + // For each letter b of T, if b is 'A', add 1 to Count. + if (this.word() === ',' && this.tok(1)?.t === 'w' && !this.atEnd()) { + this.i++; + const inner = { ...this.ctx, loops: this.ctx.loops + 1, scope: new Scope(this.ctx.scope) }; + const rest = new Sentence(this.toks.slice(this.i), this.rd, inner, this.line); + this.i = this.toks.length; + const st = rest.statement({ children: [] }, null); + if (st) { st.line = this.line; loop.inline = st; } + loop.autoName = autoName; + return loop; + } + this.take(','); + this.take(':'); + if (!this.atEnd()) this.fail(this.tok(), `The loop's line should end here, before "${this.rest()}". ${/fit|room|end|space|enough/.test(this.rest()) ? 'Say exactly which positions, with a range: "from 0 up to and including the length of T minus the length of P".' : ''}`.trim(), { code: 'loop-extra' }); + loop.autoName = autoName; + return loop; + } + + stepPhrase() { + const k = this.i; + this.take(','); + if (this.take(['counting by', 'counting up by', 'stepping by', 'going up by', 'going by', 'moving by', 'jumping by', 'in steps of', 'by'])) { + const n = this.tok(); + if (n?.t !== 'n' || n.float) this.fail(n || this.toks[k], 'A loop steps by a whole number: "counting by 3".'); + this.i++; + return Number(n.v); + } + this.i = k; + return null; + } + + /** from A up to but not including B, and the rest. */ + range(a) { + const out = { lo: null, hi: null, step: 1, down: false }; + if (this.take('of|in|across|along|within')) { + const b = this.i; const coll = this.hint(this.primary(), 'sized'); this.mark(coll, b); + out.lo = N.num(0); out.hi = N.len(coll); + if (this.take([', starting at', 'starting at', ', beginning at', 'beginning at', ', from', 'from'])) out.lo = this.sum(); + return out; + } + if (this.word() === '[') { + this.i++; out.lo = this.sum(); this.expect(','); out.hi = this.sum(); this.expect(')', '")" (the range is half-open)'); + return out; + } + if (this.word() === 'between') { + const k0 = this.i; this.i++; + const lo = this.sum(); this.expect('and'); const hiB = this.i; const hi = this.sum(); + const col = this.colOf(k0); const endCol = this.endOf(this.i); + const loT = this.text(k0 + 1, hiB - 1); const hiT = this.text(hiB, this.i); + void lo; void hi; + this.rd.diag('error', this.line, col, endCol, `"Between ${loT} and ${hiT}" leaves open whether ${loT} and ${hiT} themselves are included.`, { + code: 'between', question: { + prompt: 'Which values does it take?', options: [ + { label: `From ${loT}, stopping just before ${hiT}`, replace: { col, endCol, text: `from ${loT} up to but not including ${hiT}` } }, + { label: `From ${loT} to ${hiT}, both included`, replace: { col, endCol, text: `from ${loT} up to and including ${hiT}` } }, + ] + }, + }); + throw STOP; + } + if (!this.take(['from', 'starting at', 'starting from', 'beginning at'])) this.fail(this.tok() || this.toks[a], 'Say which values it takes: "from 0 up to but not including N", or "of T" for every position of T.', { code: 'range' }); + out.lo = this.sum(); + const k = this.i; + if (this.take(['up to but not including', 'up to and not including', 'to but not including', 'until but not including', 'below', 'up to but excluding', 'up to excluding'])) { out.hi = this.sum(); return this.checkUp(out, k); } + if (this.take(['up to and including', 'up to including', 'to and including', 'through', 'up through', 'up to inclusive'])) { out.hi = add1(this.sum()); return this.checkUp(out, k); } + if (this.take(['down to but not including', 'down to and not including', 'down to excluding'])) { out.hi = this.sum(); out.down = true; out.step = -1; return out; } + if (this.take(['down to and including', 'down through', 'down to including'])) { out.hi = sub1(this.sum()); out.down = true; out.step = -1; return out; } + if (this.take(['up to', 'to', 'until', 'till', 'down to'])) { + const down = this.toks[k].lc === 'down'; + const b = this.i; + const hi = this.sum(); + if (this.take(['inclusive', 'included', ',? including it', ',? including that one'])) { out.hi = down ? sub1(hi) : add1(hi); out.down = down; out.step = down ? -1 : 1; return down ? out : this.checkUp(out, k); } + if (this.take(['exclusive', 'excluded', ',? not including it', ',? excluding it'])) { out.hi = hi; out.down = down; out.step = down ? -1 : 1; return down ? out : this.checkUp(out, k); } + const endText = this.text(b, this.i); + const col = this.colOf(k); const endCol = this.endOf(this.i); + this.rd.diag('error', this.line, col, endCol, `"${this.text(k, this.i)}" leaves open whether ${endText} itself is included. A loop over positions usually stops just before the end: the last position of T is one less than its length.`, { + code: 'range-ends', question: { + prompt: `Is ${endText} itself included?`, options: [ + { label: `No: stop just before ${endText}`, replace: { col, endCol, text: `${down ? 'down' : 'up'} to but not including ${endText}` } }, + { label: `Yes: ${endText} is the last one`, replace: { col, endCol, text: `${down ? 'down' : 'up'} to and including ${endText}` } }, + ] + }, + }); + throw STOP; + } + this.fail(this.tok() || this.toks[a], `Say where it stops: "from ${this.text(a, this.i).replace(/^.*from /i, '')} up to but not including ...".`, { code: 'range' }); + return out; + } + + /** A loop counting up whose start is past its end runs no passes; with numbers, ask whether it meant to count down. */ + checkUp(out, k) { + const lo = out.lo; const hi = out.hi; + if (lo?.kind === 'Num' && hi?.kind === 'Num' && lo.value >= hi.value) { + const col = this.colOf(k); const endCol = this.endOf(this.i); + this.rd.diag('error', this.line, col, endCol, `Counting up from ${lo.value}, the loop never reaches ${hi.value - 1 < lo.value ? hi.value - 1 : hi.value}, so it runs no passes at all. To count down, say "down to".`, { + code: 'range-backwards', question: { + prompt: 'Did you mean to count down?', options: [ + { label: `Count down to ${hi.value - 1}, including it`, replace: { col, endCol, text: `down to and including ${hi.value - 1}` } }, + ] + }, + }); + throw STOP; + } + return out; + } + + /* ---------- expressions ---------- */ + + /** Try a parse; on failure, restore the position and forget its messages. */ + tryParse(fn) { + const k = this.i; const n = this.rd.diags.length; + try { return fn(); } catch (err) { + if (!(err instanceof Stop) || err instanceof TooDeep) throw err; + this.i = k; this.rd.diags.length = n; + return null; + } + } + /** Parse tokens [a, b) on their own, all of them. */ + sub(a, b, fn) { + const s = new Sentence(this.toks.slice(a, b), this.rd, this.ctx, this.line); + const v = fn(s); + if (!s.atEnd()) s.end(); + return v; + } + + findWord(w, from) { const ws = Array.isArray(w) ? w : [w]; for (let k = from; k < this.toks.length; k++) if (this.toks[k].t === 'w' && ws.includes(this.toks[k].lc) && !this.inQuant(k)) return k; return -1; } + findLastWord(w, from) { const ws = Array.isArray(w) ? w : [w]; for (let k = this.toks.length - 1; k >= from; k--) if (this.toks[k].t === 'w' && ws.includes(this.toks[k].lc)) return k; return -1; } + findSym(ss, from) { for (let k = from; k < this.toks.length; k++) if (this.toks[k].t === 'y' && ss.includes(this.toks[k].v)) return k; return -1; } + inQuant() { return false; } + + /** A value: a condition or an amount. */ + value() { return this.condition(); } + + /** + * A condition that has to end the sentence or stop at a comma, "then", or + * a colon. Words left over mean the condition used a phrase the tool does + * not know: the student is asked to define it. + */ + test() { + const a = this.i; + const c = this.condition(); + if (this.atEnd() || [',', ':', 'then'].includes(this.word())) return c; + // the unknown phrase runs from the start of the condition to the next comma or colon + let b = this.i; + while (b < this.toks.length && ![',', ':'].includes(this.toks[b].v) && this.toks[b].lc !== 'then') b++; + // "If b is 'A' add 1 to Count": a step after the condition, with no comma between + for (let v = this.i; v < b; v++) { + if (this.toks[v].t !== 'w' || !STEP_VERBS.includes(this.toks[v].lc) || ['the', 'to', 'for', 'count', 'if'].includes(this.toks[v].lc)) continue; + const probe = new Sentence(this.toks.slice(a, v), this.rd, this.ctx, this.line); + const n = this.rd.diags.length; + let fits = false; + try { probe.condition(); fits = probe.atEnd(); } catch (err) { if (!(err instanceof Stop)) throw err; } + this.rd.diags.length = n; + if (fits) { + const col = this.toks[v - 1].end; + this.rd.diag('error', this.line, this.colOf(a), this.endOf(this.toks.length), `Put a comma between the condition and the step: "${this.text(a, v)}, ${this.text(v, this.toks.length)}".`, { code: 'missing-comma', question: { prompt: 'Add the comma?', options: [{ label: 'Add a comma', replace: { col, endCol: col, text: ',' } }] } }); + throw STOP; + } + } + const phrase = this.text(a, b); + this.unknownPhrase(a, b, phrase, 'predicate'); + return c; + } + + /** Ask the student to define a phrase the tool does not know, and offer to start the definition. */ + unknownPhrase(a, b, phrase, kind) { + const col = this.colOf(a); const endCol = this.endOf(b); + // a phrase that nearly matches one already defined: say which + const words = phrase.toLowerCase().split(/\s+/); + const near = this.rd.defs.find((d) => { const hw = d.head.text.toLowerCase().split(/\s+/); const lits = d.head.elems.filter((e) => e.lit && !ARTICLES.has(e.lit)).map((e) => e.lit); return lits.length && lits.filter((w) => words.includes(w)).length >= Math.max(1, lits.length - 1) && hw.length !== words.length; }); + if (near) { + this.rd.diag('error', this.line, col, endCol, `"${phrase}" is close to "${near.head.text}", defined on line ${near.line}, and not the same. A phrase is used with exactly the words it was defined with, with values in place of its inputs.`, { code: 'near-phrase' }); + throw STOP; + } + const isValue = kind === 'value'; + const clean = phrase.replace(/[.,:;]+$/, ''); + const def = isValue ? `${/^the /i.test(clean) ? clean[0].toUpperCase() + clean.slice(1) : `The ${clean}`} is .` : `${clean} when .`; + this.rd.diag('error', this.line, col, endCol, isValue + ? `The tool does not know "${clean}". If it is a new value, give it a starting value in an earlier step. If it names an idea that needs more words, say what it is on a line of its own, below the routine: "${def}"` + : `The tool does not know what "${clean}" means. Say it exactly, on a line of its own below the routine: "${def}" Every word in that line has to be something the tool already knows, or another phrase you define the same way.`, { + code: 'unknown-phrase', phrase: clean, + question: { prompt: `What does "${clean}" mean?`, options: [{ label: `Define it on a new line: "${def}"`, insert: { text: def } }] }, + }); + throw STOP; + } + + condition() { return this.or(); } + or() { + const a = this.i; + const parts = [[a, null]]; + let l = this.and(); + parts[0][1] = this.i; + let mixed = l.andChain; + while (this.word() === 'or') { + this.i++; + const b = this.i; + const r = this.and(); + parts.push([b, this.i]); + if (r.andChain) mixed = true; + l = N.bin('OR', l, r); + } + // "A or B and C": which comes first is a real choice in English + if (mixed && parts.length > 1) { + const col = this.colOf(a); const endCol = this.endOf(this.i); + const whole = this.text(a, this.i); + const opts = []; + if (parts.length === 2) { + const [[a0, a1], [b0, b1]] = parts; + const left = this.text(a0, a1); const right = this.text(b0, b1); + const ra = /^(.*?) and (.*)$/.exec(right); const la = /^(.*) and (.*?)$/.exec(left); + if (ra) opts.push({ label: `(${left} or ${ra[1]}) and ${ra[2]}`, replace: { col, endCol, text: `(${left} or ${ra[1]}) and ${ra[2]}` } }, { label: `${left} or (${right})`, replace: { col, endCol, text: `${left} or (${right})` } }); + else if (la) opts.push({ label: `${la[1]} and (${la[2]} or ${right})`, replace: { col, endCol, text: `${la[1]} and (${la[2]} or ${right})` } }, { label: `(${left}) or ${right}`, replace: { col, endCol, text: `(${left}) or ${right}` } }); + } + this.rd.diag('error', this.line, col, endCol, `"${whole}" mixes "and" and "or", and English does not say which joins first. Put parentheses around the part that goes together.`, { + code: 'and-or', question: opts.length ? { prompt: 'Which goes together?', options: opts } : undefined, + }); + throw STOP; + } + return l; + } + and() { + let l = this.not(); + let n = 0; + while (this.word() === 'and' && !this.andEndsHere()) { this.i++; const r = this.not(); l = N.bin('AND', l, r); n++; } + if (n) l.andChain = true; + return l; + } + /** "and then" joins steps, not conditions. */ + andEndsHere() { return this.word(1) === 'then'; } + not() { + if (this.take(['it is not true that', 'it is not the case that'])) return N.un('NOT', this.nested(() => this.or())); + const a = this.i; + if (this.take('not')) { + const inner = this.nested(() => this.not()); + if (['and', 'or'].includes(this.word()) && !this.andEndsHere()) { + const b = this.i; const conj = this.word(); + const col = this.colOf(a); const endCol = this.endOf(this.toks.length); + const first = this.text(a + 1, b); + this.rd.diag('error', this.line, col, endCol, `Does "not" apply to "${first}" only, or to everything after it, "${conj}" included?`, { + code: 'not-scope', question: { + prompt: 'What does "not" cover?', options: [ + { label: `Only "${first}"`, replace: { col, endCol: this.endOf(b), text: `(not ${first})` } }, + { label: 'Everything after it', replace: { col, endCol: this.toks[a].end, text: 'it is not true that' } }, + ] + }, + }); + throw STOP; + } + return N.un('NOT', inner); + } + return this.comparison(); + } + + comparison() { + const a = this.i; + // quantifiers + const q = this.nested(() => this.quantifier()); + if (q) return this.mark(q, a); + // defined yes-or-no phrases + for (const d of this.heads('predicate')(this.word())) { + const args = this.tryParse(() => this.templateAt(d.head.elems, d, false)); + if (args) return this.mark(this.defCall(d, args), a); + } + let left; + if (this.ctx.subject && this.isComparisonWord()) left = this.ctx.subject; + else if (this.lastLeft && this.match(['less than', 'more than', 'greater than', 'fewer than', 'at least', 'at most', 'smaller than', 'larger than', 'above', 'below', 'equal to', 'not equal to', 'different from']) >= 0) { + // "N is more than 2 and less than 5": the same subject again + left = this.lastLeft; + const col = this.colOf(this.i); + this.rd.diag('note', this.line, col, null, `Reading this as "${this.lastLeft.text || 'the same value'} is ${this.rest().split(/\s+(and|or)\s+/)[0]}".`, { code: 'same-subject' }); + this.i -= 0; + const saveI = this.i; + this.toks.splice(saveI, 0, { t: 'w', v: 'is', lc: 'is', col, end: col }); + } else left = this.sum(); + if (left !== this.ctx.subject) this.lastLeft = left; + // membership + for (const [p, op] of IN_PHRASES) { + if (this.take(p)) { + const right = this.collectionOrList(); + const e = N.bin('IN', left, right); + return this.mark(op === 'in' ? e : N.un('NOT', e), a); + } + } + if (this.take(['is between', 'are between', 'lies between'])) { + const lo = this.sum(); this.expect('and'); const hi = this.sum(); + const col = this.colOf(a); const endCol = this.endOf(this.i); + const lt = sourceText(left); const ltxt = this.text(a, this.i).replace(/\s+(is|are|lies) between.*$/i, ''); + void lt; + this.rd.diag('error', this.line, col, endCol, `"Between ${this.textOf(lo)} and ${this.textOf(hi)}" leaves open whether the ends count.`, { + code: 'between', question: { + prompt: 'Do the ends count?', options: [ + { label: 'The first counts, the last does not', replace: { col, endCol, text: `${ltxt} is at least ${this.textOf(lo)} and ${ltxt} is less than ${this.textOf(hi)}` } }, + { label: 'Both ends count', replace: { col, endCol, text: `${ltxt} is at least ${this.textOf(lo)} and ${ltxt} is at most ${this.textOf(hi)}` } }, + { label: 'Neither end counts', replace: { col, endCol, text: `${ltxt} is greater than ${this.textOf(lo)} and ${ltxt} is less than ${this.textOf(hi)}` } }, + ] + }, + }); + throw STOP; + } + if (this.take('contains|has|includes')) { + const b = this.i; + const right = this.sum(); + if (left.kind === 'Name' && right.kind === 'Str') this.hint(left, 'text'); + void b; + if (this.ctx.types?.get?.(left)) { /* resolved later */ } + return this.mark({ kind: 'Contains', coll: left, item: right }, a); + } + // X is neither 'G' nor 'C' + if (this.take('is|are neither')) { + const vals = [this.sum()]; + while (this.take('nor') || (this.word() === ',' && this.isBareValueAt(this.i + 1) && this.take(','))) vals.push(this.sum()); + if (vals.length < 2) this.fail(this.tok() || this.toks[a], 'After "neither", say both: "is neither \'G\' nor \'C\'".'); + return this.mark(vals.map((v) => N.bin('!=', left, v)).reduce((x, y) => N.bin('AND', x, y)), a); + } + // X is empty / X is not empty / X has no letters + if (this.take(['is empty', 'are empty', 'has no letters', 'has no items', 'is an empty list', 'is the empty text', 'is empty text', 'has nothing in it'])) { this.hint(left, 'sized'); return this.mark(N.bin('==', N.len(left), N.num(0)), a); } + if (this.take(['is not empty', 'are not empty', 'has some letters', 'has letters', 'has items', 'is nonempty'])) { this.hint(left, 'sized'); return this.mark(N.bin('>', N.len(left), N.num(0)), a); } + // comparisons + for (const [p, op] of COMPARISONS) { + const n = this.match(p); + if (n < 0) continue; + // "is a key of" and friends were handled; "is either" + this.i += n; + this.take('either'); + const right = this.sum(); + let e = N.bin(op, left, right); + if (!['==', '!='].includes(op) && this.word() === 'or' && this.isBareValueAt(this.i + 1)) { + const col = this.colOf(a); const b = this.i + 1; const probe = new Sentence(this.toks, this.rd, this.ctx, this.line); probe.i = b; try { probe.sum(); } catch { /* checked above */ } + const other = this.text(b, probe.i); const endCol = this.endOf(probe.i); + const subj = this.text(a, this.i).replace(new RegExp(`\\s*${p.replace(/ /g, '\\s+')}.*$`, 'i'), ''); + this.rd.diag('error', this.line, col, endCol, `"${this.text(a, probe.i)}" leaves the second comparison half said. Say it in full.`, { + code: 'or-half', question: { prompt: 'Say both comparisons in full:', options: [{ label: `${this.text(a, this.i)} or ${subj} ${p} ${other}`, replace: { col, endCol, text: `${this.text(a, this.i)} or ${subj} ${p} ${other}` } }] }, + }); + throw STOP; + } + // "is G or C": more values for the same subject + if (op === '==' || op === '!=') { + const alts = [right]; + while (this.word() === 'or' && this.isBareValueAt(this.i + 1)) { this.i++; alts.push(this.sum()); } + if (alts.length > 1) { + if (op === '!=') { + const col = this.colOf(a); const endCol = this.endOf(this.i); + this.rd.diag('error', this.line, col, endCol, `"${this.text(a, this.i)}" could mean "is neither of them" or "is not one of them, but maybe the other". Say "is not one of ${alts.map((x) => this.textOf(x)).join(', ')}" or "is neither ${alts.map((x) => this.textOf(x)).join(' nor ')}".`, { code: 'not-or' }); + throw STOP; + } + e = alts.map((x) => N.bin('==', left, x)).reduce((x, y) => N.bin('OR', x, y)); + } + } + return this.mark(e, a); + } + return left; + } + + isComparisonWord() { + return COMPARISONS.some(([p]) => this.match(p) >= 0) || IN_PHRASES.some(([p]) => this.match(p) >= 0); + } + /** A value standing alone (a letter, a number, some text), not the start of a new comparison. */ + /** A literal standing alone: a quoted text, a number, or a bare base letter. */ + isLiteralAt(k) { + const t = this.toks[k]; + if (!t) return false; + const next = this.toks[k + 1]; + if (next && next.t === 'w' && ['is', 'are', 'equals'].includes(next.lc)) return false; + if (t.t === 's' || t.t === 'n') return true; + return t.t === 'w' && /^[ACGTUN]$/.test(t.v) && !this.ctx.scope.get(t.lc); + } + isBareValueAt(k) { + const t = this.toks[k]; + if (!t) return false; + const next = this.toks[k + 1]; + const nextIsCmp = next && next.t === 'w' && ['is', 'are', 'equals'].includes(next.lc); + if (nextIsCmp) return false; + if (t.t === 's' || t.t === 'n') return true; + if (t.t === 'w' && /^[ACGTUN]$/.test(t.v) && !this.ctx.scope.get(t.lc)) return true; + // a longer value, as long as no comparison follows it: "is 0 or the length of T minus 1" + const probe = new Sentence(this.toks, this.rd, this.ctx, this.line); probe.i = k; + const n = this.rd.diags.length; + try { probe.sum(); } catch (err) { this.rd.diags.length = n; if (err instanceof Stop) return false; throw err; } + this.rd.diags.length = n; + return probe.atEnd() || [',', ':', 'then', 'and', 'or'].includes(probe.word()); + } + textOf(e) { return e.text || sourceText(e); } + + /** "one of 'A', 'G'" / a collection. */ + collectionOrList() { + const a = this.i; + const items = []; + if (this.isLiteralAt(this.i) || this.tok()?.t === 's') { + do { + items.push(this.unary()); + if (this.word() === ',' && (this.isLiteralAt(this.i + 1) || this.match(', or|and') >= 0)) { this.i++; this.take('or|and'); continue; } + if ((this.word() === 'or' || this.word() === 'and') && this.isLiteralAt(this.i + 1)) { this.i++; continue; } + break; + } while (true); // eslint-disable-line no-constant-condition + return this.mark({ kind: 'SetLit', items }, a); + } + const c = this.sum(); + return c; + } + + quantifier() { + const a = this.i; + // for every j from A up to but not including B, C + let q = null; + if (this.take(['for every', 'for each', 'for all'])) q = 'FORALL'; + else if (this.take(['for some', 'for at least one'])) q = 'EXISTS'; + if (q) { + this.take('position|index|place|value|number'); + const vt = this.tok(); + if (vt?.t !== 'w') this.fail(vt || this.toks[a], 'Name the value it ranges over: "for every j from 0 up to but not including the length of P, ...".'); + this.i++; + const range = this.range(a); + this.take(','); + this.take(['we have', 'it holds that']); + return this.bound(q, vt.v, range, () => this.condition(), a); + } + // there is a position k of T where C + if (this.take(['there is', 'there are', 'there exists', 'there exist', 'there\'s'])) { + this.take(['a', 'an', 'some', 'at least one']); + this.take('position|positions|index|place|places|value|number'); + const vt = this.tok(); + if (vt?.t !== 'w') this.fail(vt || this.toks[a], 'Name it: "there is a position k of T where ...".'); + this.i++; + const range = this.range(a); + this.take(['where', 'such that', 'with', 'at which', 'for which', ',']); + return this.bound('EXISTS', vt.v, range, () => this.condition(), a); + } + // every letter of P is ... / some item of L is ... / no letter of T is 'N' + const lead = this.match(['every', 'each', 'all', 'all the', 'all of the', 'some', 'any', 'at least one', 'no', 'none of the']); + if (lead >= 0) { + const b = this.i; + const w0 = this.word(); + this.i += lead; + const kw = this.word(); + if (LETTER_WORDS.includes(kw) || ITEM_WORDS.includes(kw) || POSITION_WORDS.includes(kw)) { + this.i++; + if (this.take('of|in')) { + const cb = this.i; + const coll = this.primary(); + this.mark(coll, cb); + const qn = ['every', 'each', 'all'].includes(w0) ? 'FORALL' : 'EXISTS'; + const neg = w0 === 'no' || w0 === 'none'; + const v = this.rd.freshName('k'); + const vEntry = { key: '#k', words: [], display: v, pseudo: v, kind: 'bound', type: T.INT }; + const subj = POSITION_WORDS.includes(kw) ? N.name(vEntry) : N.index(coll, N.name(vEntry)); + subj.text = POSITION_WORDS.includes(kw) ? 'the position' : `the ${singular(kw)}`; + const inner = { ...this.ctx, subject: subj, position: N.name(vEntry), scope: new Scope(this.ctx.scope) }; + const saved = this.ctx; this.ctx = inner; + let body; + try { + this.take('that|which|where'); + body = this.comparison(); + } finally { this.ctx = saved; } + const node = { kind: 'Quant', q: qn, var: v, domain: { kind: 'Range', lo: N.num(0), hi: N.len(coll) }, body }; + return neg ? N.un('NOT', node) : node; + } + } + this.i = b; + } + return null; + } + + /** A quantifier or count with a named bound variable. */ + bound(q, name, range, bodyFn, a) { + void a; + const v = this.rd.pseudoName([name]); + const entry = { key: name.toLowerCase(), words: [name.toLowerCase()], display: name, pseudo: v, kind: 'bound', type: T.INT }; + const inner = { ...this.ctx, scope: new Scope(this.ctx.scope), position: N.name(entry) }; + inner.scope.add(entry); + const saved = this.ctx; this.ctx = inner; + let body; + try { body = bodyFn(); } finally { this.ctx = saved; } + if (range.step !== 1 || range.down) this.fail(this.toks[a], 'A range inside a condition counts up by one.'); + return { kind: 'Quant', q, var: v, domain: { kind: 'Range', lo: range.lo, hi: range.hi }, body }; + } + + sum() { + const a = this.i; + const saved = this.opLog; this.opLog = []; + let l = this.term(); + for (; ;) { + if (this.word() === '+' || this.word() === 'plus') { this.opLog.push({ at: this.i, add: true, word: this.word() === 'plus' }); this.i++; l = N.bin('+', l, this.term()); continue; } + if (this.word() === '-' || this.word() === 'minus') { this.opLog.push({ at: this.i, add: true, word: this.word() === 'minus' }); this.i++; l = N.bin('-', l, this.term()); continue; } + if (this.word() === '&' || this.match(['joined with', 'joined to', 'followed by']) >= 0) { this.i += this.word() === '&' ? 1 : 2; l = N.bin('&', l, this.term()); continue; } + break; + } + const log = this.opLog; this.opLog = saved; + // "i plus 1 times 2": in words, which comes first is not obvious + if (log.some((o) => o.add) && log.some((o) => !o.add) && log.some((o) => o.word)) { + const col = this.colOf(a); const endCol = this.endOf(this.i); + const opts = []; + if (log.length === 2) { + const [o1, o2] = log.slice().sort((x, y) => x.at - y.at); + const w1 = o1.len || 1; const w2 = o2.len || 1; + const x = this.text(a, o1.at); const op1 = this.text(o1.at, o1.at + w1); const y = this.text(o1.at + w1, o2.at); const op2 = this.text(o2.at, o2.at + w2); const z = this.text(o2.at + w2, this.i); + opts.push({ label: `(${x} ${op1} ${y}) ${op2} ${z}`, replace: { col, endCol, text: `(${x} ${op1} ${y}) ${op2} ${z}` } }); + opts.push({ label: `${x} ${op1} (${y} ${op2} ${z})`, replace: { col, endCol, text: `${x} ${op1} (${y} ${op2} ${z})` } }); + } + this.rd.diag('error', this.line, col, endCol, `"${this.text(a, this.i)}" mixes adding and multiplying in words, and the words do not say which comes first. Put parentheses around the part done first.`, { code: 'arith-order', question: opts.length ? { prompt: 'Which part comes first?', options: opts } : undefined }); + throw STOP; + } + return l; + } + term() { + let l = this.unary(); + for (; ;) { + if (this.word() === '*' || this.word() === 'times' || this.match('multiplied by') >= 0) { this.opLog?.push({ at: this.i, add: false, word: this.word() !== '*', len: this.word() === 'multiplied' ? 2 : 1 }); this.i += this.word() === 'multiplied' ? 2 : 1; l = N.bin('*', l, this.unary()); continue; } + if (this.word() === '/' || this.match('divided by') >= 0) { + this.opLog?.push({ at: this.i, add: false, word: this.word() !== '/', len: this.word() === 'divided' ? 2 : 1 }); + this.i += this.word() === 'divided' ? 2 : 1; + const r = this.unary(); + if (this.take([', rounded down', 'rounded down', ', whole part', 'without the remainder'])) l = N.bin('DIV', l, r); + else l = N.bin('/', l, r); + continue; + } + if (this.word() === 'mod' || this.word() === 'modulo' || this.word() === '%') { this.opLog?.push({ at: this.i, add: false, word: this.word() !== '%' }); this.i++; l = N.bin('MOD', l, this.unary()); continue; } + return l; + } + } + unary() { + if (this.word() === '-' || this.word() === 'negative' || this.word() === 'minus') { this.i++; return N.un('-', this.nested(() => this.unary())); } + return this.postfix(); + } + postfix() { + let e = this.primary(); + while (this.word() === '[') { + this.hint(e, 'sized'); + this.i++; + const k = this.sum(); + this.expect(']'); + e = N.index(e, k); + } + return e; + } + + primary() { return this.nested(() => this.primaryInner()); } + primaryInner() { + const a = this.i; + const t = this.tok(); + if (!t) this.fail(this.toks[this.toks.length - 1], 'A value is missing at the end of this sentence.', { code: 'missing-value' }); + // literals + if (t.t === 'n') { this.i++; return this.mark(N.num(t.v, t.float), a); } + if (t.t === 's') { this.i++; return this.mark(N.str(t.v, t.q), a); } + if (t.t === 'y' && t.v === '(') { this.i++; const e = this.condition(); this.expect(')'); return this.mark({ ...e, paren: true, andChain: false }, a); } + if (t.t === 'y' && t.v === '[') { + this.i++; const items = []; + if (this.word() !== ']') do { items.push(this.sum()); } while (this.take(',')); + this.expect(']'); + return this.mark({ kind: 'ListLit', items }, a); + } + if (t.t === 'y' && t.v === '{') { + this.i++; const items = []; + if (this.word() !== '}') do { items.push(this.sum()); } while (this.take(',')); + this.expect('}'); + return this.mark({ kind: 'SetLit', items }, a); + } + if (t.t !== 'w') this.fail(t, `A value is missing before "${t.v}".`, { code: 'missing-value' }); + if (this.take('yes|true')) return this.mark(N.bool(true), a); + if (this.take('no|false') && !LETTER_WORDS.includes(this.word()) && !ITEM_WORDS.includes(this.word())) return this.mark(N.bool(false), a); + this.i = a; + if (NUMBER_WORDS[t.lc] !== undefined && !(t.lc === 'one' && this.word(1) === 'of') && !(t.lc === 'one' && ['more', 'less', 'fewer'].includes(this.word(1)))) { + if (!(LETTER_WORDS.includes(this.word(1)) || ITEM_WORDS.includes(this.word(1)))) { this.i++; return this.mark(N.num(NUMBER_WORDS[t.lc]), a); } + } + if (this.match(['an? empty list', 'nothing', 'an? empty set']) >= 0 && this.ctx.constant) { this.i += this.match(['an? empty list', 'nothing', 'an? empty set']); return this.mark({ kind: 'ListLit', items: [] }, a); } + + // defined value phrases, longest first + for (const d of this.heads('value')(this.word(Math.max(0, this.match('the'))))) { + const args = this.tryParse(() => { this.take('the'); return this.templateAt(d.head.elems, d, false); }); + if (args) return this.mark(this.defCall(d, args), a); + } + // value routines used inside a sentence: "the GC count of S" + for (const r of this.heads('routines')(this.word(Math.max(0, this.match(['the result of', 'the answer to', 'what']))))) { + const args = this.tryParse(() => { this.take(['the result of', 'the answer to', 'what']); return this.templateAt(r.head.elems, r, false); }); + if (args) return this.mark({ kind: 'RoutineCall', routine: r, args }, a); + } + + const ph = this.phrase(); + if (ph) return this.mark(ph, a); + + // a name + const nm = this.nameAt(); + if (nm) { + this.i += nm.n; + let e = N.name(nm.entry); + // T's letter at i, L's last item + if (this.word() === "'s") { this.i++; e = this.possessive(e, a); } + return this.mark(e, a); + } + // its letter / its position, inside "every letter of P ..." + if (this.take(['its', 'the current']) && this.ctx.subject) { + if (this.take('position|index|place')) return this.mark(this.ctx.position, a); + if (this.take('letter|base|character|item|value')) return this.mark(this.ctx.subject, a); + this.i = a; + } + if (this.take('the? letter|base|character|text|word|sequence|string') && this.tok()?.t === 's') { const q = this.tok(); this.i++; return this.mark(N.str(q.v, q.q), a); } + this.i = a; + if (this.take(['the letter', 'the base', 'the item', 'the value', 'the character']) && this.ctx.subject && !['of', 'at'].includes(this.word())) return this.mark(this.ctx.subject, a); + this.i = a; + if (this.take(['the position', 'the index']) && this.ctx.position && !['of', 'at'].includes(this.word())) return this.mark(this.ctx.position, a); + this.i = a; + // a bare base letter + if (/^[ACGTUN]$/.test(t.v) && !this.ctx.scope.get(t.lc)) { + this.i++; + this.rd.diag('warning', this.line, t.col, t.end, `Reading ${t.v} as the letter '${t.v}'. Quote it, '${t.v}', so it cannot be mistaken for a name.`, { code: 'bare-letter', fix: { col: t.col, endCol: t.end, text: `'${t.v}'` }, question: { prompt: `Is ${t.v} the letter ${t.v}?`, options: [{ label: `Yes: write '${t.v}'`, replace: { col: t.col, endCol: t.end, text: `'${t.v}'` } }] } }); + return this.mark(N.str(t.v, '\''), a); + } + // symbolic calls: LENGTH(T), MIN(a, b) + if (this.word(1) === '(' && ['length', 'len', 'min', 'max', 'abs', 'sqrt'].includes(t.lc)) { + this.i += 2; const args = []; if (this.word() !== ')') do { args.push(this.sum()); } while (this.take(',')); + this.expect(')'); + const f = { len: 'LENGTH', length: 'LENGTH' }[t.lc] || t.lc.toUpperCase(); + if (f === 'LENGTH') this.hint(args[0], 'sized'); + return this.mark(N.call(f, args, { builtin: f }), a); + } + this.unknownName(a); + return null; + } + + unknownName(a) { + // how many words make up the unknown thing: up to a word that continues the sentence + let k = a; + while (k < this.toks.length && this.toks[k].t === 'w' && !STOP_AFTER_VALUE.has(this.toks[k].lc)) k++; + if (k === a) k = a + 1; + const words = this.text(a, k); + const t = this.toks[a]; + const col = t.col; const endCol = this.endOf(k); + const isRoutineCtx = Boolean(this.ctx.routine); + if (/^[A-Z]{2,}$/.test(words)) { + this.rd.diag('error', this.line, col, endCol, `${words} is not a name. If it is a piece of sequence, quote it: "${words}".`, { code: 'unknown-name', question: { prompt: `Is ${words} a piece of sequence?`, options: [{ label: `Yes: write "${words}"`, replace: { col, endCol, text: `"${words}"` } }] } }); + throw STOP; + } + // several words starting with "the": a phrase to define, like "the partner of b" + if (/^the\s+\S+\s+\S/i.test(words + ' ' + (this.toks[k]?.v || '')) && this.toks[k]?.lc === 'of') { + let b = k; + while (b < this.toks.length && !['is', 'are', ',', ':', 'then', 'and', 'or', 'to', 'plus', 'minus', 'times', 'divided'].includes(this.toks[b].t === 'w' ? this.toks[b].lc : this.toks[b].v)) b++; + this.unknownPhrase(a, b, this.text(a, b), 'value'); + } + const known = [...this.ctx.scope.all().values()].filter((e) => e.kind !== 'bound').map((e) => e.display); + const lw = words.toLowerCase(); + if (/^(it|its|they|them|this|that|these|those)\b/.test(lw)) { + this.rd.diag('error', this.line, col, endCol, `"${words.split(' ')[0]}" could stand for several things here${known.length ? ` (${known.join(', ')})` : ''}. Name the thing you mean.`, { code: 'pronoun' }); + throw STOP; + } + if (/^the (next|previous|last|first) (letter|base|item|position)$/.test(lw) || /^the (letter|base|item|character)$/.test(lw)) { + this.rd.diag('error', this.line, col, endCol, `Which ${lw.split(' ').pop()}? Say which text and which position: "the letter of T at i"${/next/.test(lw) ? ', or for the next one, "the letter of T at i + 1"' : ''}.`, { code: 'which-letter' }); + throw STOP; + } + const guess = closest(words, known); + if (guess) { + this.rd.diag('error', this.line, col, endCol, `The tool does not know "${words}". Did you mean ${guess}?`, { code: 'misspelled', question: { prompt: `Use ${guess}?`, options: [{ label: `Replace with ${guess}`, replace: { col, endCol, text: guess } }] } }); + throw STOP; + } + const tail = /\b(fits?|room|space|enough|off the end|end)\b/.test(this.rest(a)) ? ' To say where something fits, compare positions: "i + the length of P is at most the length of T".' : ''; + this.rd.diag('error', this.line, col, endCol, `The tool does not know "${words}".${known.length ? ` The names known here are ${known.join(', ')}.` : ''} ${isRoutineCtx ? `If "${words}" is a new value, give it a starting value in an earlier step ("Start with ${words.replace(/^the /i, '')} at 0."). If it is an idea that needs more words, define it on a line of its own, below the routine: "${this.rest(a)} when ...".` : ''}${tail}`.replace(/\s+/g, ' ').trim(), { code: 'unknown-name', unknown: words, phrase: this.rest(a) }); + throw STOP; + } + + possessive(e, a) { + void a; + const at = this.i; + if (this.take('length|size')) return N.len(e); + if (this.take(['first letter', 'first item', 'first value', 'first base'])) return N.index(e, N.num(0)); + if (this.take(['last letter', 'last item', 'last value', 'last base'])) return N.index(e, N.bin('-', N.len(e), N.num(1))); + if (this.take('letter|item|value|base|character|entry|element|score')) { this.take(['at', 'number', 'in position', 'at position']); return N.index(e, this.sum()); } + if (this.take('count|tally|entry|value for|of')) return N.index(e, this.sum()); + this.i = at; + this.fail(this.tok() || this.toks[at - 1], `After "'s", say which part: "T's letter at i", "L's length", or "Tally's count for w".`); + return null; + } + + /** Built-in phrases: letters, lengths, counts, slices, and the rest. */ + phrase() { + const a = this.i; + const w = this.word(); + const tryAt = (fn) => this.tryParse(fn); + // "the length of X", "the number of letters in X" + if (this.take('the? length|size of') || this.take('how many letters|items|values|characters|bases|entries|keys|members|elements in|of|does|do')) { + if (this.toks[this.i - 1].lc === 'does' || this.toks[this.i - 1].lc === 'do') { const x = this.hint(this.primary(), 'sized'); this.take('have|contain|hold'); return N.len(x); } + return N.len(this.hint(this.primary(), 'sized')); + } + if (this.take('how many')) { + const fam = this.word(); + this.take('letters|items|values|characters|bases|entries|keys|members|elements'); + const x = this.hint(this.primary(), LETTER_WORDS.includes(fam) ? 'text' : 'sized'); this.take('has|contains|holds|have'); + return N.len(x); + } + if (this.take(['the? number of', 'count of', 'how many'])) { + const r = this.countPhrase(a); + if (r) return r; + this.i = a; + } + // the last position of X + if (this.take('the? last position|index of')) { const x = this.primary(); return N.bin('-', N.len(x), N.num(1)); } + // the first/last letter of X, the first K letters of X + const fl = this.match('the? first|last'); + if (fl >= 0) { + const which = this.toks[this.i + fl - 1].lc; + const k = this.i; + this.i += fl; + if (this.take('letter|item|value|base|character|element|entry|score|number of|in')) { const x = this.hint(this.primary(), 'sized'); return which === 'first' ? N.index(x, N.num(0)) : N.index(x, N.bin('-', N.len(x), N.num(1))); } + const cnt = tryAt(() => { const t0 = this.tok(); if (!t0 || !['letters', 'items', 'values', 'bases', 'characters', 'elements', 'entries', 'scores', 'numbers'].includes(this.word(1))) throw STOP; const n = t0.t === 'n' ? (this.i++, N.num(t0.v)) : (() => { const nm = this.nameAt(); if (!nm || nm.n !== 1) throw STOP; this.i += 1; return N.name(nm.entry); })(); if (!this.take('letters|items|values|bases|characters|elements|entries|scores|numbers of|in')) throw STOP; const x = this.primary(); return { n, x }; }); + if (cnt) return which === 'first' ? N.call('SLICE', [cnt.x, N.num(0), cnt.n], { builtin: 'SLICE' }) : N.call('SLICE', [cnt.x, N.bin('-', N.len(cnt.x), cnt.n), N.len(cnt.x)], { builtin: 'SLICE' }); + this.i = k; + } + // the letter of X at E, the letter at E of X, letter E of X + const lw = this.match('the? letter|base|character|nucleotide|item|value|element|entry|score|number'); + if (lw >= 0 && this.word(lw) !== 'of' || (lw >= 0 && this.word(lw) === 'of')) { + const k = this.i; + this.i += lw; + if (this.word(-1) === 'number' && this.word() === 'of') { this.i = k; } + else if (this.take('of')) { + const x = this.primary(); + this.hint(x, LETTER_WORDS.includes(this.toks[k + lw - 1].lc) ? 'text' : 'list'); + if (this.take(['at', 'in position', 'at position', 'number'])) return N.index(x, this.sum()); + this.i = k; + } else if (this.take(['at', 'at position', 'in position'])) { + const at = this.findWord('of', this.i); + if (at > 0) { const i = this.sub(this.i, at, (z) => z.sum()); this.i = at + 1; const x = this.primary(); this.hint(x, LETTER_WORDS.includes(this.toks[k + lw - 1].lc) ? 'text' : 'list'); return N.index(x, i); } + this.i = k; + } else if (this.tok() && (this.tok().t === 'w' || this.tok().t === 'n' || this.word() === '(') && this.findWord('of', this.i) === this.i + 1) { + const i = this.primary(); this.expect('of'); const x = this.primary(); return N.index(x, i); + } else this.i = k; + } + // K letters of X starting at E + // (only a number or a single name can stand before "letters", which keeps this from recursing) + const lead = this.word() === 'the' ? 1 : 0; + const lettersNext = ['letters', 'items', 'values', 'bases', 'characters', 'elements', 'entries'].includes(this.word(1 + lead)); + const kl = lettersNext && (this.tok(lead)?.t === 'n' || this.tok(lead)?.t === 'w') && tryAt(() => { + this.i += lead; + const t0 = this.tok(); + const n = t0.t === 'n' ? (this.i++, N.num(t0.v)) : (() => { const nm = this.nameAt(); if (!nm || nm.n !== 1) throw STOP; this.i += 1; return N.name(nm.entry); })(); + if (!this.take('letters|items|values|bases|characters|elements of|in')) throw STOP; + const x = this.hint(this.primary(), LETTER_WORDS.includes(this.word(-1)) || true ? 'sized' : 'sized'); + if (!this.take(['starting at', 'beginning at', 'from', 'starting from', 'from position', 'starting at position'])) throw STOP; + const s = this.sum(); + return N.call('SLICE', [x, s, N.bin('+', s, n)], { builtin: 'SLICE' }); + }); + if (kl) return kl; + this.i = a; + // the part of X from A up to but not including B + if (this.take('the? part|piece|slice|stretch|section|window of')) { + const x = this.hint(this.primary(), 'sized'); + const r = this.range(a); + if (r.down || r.step !== 1) this.fail(this.toks[a], 'A part of a text runs forwards.'); + return N.call('SLICE', [x, r.lo, r.hi], { builtin: 'SLICE' }); + } + // the count for K in X / the tally of K in X + if (this.take('the? count|tally|entry|value|number for|of')) { + const inAt = this.findWord('in', this.i); + if (inAt > 0) { + const kk = this.sub(this.i, inAt, (z) => z.sum()); + this.i = inAt + 1; + const x = this.primary(); + return N.index(x, kk); + } + this.i = a; + } + // the smaller of A and B + if (this.take('the? smaller|smallest|lesser|lower|minimum of')) { + const x = this.sum(); + if (this.take('and')) { const y = this.ofArgument(a, this.text(a, this.i)); return N.call('MIN', [x, y], { builtin: 'MIN' }); } + return N.call('MIN', [x], { builtin: 'MIN' }); + } + if (this.take('the? larger|largest|greater|greatest|bigger|biggest|higher|highest|maximum of')) { + const x = this.sum(); + if (this.take('and')) { const y = this.ofArgument(a, this.text(a, this.i)); return N.call('MAX', [x, y], { builtin: 'MAX' }); } + return N.call('MAX', [x], { builtin: 'MAX' }); + } + if (this.take('the? smallest|lowest|largest|highest|biggest item|value|score|number of|in')) { + const f = ['smallest', 'lowest'].includes(this.toks[this.i - 3]?.lc || this.toks[this.i - 4]?.lc) ? 'MIN' : 'MAX'; + return N.call(f, [this.primary()], { builtin: f }); + } + if (this.take('the? absolute value of')) return N.call('ABS', [this.ofArgument(a, 'the absolute value of')], { builtin: 'ABS' }); + if (this.take('the? square root of')) return N.call('SQRT', [this.ofArgument(a, 'the square root of')], { builtin: 'SQRT' }); + if (this.take('the? remainder of|when')) { + const x = this.sum(); + // "the remainder of A divided by B": the division was read as part of A + if (x.kind === 'Binary' && x.op === '/') return N.bin('MOD', x.left, x.right); + this.expect(['divided by', 'is divided by'], '"divided by"'); return N.bin('MOD', x, this.unary()); + } + if (this.take('the? remainder')) { this.take('after|when'); const x = this.sum(); this.take('is'); this.expect('divided by', '"divided by"'); return N.bin('MOD', x, this.unary()); } + if (this.take('one more than')) return N.bin('+', this.unary(), N.num(1)); + if (this.take('one less than')) return N.bin('-', this.unary(), N.num(1)); + if (this.take('twice')) return N.bin('*', N.num(2), this.unary()); + if (this.take('the? sum|total of')) { + if (this.take('the? items|values|scores|numbers|entries|elements of|in')) { + const x = this.primary(); + const v = this.rd.freshName('k'); + return { kind: 'Quant', q: 'SUM', var: v, domain: { kind: 'Range', lo: N.num(0), hi: N.len(x) }, body: N.index(x, { kind: 'Name', name: v }) }; + } + const b = this.i; + const forAt = this.findWord('for', this.i); + if (forAt > 0 && this.match('for each|every|all', forAt) >= 0) { + const vt = this.toks[forAt + 2]; + const save = this.i; + this.i = forAt + 3; + const range = this.range(a); + const end = this.i; + const node = this.bound('SUM', vt.v, range, () => this.sub(b, forAt, (z) => z.sum()), a); + void save; + this.i = end; + return node; + } + this.i = a; + } + // the reverse of X + void w; + return null; + } + + /** + * The last value of an "of" phrase. "The absolute value of X minus 5" could + * take the absolute value of X, or of X minus 5; when arithmetic follows, + * the student says which with parentheses. + */ + ofArgument(a, phrase) { + const x = this.unary(); + const op = this.word(); + if (['plus', 'minus', 'times', '+', '-', '*', '/', 'divided', 'multiplied'].includes(op)) { + const b = this.i; + let e = this.i; + while (e < this.toks.length && ![',', ':', 'is', 'are', 'and', 'or', 'then', 'to', 'when', 'otherwise'].includes(this.toks[e].t === 'w' ? this.toks[e].lc : this.toks[e].v)) e++; + const col = this.colOf(a); const endCol = this.endOf(e); + const argText = this.text(this.i - (this.i > a ? 1 : 0), b); + void argText; + const head = this.text(a, b); const tail = this.text(b, e); + const whole = this.text(a, e); + const inner = `${phrase.replace(/^the /i, 'the ')}`; + void inner; + this.rd.diag('error', this.line, col, endCol, `"${whole}" can be read two ways: "${head}", then ${tail}; or ${phrase.replace(/^the /i, 'the ')} everything after it. Put parentheses around the part it covers.`, { + code: 'of-scope', question: { + prompt: 'Which does it mean?', options: [ + { label: `(${head}) ${tail}`, replace: { col, endCol, text: `(${head}) ${tail}` } }, + { label: `${phrase} (${this.text(a, b).slice(phrase.length).trim()} ${tail})`, replace: { col, endCol, text: `${phrase} (${this.text(a, b).slice(phrase.length).trim()} ${tail})` } }, + ] + }, + }); + throw STOP; + } + return x; + } + + /** After "the number of": positions where C, letters of X that are ..., times E appears in X. */ + countPhrase(a) { + const count = (v, lo, hi, cond) => N.len({ kind: 'SetBuilder', var: v, domain: { kind: 'Range', lo, hi }, body: cond }); + // times E appears in X + if (this.take('times')) { + const inAt = this.findWord(['in', 'inside', 'within'], this.i); + if (inAt < 0) return null; + let e; + const vt = this.findWord(['appears', 'occurs', 'shows'], this.i); + if (vt < 0 || vt > inAt) return null; + e = this.hint(this.sub(this.i, vt, (z) => z.sum()), 'sized'); + this.i = inAt + 1; + const x = this.hint(this.primary(), 'sized'); + const v = this.rd.freshName('k'); + const kv = { kind: 'Name', name: v }; + this.rd.diag('note', this.line, this.colOf(a), this.endOf(this.i), `Counting every start where ${this.textOf(e)} appears, so appearances that overlap each count ("AA" appears 3 times in "AAAA").`, { code: 'times-overlap' }); + return count(v, N.num(0), N.bin('+', N.bin('-', N.len(x), N.len(e)), N.num(1)), N.bin('==', N.call('SLICE', [x, kv, N.bin('+', kv, N.len(e))], { builtin: 'SLICE' }), e)); + } + const kw = this.word(); + if (POSITION_WORDS.includes(kw)) { + this.i++; + let name = null; + if (this.tok()?.t === 'w' && !['of', 'in', 'from', 'where', 'at', 'that'].includes(this.word())) { name = this.tok().v; this.i++; } + const range = this.range(a); + this.expect(['where', 'at which', 'such that', 'with', 'that', 'for which', 'in which'], '"where"'); + const coll = range.hi?.kind === 'CallExpr' && range.hi.builtin === 'LENGTH' ? range.hi.args[0] : null; + const vName = name || 'position'; + const node = this.bound('SUM', vName, range, () => { + if (coll) { const saved = this.ctx; this.ctx = { ...saved, subject: N.index(coll, this.ctx.position) }; try { return this.condition(); } finally { this.ctx = saved; } } + return this.condition(); + }, a); + return count(node.var, range.lo, range.hi, node.body); + } + if (LETTER_WORDS.includes(kw) || ITEM_WORDS.includes(kw)) { + this.i++; + if (!this.take('of|in')) return null; + const x = this.hint(this.primary(), LETTER_WORDS.includes(kw) ? 'text' : 'list'); + if (!this.take('that|which|where|with')) return N.len(x); + const v = this.rd.freshName('k'); + const kv = { kind: 'Name', name: v }; + const subj = N.index(x, kv); subj.text = `the ${singular(kw)}`; + const saved = this.ctx; + this.ctx = { ...saved, subject: subj, position: kv, scope: new Scope(saved.scope) }; + let cond; + try { cond = this.condition(); } finally { this.ctx = saved; } + return count(v, N.num(0), N.len(x), cond); + } + // the number of j from A ... where C + if (this.tok()?.t === 'w' && this.word(1) === 'from') { + const name = this.tok().v; this.i++; + const range = this.range(a); + this.expect(['where', 'such that', 'with', 'for which'], '"where"'); + const node = this.bound('SUM', name, range, () => this.condition(), a); + return count(node.var, range.lo, range.hi, node.body); + } + return null; + } + + /** + * The routines (`'routines'`) or phrase definitions (`'predicate'`, `'value'`, longest first) whose heads have a + * word, as a function of the word where a head would start: the heads that start with that word, or with an + * article or a slot, in their order. A head that starts with any other word fails at once in `templateAt`, so + * leaving it out changes nothing, and a sentence no longer tries every routine of a long text. + */ + heads(which) { + const cache = this.rd.headIndex || (this.rd.headIndex = new Map()); + const n = `${this.rd.routines.length},${this.rd.defs.length}`; + let got = cache.get(which); + if (!got || got.n !== n) { + const withWords = (x) => x.head.elems.some((e) => e.lit); + const list = which === 'routines' ? this.rd.routines.filter(withWords) + : this.rd.defs.filter((x) => x.kind === which && withWords(x)).sort((x, y) => y.head.elems.length - x.head.elems.length); + const keyed = new Map(); + const open = []; + list.forEach((x, i) => { + const f = x.head.elems[0]; + if (f && f.lit && !ARTICLES.has(f.lit)) { if (!keyed.has(f.lit)) keyed.set(f.lit, []); keyed.get(f.lit).push(i); } else open.push(i); + }); + const at = (w) => { + const hit = (w != null && keyed.get(w)) || []; + return (open.length && hit.length ? [...hit, ...open].sort((a, b) => a - b) : hit.length ? hit : open).map((i) => list[i]); + }; + got = { n, at }; + cache.set(which, got); + } + return got.at; + } + + /** Match a head's words and slots here; returns the argument expressions. */ + templateAt(elems, owner, full) { + const args = []; + elems.forEach((el, idx) => { + if (el.lit) { + if (this.word() === el.lit) { this.i++; return; } + // an article before a word is optional + if (ARTICLES.has(el.lit)) return; + throw STOP; + } + // a slot: stop before the next literal word + const next = elems.slice(idx + 1).find((x) => x.lit && !ARTICLES.has(x.lit)); + const last = idx === elems.length - 1; + if (next) { + const at = this.findWordFlat(next.lit, this.i); + if (at <= this.i) throw STOP; + const e = this.subFull(this.i, at); + if (!e) throw STOP; + args.push(e); this.i = at; + } else if (last && owner.kind === 'predicate') { + args.push(this.sum()); + } else { + args.push(this.unary()); + } + }); + if (full && !this.atEnd()) throw STOP; + return args; + } + findWordFlat(w, from) { let depth = 0; for (let k = from; k < this.toks.length; k++) { const t = this.toks[k]; if (t.v === '(' || t.v === '[') depth++; if (t.v === ')' || t.v === ']') depth--; if (depth === 0 && t.t === 'w' && t.lc === w) return k; } return -1; } + subFull(a, b) { + const s = new Sentence(this.toks.slice(a, b), this.rd, this.ctx, this.line); + const n = this.rd.diags.length; + try { const e = s.sum(); if (!s.atEnd()) { this.rd.diags.length = n; return null; } return e; } catch (err) { if (!(err instanceof Stop)) throw err; this.rd.diags.length = n; return null; } + } + + defCall(d, args) { + return { kind: 'DefCall', def: d, args }; + } + + /** A value definition's pieces: "E1 when C1, E2 when C2, and E3 otherwise." */ + pieces() { + const out = []; + for (; ;) { + const b = this.i; + if (this.take(['otherwise', 'in every other case', 'else'])) { this.take(','); const e = this.sum(); out.push({ value: e, cond: null, line: this.line, text: this.text(b, this.i) }); break; } + const e = this.sum(); + if (this.take('when|if|whenever')) { + const c = this.condition(); + out.push({ value: e, cond: c, text: this.text(b, this.i) }); + } else if (this.take(['otherwise', 'in every other case'])) { out.push({ value: e, cond: null, text: this.text(b, this.i) }); break; } else { out.push({ value: e, cond: null, text: this.text(b, this.i) }); break; } + if (this.atEnd()) break; + this.take(','); + this.take('and|or'); + this.take(','); + if (this.atEnd()) break; + } + return out; + } +} + +function sourceText(e) { try { return sourceOf(e); } catch { return '?'; } } + +/** A short list of sentences the tool reads, chosen by the sentence's first word. */ +function hintFor(text) { + const w = String(text).trim().split(/\s+/)[0]?.toLowerCase() || ''; + const forms = { + for: 'A loop: "For each position i of T:", "For each letter b of T:", or "For each i from 0 up to but not including N:".', + add: 'Adding: "Add 1 to Count.", "Add i to Starts.", or "Add b to the end of R."', + set: 'Changing a value: "Set Best to i."', + if: 'A test: "If the letter of T at i is B, add 1 to Count."', + }; + return forms[w] || 'Each step starts with what it does: Start with, Set, Add, Count, For each, If, Otherwise, Repeat while, Stop and answer, Show. See the list of sentences the tool reads.'; +} + +/* ------------------------------------------------------------------ */ +/* Types, by inference */ +/* ------------------------------------------------------------------ */ + +class Typer { + constructor(rd) { this.rd = rd; this.changed = false; this.conflicts = new Map(); } + + setType(entry, t, line, why) { + if (!entry || !t || unknown(t)) return; + const j = join(entry.type, t); + if (!j) { + const key = `${entry.key}@${line}`; + if (!this.conflicts.has(key)) this.conflicts.set(key, { entry, line, was: entry.type, now: t, why }); + return; + } + if (!sameT(j, entry.type)) { entry.type = j; this.changed = true; } + } + + typeOf(e) { + if (!e) return T.ANY; + switch (e.kind) { + case 'Num': return e.float ? T.FLOAT : T.INT; + case 'Str': return Array.from(e.value).length === 1 ? T.LETTER : T.STR; + case 'Bool': return T.BOOL; + case 'Name': return e.entry ? e.entry.type : T.INT; + case 'Index': { + const t = this.typeOf(e.target); + if (isText(t)) return T.CHAR; + if (t.k === 'LIST') return t.of; + if (t.k === 'HASHMAP') return t.val; + return T.ANY; + } + case 'CallExpr': { + switch (e.builtin) { + case 'LENGTH': return T.INT; + case 'MIN': case 'MAX': { if (e.args.length === 1) { const t = this.typeOf(e.args[0]); return t.k === 'LIST' ? t.of : t; } return e.args.map((x) => this.typeOf(x)).reduce((x, y) => join(x, y) || x); } + case 'ABS': return this.typeOf(e.args[0]); + case 'SQRT': return T.FLOAT; + case 'SLICE': { const t = this.typeOf(e.args[0]); return isText(t) || unknown(t) ? T.STR : t; } + case 'TO_STRING': return T.STR; + default: return T.ANY; + } + } + case 'DefCall': return e.def.ret; + case 'RoutineCall': return e.routine.ret; + case 'Binary': { + if (['==', '!=', '<', '<=', '>', '>=', 'AND', 'OR', 'IN'].includes(e.op)) return T.BOOL; + if (e.op === '&') return T.STR; + if (e.op === '/') return T.FLOAT; + if (e.op === 'DIV' || e.op === 'MOD') return T.INT; + const l = this.typeOf(e.left); const r = this.typeOf(e.right); + if (l.k === 'FLOAT' || r.k === 'FLOAT') return T.FLOAT; + if (e.op === '+' && (isText(l) || isText(r))) return T.STR; + return T.INT; + } + case 'Unary': return e.op === 'NOT' ? T.BOOL : this.typeOf(e.arg); + case 'Quant': return e.q === 'SUM' ? (this.typeOf(e.body).k === 'FLOAT' ? T.FLOAT : T.INT) : T.BOOL; + case 'SetLit': return T.set(e.items.map((x) => this.typeOf(x)).reduce((x, y) => join(x, y) || x, T.ANY)); + case 'ListLit': return T.list(e.items.map((x) => this.typeOf(x)).reduce((x, y) => join(x, y) || x, T.ANY)); + case 'Contains': return T.BOOL; + default: return T.ANY; + } + } + + /** Walk every expression, so calls inside conditions pass their argument types on. */ + exprs(e, fn) { + if (!e || typeof e !== 'object') return; + fn(e); + for (const k of ['left', 'right', 'arg', 'target', 'body', 'coll', 'item']) if (e[k]) this.exprs(e[k], fn); + for (const k of ['args', 'items', 'indices']) if (Array.isArray(e[k])) for (const x of e[k]) this.exprs(x, fn); + if (e.domain) { this.exprs(e.domain.lo, fn); this.exprs(e.domain.hi, fn); } + } + callArgs(e) { + this.exprs(e, (x) => { + // a name nothing has typed takes its type from what it is compared with, or does arithmetic with + if (x.kind === 'Binary') { + const cmp = ['==', '!=', '<', '<=', '>', '>='].includes(x.op); + const arith = ['+', '-', '*', '/', 'DIV', 'MOD'].includes(x.op); + for (const [a, b] of [[x.left, x.right], [x.right, x.left]]) { + if (a?.kind !== 'Name' || !a.entry || !unknown(a.entry.type)) continue; + // only a comparison needs the other side's type: worked out for every + of a long sum, it made reading + // the sum take the square of its length + const bt = cmp ? this.typeOf(b) : null; + if (cmp && !unknown(bt)) this.setType(a.entry, bt.k === 'LETTER' ? T.LETTER : bt, 0); + if (arith) (a.entry.hints || (a.entry.hints = new Set())).add('number'); + } + } + if (x.kind === 'DefCall') { + x.def.params.forEach((p, i) => this.setType(p, this.typeOf(x.args[i]), x.def.line)); + x.def.calls.push(x); + } + if (x.kind === 'RoutineCall') x.routine.params.forEach((p, i) => this.setType(p, this.typeOf(x.args[i]), x.routine.line)); + if (x.kind === 'Quant' || (x.kind === 'CallExpr' && x.args?.[0]?.kind === 'SetBuilder')) { /* bound names are whole numbers */ } + }); + } + + stmt(s, r) { + const ex = (e) => this.callArgs(e); + switch (s.kind) { + case 'Assign': + ex(s.value); + if (s.target.kind === 'Name') this.setType(s.target.entry, this.typeOf(s.value), s.line, 'assign'); + else if (s.target.kind === 'Index') { ex(s.target); const ct = this.typeOf(s.target.target); if (ct.k === 'LIST' && s.target.target.entry) this.setType(s.target.target.entry, T.list(this.typeOf(s.value)), s.line); if (ct.k === 'HASHMAP' && s.target.target.entry) this.setType(s.target.target.entry, T.map(this.typeOf(s.target.indices[0]), this.typeOf(s.value)), s.line); } else if (s.target.kind === 'MapEntry') { if (s.target.map.entry) this.setType(s.target.map.entry, T.map(this.typeOf(s.target.key), this.typeOf(s.value)), s.line); } + break; + case 'Empty': this.setType(s.target.entry, s.type, s.line, 'empty'); break; + case 'AddTo': { + ex(s.value); + const tt = s.target.kind === 'MapEntry' ? (s.target.map.entry ? this.typeOf(s.target.map) : T.ANY) : this.typeOf(s.target); + const vt = this.typeOf(s.value); + if (s.target.kind === 'MapEntry') { if (s.target.map.entry) this.setType(s.target.map.entry, T.map(this.typeOf(s.target.key), vt), s.line); break; } + if (s.target.kind !== 'Name') break; + const e = s.target.entry; + if (tt.k === 'LIST') this.setType(e, T.list(vt), s.line); + else if (tt.k === 'SET') this.setType(e, T.set(vt), s.line); + else if (isText(tt)) this.setType(e, T.STR, s.line); + else if (isNum(tt)) this.setType(e, vt.k === 'FLOAT' ? T.FLOAT : T.INT, s.line); + break; + } + case 'Insert': ex(s.value); if (s.target.entry && this.typeOf(s.target).k === 'LIST') this.setType(s.target.entry, T.list(this.typeOf(s.value)), s.line); break; + case 'Change': ex(s.amount); if (s.target.kind === 'Name') this.setType(s.target.entry, s.op === '/' ? T.FLOAT : this.typeOf(s.amount).k === 'FLOAT' ? T.FLOAT : T.INT, s.line); break; + case 'Tally': ex(s.key); if (s.map.entry) this.setType(s.map.entry, T.map(this.typeOf(s.key), T.INT), s.line); break; + case 'For': ex(s.lo); ex(s.hi); if (s.filter) ex(s.filter); this.setType(s.var, T.INT, s.line); break; + case 'ForEach': { + ex(s.coll); + const ct = this.typeOf(s.coll); + if (isText(ct)) this.setType(s.var, T.CHAR, s.line); + else if (ct.k === 'LIST' || ct.k === 'SET') this.setType(s.var, ct.of, s.line); + else if (ct.k === 'HASHMAP') this.setType(s.var, ct.key, s.line); + if (s.filter) ex(s.filter); + break; + } + case 'While': ex(s.cond); break; + case 'If': for (const b of s.branches) ex(b.cond); break; + case 'Return': if (s.value) { ex(s.value); const t = this.typeOf(s.value); const j = join(r.ret, t); if (j && !sameT(j, r.ret)) { r.ret = j; this.changed = true; } else if (!j && !this.conflicts.has(`#ret@${s.line}`)) this.conflicts.set(`#ret@${s.line}`, { entry: { display: 'the answer' }, line: s.line, was: r.ret, now: t }); } break; + case 'Output': s.values.forEach(ex); break; + case 'Assert': ex(s.cond); break; + case 'CallStep': s.routine.params.forEach((p, i) => { ex(s.args[i]); this.setType(p, this.typeOf(s.args[i]), s.line); }); break; + case 'Remove': if (s.what.at) ex(s.what.at); if (s.what.value) ex(s.what.value); break; + default: break; + } + for (const b of childBodies(s)) for (const x of b) this.stmt(x, r); + } + + run() { + const rd = this.rd; + for (let pass = 0; pass < 12; pass++) { + this.changed = false; + for (const r of rd.routines) for (const s of r.body) this.stmt(s, r); + for (const d of rd.defs) { + d.calls = []; + if (d.kind === 'predicate' && d.cond) this.callArgs(d.cond); + if (d.kind === 'value' && d.pieces) { + for (const p of d.pieces) { + if (p.cond) this.callArgs(p.cond); + this.callArgs(p.value); + const t = this.typeOf(p.value); + const j = join(d.ret, t); + if (j && !sameT(j, d.ret)) { d.ret = j; this.changed = true; } + } + } + for (const ip of d.implicit) this.setType(ip, ip.outer.type, d.line); + } + for (const ex of rd.examples) ex.routine.params.forEach((p, i) => this.setType(p, this.typeOf(ex.args[i]), ex.line)); + // an input nothing else has typed: its use says whether it is a text or a list + if (!this.changed) { + for (const owner of [...rd.routines, ...rd.defs]) for (const p of owner.params) { + if (!unknown(p.type) || !p.hints) continue; + const t = p.hints.has('list') ? T.list(T.ANY) : (p.hints.has('text') || p.hints.has('sized')) ? T.STR : p.hints.has('number') ? T.INT : null; + if (t) { p.type = t; p.typedByUse = true; this.changed = true; } + } + } + if (!this.changed) break; + } + // an input nothing says anything about: sequences are this course's data + for (const owner of [...rd.routines, ...rd.defs]) for (const p of owner.params) if (unknown(p.type)) { p.type = T.STR; p.typedByDefault = true; } + } +} + +function childBodies(s) { + switch (s.kind) { + case 'If': return [...s.branches.map((b) => b.body), ...(s.elseBody ? [s.elseBody] : [])]; + case 'For': case 'ForEach': case 'While': return [s.body || []]; + default: return []; + } +} + +/* ------------------------------------------------------------------ */ +/* Pseudocode */ +/* ------------------------------------------------------------------ */ + +class Emitter { + constructor(rd, typer) { this.rd = rd; this.typer = typer; this.out = []; this.map = []; } + emit(depth, text, eline, stmt = null, role = null) { this.out.push(`${' '.repeat(depth)}${text}`); this.map.push({ line: eline, stmt, role }); } + tp(e) { return this.typer.typeOf(e); } + + /** An expression as pseudocode, with each quoted letter a CHAR or a STRING as its place demands. */ + x(e, want = null) { + return sourceOf(this.lower(e, want)); + } + lower(e, want) { + if (!e) return { kind: 'Name', name: '?' }; + switch (e.kind) { + case 'Str': { + const one = Array.from(e.value).length === 1; + const asChar = one && (!want || want.k === 'CHAR' || want.k === 'LETTER' || unknown(want)); + return asChar ? { kind: 'Char', value: e.value } : { kind: 'Str', value: e.value }; + } + case 'Name': return { kind: 'Name', name: e.name }; + case 'Num': return { kind: 'Num', value: e.value, float: e.float }; + case 'Bool': return e; + case 'Index': return { kind: 'Index', target: this.lower(e.target), indices: e.indices.map((i) => this.lower(i, T.INT)) }; + case 'CallExpr': { + if (e.builtin === 'LENGTH' && e.args[0]?.kind === 'SetBuilder') { + const sb = e.args[0]; + return { kind: 'CallExpr', name: 'LENGTH', args: [{ kind: 'SetBuilder', var: sb.var, domain: { kind: 'Range', lo: this.lower(sb.domain.lo), hi: this.lower(sb.domain.hi) }, body: this.lower(sb.body, T.BOOL) }] }; + } + return { kind: 'CallExpr', name: e.name, args: e.args.map((a) => this.lower(a)) }; + } + case 'DefCall': { + const d = e.def; + const args = e.args.map((a, i) => this.lower(a, d.params[i]?.type)); + for (const ip of d.implicit) args.push({ kind: 'Name', name: ip.callerPseudo || ip.outer.pseudo }); + return { kind: 'CallExpr', name: d.pseudo, args }; + } + case 'RoutineCall': return { kind: 'CallExpr', name: e.routine.pseudo, args: e.args.map((a, i) => this.lower(a, e.routine.params[i]?.type)) }; + case 'Binary': { + if (['==', '!=', '<', '<=', '>', '>='].includes(e.op)) { + const lt = this.tp(e.left); const rt = this.tp(e.right); + let l = this.lower(e.left, rt); let r = this.lower(e.right, lt); + // a letter against a text: compare them as texts + const lk = l.kind === 'Char' ? 'CHAR' : lt.k; const rk = r.kind === 'Char' ? 'CHAR' : rt.k; + if ((lk === 'CHAR' || lk === 'LETTER') && rk === 'STRING') l = { kind: 'CallExpr', name: 'TO_STRING', args: [l] }; + if ((rk === 'CHAR' || rk === 'LETTER') && lk === 'STRING') r = { kind: 'CallExpr', name: 'TO_STRING', args: [r] }; + return { kind: 'Binary', op: e.op, left: l, right: r }; + } + if (e.op === 'IN') { + const ct = this.tp(e.right); + const elem = ct.k === 'SET' || ct.k === 'LIST' ? ct.of : ct.k === 'HASHMAP' ? ct.key : null; + const itemT = this.tp(e.left); + const right = e.right.kind === 'SetLit' ? { kind: 'SetLit', items: e.right.items.map((x) => this.lower(x, itemT)) } : this.lower(e.right); + return { kind: 'Binary', op: 'IN', left: this.lower(e.left, elem), right }; + } + if (e.op === '&') return { kind: 'Binary', op: '&', left: this.lower(e.left, T.STR), right: this.lower(e.right, T.STR) }; + return { kind: 'Binary', op: e.op, left: this.lower(e.left, want), right: this.lower(e.right, want) }; + } + case 'Unary': return { kind: 'Unary', op: e.op, arg: this.lower(e.arg, e.op === 'NOT' ? T.BOOL : want) }; + case 'Quant': return { kind: 'Quant', q: e.q, var: e.var, domain: { kind: 'Range', lo: this.lower(e.domain.lo), hi: this.lower(e.domain.hi) }, body: this.lower(e.body) }; + case 'SetLit': return { kind: 'SetLit', items: e.items.map((x) => this.lower(x, want?.of || want)) }; + case 'ListLit': return { kind: 'ListLit', items: e.items.map((x) => this.lower(x, want?.of)) }; + case 'Contains': { + const ct = this.tp(e.coll); + if (isText(ct)) { + // a text contains a letter, or a piece of text somewhere in it + const it = this.tp(e.item); + if (it.k === 'CHAR' || it.k === 'LETTER') return { kind: 'Binary', op: 'IN', left: this.lower(e.item, T.CHAR), right: this.lower(e.coll) }; + const k = this.rd.freshName('k'); const kv = { kind: 'Name', name: k }; + const coll = this.lower(e.coll); const item = this.lower(e.item, T.STR); + const len = (x) => ({ kind: 'CallExpr', name: 'LENGTH', args: [x] }); + return { kind: 'Quant', q: 'EXISTS', var: k, domain: { kind: 'Range', lo: { kind: 'Num', value: 0 }, hi: { kind: 'Binary', op: '+', left: { kind: 'Binary', op: '-', left: len(coll), right: len(item) }, right: { kind: 'Num', value: 1 } } }, body: { kind: 'Binary', op: '==', left: { kind: 'CallExpr', name: 'SLICE', args: [coll, kv, { kind: 'Binary', op: '+', left: kv, right: len(item) }] }, right: item } }; + } + return { kind: 'Binary', op: 'IN', left: this.lower(e.item, ct.k === 'HASHMAP' ? ct.key : ct.of), right: this.lower(e.coll) }; + } + default: return e; + } + } + + program() { + const rd = this.rd; + // in the order the English gives them, so the two read side by side + for (const o of [...rd.defs, ...rd.routines].sort((a, b) => a.line - b.line)) (o.kind === 'routine' ? this.routine(o) : this.def(o)); + return { source: this.out.join('\n'), map: this.map }; + } + + params(list) { return list.map((p) => `${p.pseudo} : ${typeText(resolved(p.type))}`).join(', '); } + + def(d) { + const params = [...d.params, ...d.implicit]; + if (d.kind === 'predicate') { + this.emit(0, `FUNCTION ${d.pseudo}(${this.params(params)}) RETURNS BOOL`, d.line, d, 'def-head'); + this.emit(1, `RETURN ${d.cond ? this.x(d.cond, T.BOOL) : 'FALSE'}`, d.line, d, 'def-return'); + this.emit(0, 'ENDFUNCTION', d.line, d, 'def-end'); + this.emit(0, '', null); + return; + } + const ret = resolved(d.ret); + this.emit(0, `FUNCTION ${d.pseudo}(${this.params(params)}) RETURNS ${typeText(ret)}`, d.line, d, 'def-head'); + const pieces = d.pieces || []; + const conds = pieces.filter((p) => p.cond); + const other = pieces.find((p) => !p.cond); + if (!conds.length && other) this.emit(1, `RETURN ${this.x(other.value, ret)}`, d.line, d, 'def-return'); + else { + conds.forEach((p, i) => { this.emit(1, `${i ? 'ELSEIF' : 'IF'} ${this.x(p.cond, T.BOOL)} THEN`, d.line, d, 'def-test'); this.emit(2, `RETURN ${this.x(p.value, ret)}`, d.line, d, 'def-return'); }); + if (other) { this.emit(1, 'ELSE', d.line, d, 'def-test'); this.emit(2, `RETURN ${this.x(other.value, ret)}`, d.line, d, 'def-return'); } + if (conds.length) this.emit(1, 'ENDIF', d.line, d, 'def-test'); + } + this.emit(0, 'ENDFUNCTION', d.line, d, 'def-end'); + this.emit(0, '', null); + } + + routine(r) { + const isFn = r.answers > 0; + const ret = resolved(r.ret); + this.emit(0, isFn ? `FUNCTION ${r.pseudo}(${this.params(r.params)}) RETURNS ${typeText(ret)}` : `PROCEDURE ${r.pseudo}(${this.params(r.params)})`, r.line, r, 'head'); + // every variable the routine gives a value to, declared at the top + const locals = new Map(); + const collect = (body) => { for (const s of body) { const t = s.target?.kind === 'Name' ? s.target.entry : null; if ((s.kind === 'Assign' || s.kind === 'Empty') && t && t.kind === 'var') locals.set(t.pseudo, t); for (const b of childBodies(s)) collect(b); } }; + collect(r.body); + for (const v of locals.values()) this.emit(1, `DECLARE ${v.pseudo} : ${typeText(resolved(v.type))}`, v.line, null, 'declare'); + // a phrase that uses this routine's inputs gets them from here + this.block(r.body, 1, r); + this.emit(0, isFn ? 'ENDFUNCTION' : 'ENDPROCEDURE', r.line, r, 'end'); + this.emit(0, '', null); + } + + block(body, d, r) { for (const s of body) this.stmt(s, d, r); } + + stmt(s, d, r) { + const L = s.line; + const tv = (e) => this.tp(e); + switch (s.kind) { + case 'Assign': { + if (s.target.kind === 'MapEntry') { this.emit(d, `${this.x(s.target.map)}[${this.x(s.target.key, tv(s.target.map).key)}] <- ${this.x(s.value, tv(s.target.map).val)}`, L, s, 'assign'); return; } + const tt = s.target.kind === 'Name' ? s.target.entry.type : tv(s.target); + this.emit(d, `${this.x(s.target)} <- ${this.x(s.value, resolved(tt))}`, L, s, 'assign'); + return; + } + case 'Empty': { + const t = resolved(s.target.entry.type); + // a collection is empty from its DECLARE; only a text needs a value + if (t.k === 'STRING') this.emit(d, `${s.target.name} <- ""`, L, s, 'empty'); + return; + } + case 'AddTo': { + if (s.target.kind === 'MapEntry') { + const mt = tv(s.target.map); + const k = this.x(s.target.key, mt.key); + this.emit(d, `${this.x(s.target.map)}[${k}] <- ${this.x(s.target.map)}[${k}] + ${this.x(s.value)}`, L, s, 'map-add'); + return; + } + const tt = resolved(tv(s.target)); + const tgt = this.x(s.target); + if (tt.k === 'LIST') { + if (s.place === 'front') this.emit(d, `INSERT(${tgt}, 0, ${this.x(s.value, tt.of)})`, L, s, 'insert'); + else this.emit(d, `APPEND(${tgt}, ${this.x(s.value, tt.of)})`, L, s, 'append'); + } else if (tt.k === 'SET') this.emit(d, `ADD(${tgt}, ${this.x(s.value, tt.of)})`, L, s, 'set-add'); + else if (tt.k === 'STRING' || tt.k === 'CHAR') { + if (s.place === 'front') this.emit(d, `${tgt} <- ${sourceOf(this.lower({ kind: 'Binary', op: '&', left: s.value, right: s.target }))}`, L, s, 'text-add'); + else this.emit(d, `${tgt} <- ${sourceOf(this.lower({ kind: 'Binary', op: '&', left: s.target, right: s.value }))}`, L, s, 'text-add'); + } else if (tt.k === 'HASHMAP') { + this.emit(d, `// ${tgt} is a tally: count a value in it with "Count ... in ${s.target.entry?.display}"`, L, s, 'bad'); + } else this.emit(d, `${tgt} <- ${tgt} + ${this.x(s.value, tt)}`, L, s, 'add'); + return; + } + case 'Insert': { const tt = resolved(tv(s.target)); this.emit(d, `INSERT(${this.x(s.target)}, ${this.x(s.pos)}, ${this.x(s.value, tt.of)})`, L, s, 'insert'); return; } + case 'Change': { + if (s.target.kind === 'MapEntry') { const mt = tv(s.target.map); const k = this.x(s.target.key, mt.key); this.emit(d, `${this.x(s.target.map)}[${k}] <- ${this.x(s.target.map)}[${k}] ${s.op} ${this.x(s.amount)}`, L, s, 'change'); return; } + const tgt = this.x(s.target); + const amount = s.amount.kind === 'Binary' ? `(${this.x(s.amount)})` : this.x(s.amount); + this.emit(d, `${tgt} <- ${tgt} ${s.op} ${amount}`, L, s, 'change'); + return; + } + case 'Tally': { + const mt = resolved(tv(s.map)); + const m = this.x(s.map); const k = this.x(s.key, mt.key); + this.emit(d, `IF ${k} IN ${m} THEN`, L, s, 'tally-test'); + this.emit(d + 1, `${m}[${k}] <- ${m}[${k}] + 1`, L, s, 'tally-more'); + this.emit(d, 'ELSE', L, s, 'tally-else'); + this.emit(d + 1, `${m}[${k}] <- 1`, L, s, 'tally-new'); + this.emit(d, 'ENDIF', L, s, 'tally-end'); + return; + } + case 'For': { + const stepTxt = s.step && s.step !== 1 ? ` STEP ${s.step}` : ''; + this.emit(d, `FOR ${s.var.pseudo} <- ${this.x(s.lo)} TO ${this.x(s.hi)}${stepTxt}`, L, s, 'for'); + this.filtered(s, d, r); + this.emit(d, `NEXT ${s.var.pseudo}`, L, s, 'next'); + return; + } + case 'ForEach': { + this.emit(d, `FOR EACH ${s.var.pseudo} IN ${this.x(s.coll)}`, L, s, 'each'); + this.filtered(s, d, r); + this.emit(d, `NEXT ${s.var.pseudo}`, L, s, 'next'); + return; + } + case 'While': + this.emit(d, `WHILE ${this.x(s.cond, T.BOOL)}`, L, s, 'while'); + this.block(s.body, d + 1, r); + this.emit(d, 'ENDWHILE', L, s, 'endwhile'); + return; + case 'If': + s.branches.forEach((b, i) => { this.emit(d, `${i ? 'ELSEIF' : 'IF'} ${this.x(b.cond, T.BOOL)} THEN`, b.line ?? L, s, i ? 'elseif' : 'if'); this.block(b.body, d + 1, r); }); + if (s.elseBody) { this.emit(d, 'ELSE', s.elseLine ?? L, s, 'else'); this.block(s.elseBody, d + 1, r); } + this.emit(d, 'ENDIF', L, s, 'endif'); + return; + case 'Return': + if (!s.value) { this.emit(d, 'RETURN', L, s, 'return'); return; } + this.emit(d, `RETURN ${this.x(s.value, resolved(r.ret))}`, L, s, 'return'); + return; + case 'Break': this.emit(d, 'BREAK', L, s, 'break'); return; + case 'Continue': this.emit(d, 'CONTINUE', L, s, 'continue'); return; + case 'Output': this.emit(d, `OUTPUT ${s.values.map((v) => this.x(v)).join(', " ", ')}`, L, s, 'output'); return; + case 'Assert': this.emit(d, `ASSERT ${this.x(s.cond, T.BOOL)}`, L, s, 'assert'); return; + case 'CallStep': { + const args = s.args.map((a, i) => this.x(a, s.routine.params[i]?.type)); + if (s.routine.answers > 0) { this.emit(d, `// the answer of ${s.routine.pseudo} is not used`, L, s, 'bad'); return; } + this.emit(d, `CALL ${s.routine.pseudo}(${args.join(', ')})`, L, s, 'call'); + return; + } + case 'Remove': { + const tgt = this.x(s.target); const tt = resolved(tv(s.target)); + if (tt.k === 'SET' || tt.k === 'HASHMAP') { this.emit(d, `DELETE(${tgt}, ${this.x(s.what.value || s.what.at, tt.k === 'SET' ? tt.of : tt.key)})`, L, s, 'remove'); return; } + const pos = s.what.last ? `LENGTH(${tgt}) - 1` : s.what.first ? '0' : s.what.at ? this.x(s.what.at) : null; + if (pos == null) { this.emit(d, `// say which position to remove from ${tgt}`, L, s, 'bad'); return; } + this.emit(d, `REMOVE(${tgt}, ${pos})`, L, s, 'remove'); + return; + } + case 'Sort': this.emit(d, `SORT(${this.x(s.target)})`, L, s, 'sort'); return; + case 'Swap': { + const t = resolved(tv(s.a)); + const tmp = r.swapTemp || (r.swapTemp = this.rd.freshName('held')); + // the temporary is declared where it is first needed + this.out.splice(this.firstBodyLine(r), 0, ` DECLARE ${tmp} : ${typeText(t)}`); + this.map.splice(this.firstBodyLine(r), 0, { line: L, stmt: s, role: 'declare' }); + this.emit(d, `${tmp} <- ${this.x(s.a)}`, L, s, 'swap1'); + this.emit(d, `${this.x(s.a)} <- ${this.x(s.b)}`, L, s, 'swap2'); + this.emit(d, `${this.x(s.b)} <- ${tmp}`, L, s, 'swap3'); + return; + } + default: return; + } + } + firstBodyLine(r) { const k = this.map.findIndex((m) => m.stmt === r && m.role === 'head'); return k + 1; } + + filtered(s, d, r) { + if (s.filter) { + this.emit(d + 1, `IF ${this.x(s.filter, T.BOOL)} THEN`, s.line, s, 'filter'); + this.block(s.body, d + 2, r); + this.emit(d + 1, 'ENDIF', s.line, s, 'filter-end'); + } else this.block(s.body, d + 1, r); + } +} + +function resolved(t) { + if (!t || t.k === 'ANY') return T.INT; + if (t.k === 'LETTER') return T.CHAR; + if (t.k === 'LIST') return T.list(unknown(t.of) ? T.INT : resolved(t.of)); + if (t.k === 'SET') return T.set(unknown(t.of) ? T.STR : resolved(t.of)); + if (t.k === 'HASHMAP') return T.map(unknown(t.key) ? T.STR : resolved(t.key), unknown(t.val) ? T.INT : resolved(t.val)); + return t; +} + +/* ------------------------------------------------------------------ */ +/* Messages from the pseudocode checker, in the English program's terms */ +/* ------------------------------------------------------------------ */ + +function translateCheckerDiag(d, map, rd, display) { + const m = map[d.line - 1]; + const line = m?.line ?? null; + if (line == null) return null; + const nameOut = (s) => String(s).replace(/\b[A-Za-z_][A-Za-z0-9_]*\b/g, (w) => display.get(w.toUpperCase()) || w); + let message = nameOut(d.message); + switch (d.code) { + case 'missing-return': { + const r = m.stmt; + if (r?.kind === 'value') message = `"${r.head.text}" has no value when none of its cases apply. End the definition with a value for every other case: "..., and 'N' otherwise."`; + else message = `Some path through "To ${r?.head?.text ?? 'this routine'}" reaches the end without an answer. Add "The answer is ..." as the last step, or make sure every path stops with an answer.`; + break; + } + case 'type': message = `The kinds of value on this line do not fit together: ${message}`; break; + default: break; + } + return { ...d, line, col: 1, endCol: null, message, source: 'english', code: `checker-${d.code}` }; +} + +/* ------------------------------------------------------------------ */ +/* The entry point */ +/* ------------------------------------------------------------------ */ + +export function readSteps(source, options = {}) { + const rd = new Reader(source, options); + // inputs whose kind the page or the student has said + for (const [k, v] of Object.entries(options.inputs || {})) { + const t = typeof v === 'string' ? kindFromWords(v.split(/\s+/)) : v; + if (t) rd.kinds.set(k.toLowerCase(), t); + } + rd.read(); + // examples given outside the text, by the widget + const typer = new Typer(rd); + typer.run(); + // kinds said in words override what examples suggest, but must agree + for (const r of rd.routines) for (const p of r.params) { const k = rd.kinds.get(p.key); if (k) p.type = join(p.type, k) ? k : p.type; } + for (const c of typer.conflicts.values()) { + rd.diag('error', c.line, 1, null, `${c.entry.display} holds ${kindText(c.was)} earlier, and this step gives it ${kindText(c.now)}. A name keeps one kind of value for the whole routine; use a second name.`, { code: 'kind-clash' }); + } + // variables used before anything gives them a value + checkFlow(rd); + checkTyped(rd, typer); + const result = { + source: String(source), diags: rd.diags, routines: rd.routines, defs: rd.defs, examples: rd.examples, + main: rd.routines[0] || null, kinds: rd.kinds, + pseudo: '', lineMap: [], study: null, display: new Map(), ok: false, + }; + for (const r of rd.routines) { result.display.set(r.pseudo.toUpperCase(), `"${r.head.text}"`); for (const p of r.params) result.display.set(p.pseudo.toUpperCase(), p.display); walkEntries(r.scope, (e) => result.display.set(e.pseudo.toUpperCase(), e.display)); } + for (const d of rd.defs) { result.display.set(d.pseudo.toUpperCase(), `"${d.head.text}"`); for (const p of [...d.params, ...d.implicit]) result.display.set(p.pseudo.toUpperCase(), p.display); } + const errors = rd.diags.filter((d) => d.severity === 'error'); + if (!rd.routines.length && !errors.length) rd.diag('error', 1, 1, null, 'There is no routine yet. Start the steps with a line that says what they do, ending in a colon: "To count the G bases in T:".', { code: 'no-routine' }); + if (rd.diags.some((d) => d.severity === 'error')) { result.diags = sortDiags(rd.diags); return result; } + // implicit inputs of phrases come from the caller, by the same name + for (const d of rd.defs) for (const ip of d.implicit) ip.callerPseudo = ip.outer.pseudo; + const em = new Emitter(rd, typer); + const { source: pseudo, map } = em.program(); + if (map.length > MAX_PSEUDO_LINES) { + rd.diag('error', rd.routines[0]?.line ?? 1, 1, null, `These steps become more than ${MAX_PSEUDO_LINES.toLocaleString('en-US')} lines of pseudocode, which is more than this tool follows. Keep the steps to one algorithm at a time.`, { code: 'too-long' }); + result.diags = sortDiags(rd.diags); + return result; + } + result.pseudo = pseudo; + result.lineMap = map; + const st = study(pseudo, { coach: false }); + result.study = st; + for (const d of st.diags) { + if (d.severity !== 'error') continue; + const t = translateCheckerDiag(d, map, rd, result.display); + if (t) rd.diags.push(t); + } + result.diags = sortDiags(rd.diags); + result.ok = st.ok && !rd.diags.some((d) => d.severity === 'error'); + result.callFor = (values) => callFor(result, values); + return result; +} + +function walkEntries(scope, fn) { if (!scope) return; for (const e of scope.names.values()) fn(e); } +function sortDiags(ds) { const rank = { error: 0, warning: 1, note: 2 }; return ds.slice().sort((a, b) => (a.line ?? 0) - (b.line ?? 0) || rank[a.severity] - rank[b.severity] || (a.col ?? 0) - (b.col ?? 0)); } + +/** + * A value used before any step gives it one, on the path the steps are + * written in. The pseudocode interpreter would stop there too; saying it + * here, in the student's words, names the missing set-up step. + */ +function checkFlow(rd) { + for (const r of rd.routines) { + const seen = new Set(r.params.map((p) => p.key)); + const visit = (body) => { + for (const s of body) { + const reads = []; + const collect = (e) => { if (!e || typeof e !== 'object') return; if (e.kind === 'Name' && e.entry && e.entry.kind === 'var') reads.push(e); for (const k of ['left', 'right', 'arg', 'target', 'body', 'coll', 'item', 'map', 'key', 'value', 'cond', 'amount', 'lo', 'hi', 'filter']) if (e[k] && typeof e[k] === 'object') collect(e[k]); for (const k of ['args', 'items', 'indices', 'values']) if (Array.isArray(e[k])) e[k].forEach(collect); if (e.domain) { collect(e.domain.lo); collect(e.domain.hi); } if (e.what) { collect(e.what.at); collect(e.what.value); } }; + const own = { ...s }; delete own.body; delete own.branches; delete own.elseBody; + if (s.kind === 'If') for (const b of s.branches) collect(b.cond); + if ((s.kind === 'Assign' || s.kind === 'Empty') && s.target.kind === 'Name') { collect(s.value); } + else collect(own); + for (const n of reads) { + if (!seen.has(n.entry.key) && !n.entry.reported) { + n.entry.reported = true; + rd.diag('error', s.line, n.span?.[0] ?? 1, n.span?.[1] ?? null, `${n.entry.display} is used here before any step gives it a value. Set it up first, before the loop: "Start with ${n.entry.display} at 0."`, { code: 'used-before-set', setup: n.entry.display }); + } + } + if ((s.kind === 'Assign' || s.kind === 'Empty') && s.target.kind === 'Name') { if (!seen.has(s.target.entry.key)) s.target.entry.firstLine = s.line; seen.add(s.target.entry.key); } + for (const b of childBodies(s)) visit(b); + } + }; + visit(r.body); + } +} + +/** Checks that need the types: where a text grows, letters against longer texts, inputs a phrase borrows. */ +function checkTyped(rd, typer) { + const walkExpr = (e, fn) => { if (!e || typeof e !== 'object') return; if (Array.isArray(e)) { e.forEach((x) => walkExpr(x, fn)); return; } if (e.kind) fn(e); for (const [k, v] of Object.entries(e)) if (!['entry', 'def', 'routine', 'it', 'head', 'scope', 'attachTo', 'var', 'params', 'implicit', 'calls', 'body', 'branches', 'elseBody'].includes(k) && v && typeof v === 'object') walkExpr(v, fn); }; + const walkStmts = (body, fn) => { for (const st of body || []) { fn(st); for (const b of childBodies(st)) walkStmts(b, fn); } }; + const letterVsText = (e, line) => { + if (e.kind !== 'Binary' || !['==', '!='].includes(e.op)) return; + for (const [x, y] of [[e.left, e.right], [e.right, e.left]]) { + const xt = typer.typeOf(x); + if ((xt.k === 'CHAR') && y.kind === 'Str' && Array.from(y.value).length > 1) { + rd.diag('warning', line, y.span?.[0] ?? 1, y.span?.[1] ?? null, `${x.text || 'A single letter'} is one letter, so it is never the same as the ${Array.from(y.value).length}-letter text "${y.value}": this test is always ${e.op === '==' ? 'no' : 'yes'}. To compare a piece of a text, use "the ${Array.from(y.value).length} letters of T starting at i".`, { code: 'letter-vs-text' }); + } + } + }; + for (const r of rd.routines) { + walkStmts(r.body, (st) => { + // "Add b to R", with R a text: at the end, or at the front? + if (st.kind === 'AddTo' && st.place === 'plain' && st.target.kind === 'Name' && isText(st.target.entry.type) && st.toCol) { + rd.diag('error', st.line, st.toCol, st.endCol, `${st.target.entry.display} is a text, and "add to" does not say where the new letters go: at the end, or at the front. Reversing a text depends on exactly this.`, { + code: 'text-place', question: { + prompt: 'Where do they go?', options: [ + { label: `At the end of ${st.targetText}`, replace: { col: st.toCol, endCol: st.endCol, text: `to the end of ${st.targetText}` } }, + { label: `At the front of ${st.targetText}`, replace: { col: st.toCol, endCol: st.endCol, text: `to the front of ${st.targetText}` } }, + ] + }, + }); + } + const own = [st, ...(st.kind === 'If' ? st.branches.map((b) => b.cond) : [])]; + walkExpr(own, (e) => { + letterVsText(e, st.line); + // a phrase that reads an input its caller does not have + if (e.kind === 'DefCall') for (const ip of e.def.implicit) { + if (!r.scope.get(ip.key) && !r.params.some((p) => p.key === ip.key)) { + rd.diag('error', st.line, 1, null, `"${e.def.head.text}" reads ${ip.display}, and "To ${r.head.text}" has nothing called ${ip.display}. A phrase can borrow a routine's input only by the same name. Name it in the phrase instead: "${e.def.head.text} in ${ip.display}".`, { code: 'borrowed-input' }); + } + } + }); + }); + } + for (const d of rd.defs) walkExpr(d.kind === 'predicate' ? d.cond : d.pieces, (e) => letterVsText(e, d.line)); +} + +/** A call of the main routine with the given input values (by input name). */ +export function callFor(result, values, routine = result.main) { + if (!routine) throw new Error('There is no routine to call.'); + const args = routine.params.map((p) => { + const key = Object.keys(values).find((k) => k.toLowerCase() === p.key); + if (key == null) throw new Error(`The example gives no value for ${p.display}.`); + return convert(values[key], resolved(p.type), p.display); + }); + return { name: routine.pseudo, args }; +} + +/** A value typed by a student, as the interpreter holds it: "GATTACA", 'A', 3, [1, 2]. */ +export function parseValue(text, type = null) { + let s = String(text).trim(); + if (/^(yes|true)$/i.test(s)) return true; + if (/^(no|false)$/i.test(s)) return false; + if (/^(nothing|none|an? empty list|empty)$/i.test(s)) return type?.k === 'STRING' ? '' : []; + // bare DNA or words, where a text is expected + if ((type?.k === 'STRING' || type?.k === 'CHAR') && /^[A-Za-z]*$/.test(s)) return s; + s = s.replace(/[“”]/g, '"').replace(/[‘’]/g, '\''); + const p = parseExpression(s); + if (p.diags.some((d) => d.severity === 'error')) throw new Error(`"${text}" is not a value. Write text in quotes ("GATTACA"), a letter in single quotes ('A'), a number, yes or no, or a list in brackets ([1, 4]).`); + return evaluateConstant({ constants: new Map(), defines: new Map(), types: new Map(), subs: new Map() }, p.expr); +} + +function convert(v, t, name) { + if (typeof v === 'string' && !(t.k === 'STRING' || t.k === 'CHAR')) v = parseValue(v, t); + else if (typeof v === 'string' && /^["'].*["']$/.test(v.trim())) v = parseValue(v, t); + switch (t.k) { + case 'CHAR': if (typeof v !== 'string' || chars(v).length !== 1) throw new Error(`${name} is a letter, and the example gives ${formatValue(v)}.`); return v; + case 'STRING': if (typeof v !== 'string') throw new Error(`${name} is a text, and the example gives ${formatValue(v)}.`); return v; + case 'INTEGER': if (!Number.isInteger(v)) throw new Error(`${name} is a whole number, and the example gives ${formatValue(v)}.`); return v; + case 'FLOAT': if (typeof v !== 'number') throw new Error(`${name} is a number, and the example gives ${formatValue(v)}.`); return v; + case 'BOOL': if (typeof v !== 'boolean') throw new Error(`${name} is yes or no.`); return v; + case 'LIST': if (!Array.isArray(v)) throw new Error(`${name} is a list; write it in brackets, [1, 4].`); return v.map((x) => convert(x, t.of, name)); + case 'SET': return new Set((Array.isArray(v) ? v : [...v]).map((x) => convert(x, t.of, name))); + case 'HASHMAP': return v instanceof Map ? v : new Map(Object.entries(v || {})); + default: return v; + } +} + +/* ------------------------------------------------------------------ */ +/* Values and steps, in English */ +/* ------------------------------------------------------------------ */ + +/** A value the way the English side writes it: yes/no for BOOL, quotes for text. */ +export function englishValue(v, type) { + if (v === UNSET || v === undefined) return 'no value yet'; + if (typeof v === 'boolean') return v ? 'yes' : 'no'; + if (type?.k === 'LETTER') return formatValue(v, T.CHAR); + return formatValue(v, type ? resolved(type) : undefined); +} + +/** The English statement a pseudocode step came from. */ +export function englishLineOf(result, pseudoLine) { return result.lineMap[pseudoLine - 1]?.line ?? null; } + +/** + * One sentence about what a step of the run did, in the English program's + * words, or null for a step the English side does not show (a DECLARE, the + * inside of a tally). `step` is a trace step and `state` the snapshot after it. + */ +export function narrate(result, step, state) { + if (!step) return ''; + if (step.kind === 'declare') return null; + const m = result.lineMap[step.line - 1]; + if (!m) return step.kind === 'error' ? step.text : null; + const s = m.stmt; + const frame = state?.frames?.[state.frames.length - 1]; + const vars = frame?.vars || {}; + const show = (pseudo, fallback = '?') => { const v = vars[String(pseudo).toUpperCase()]; return v ? englishValue(v.value, v.type) : fallback; }; + const said = (e) => (e?.text ? e.text : e ? englishExpr(e, result) : ''); + const valueOf = (e) => { try { return englishValue(evalNode(e, vars)); } catch { return null; } }; + const owners = [...result.routines, ...result.defs]; + if (step.kind === 'call') { + const callee = owners.find((o) => o.pseudo === step.callee); + if (!callee) return step.text; + const ps = callee.kind === 'routine' ? callee.params : [...callee.params, ...callee.implicit]; + const vals = ps.map((p) => `${p.display} = ${show(p.pseudo)}`).join(', '); + return step.depth <= 1 ? `Starts "${callee.head.text}" with ${vals}.` : `Works out "${callee.head.text}" with ${vals}.`; + } + if (step.kind === 'return') { + const v = englishValue(step.value, s?.kind === 'predicate' ? { k: 'BOOL' } : undefined); + if (m.role === 'def-return') return `So "${s.head.text}" is ${v}.`; + if (!s || s.value == null) return 'Stops here.'; + return step.depth > 1 ? `So the answer of this part is ${v}.` : `The answer is ${v}.`; + } + if (step.kind === 'check') return `Checks that ${said(s?.cond)}: it holds.`; + switch (m.role) { + case 'head': return null; + case 'def-test': return null; + case 'empty': return `${s.target.entry.display} starts empty: ${show(s.target.name)}.`; + case 'assign': { + if (s.target.kind === 'Name') { + const first = s.target.entry.firstLine === s.line; + return `${s.target.entry.display} ${first ? 'starts at' : 'is now'} ${show(s.target.name)}.`; + } + return `${said(s.target)} is now ${valueOf(s.value) ?? englishValue(step.value)}.`; + } + case 'append': case 'insert': case 'set-add': { + const v = valueOf(s.value); + return `${said(s.value)}${v && v !== said(s.value) ? ` (${v})` : ''} goes ${m.role === 'insert' ? 'on the front of' : m.role === 'set-add' ? 'into' : 'on the end of'} ${s.target.entry?.display}, which is now ${show(s.target.name)}.`; + } + case 'text-add': return `${s.target.entry?.display} is now ${show(s.target.name)}.`; + case 'add': case 'change': case 'map-add': return `${s.target.entry?.display ?? said(s.target.map)} is now ${s.target.kind === 'Name' ? show(s.target.name) : show(s.target.map?.name)}.`; + case 'tally-test': { + const k = valueOf(s.key) ?? said(s.key); + return step.outcome ? `${k} is already in ${s.map.entry?.display}, so its count goes up by one.` : `${k} is not in ${s.map.entry?.display} yet, so it goes in with a count of 1.`; + } + case 'tally-more': case 'tally-new': return `${s.map.entry?.display} is now ${show(s.map.name)}.`; + case 'tally-else': case 'tally-end': return null; + case 'for': { + const v = s.var; + if (step.kind === 'loop-start') return s.times ? `Repeats ${said(s.hi)} times.` : `${v.display} starts at ${show(v.pseudo)}.`; + if (step.kind === 'loop-test') return step.outcome === false ? `${v.display} has reached the end of its range, so the loop is done.` : s.times ? `Pass ${Number(show(v.pseudo)) + 1}.` : `${v.display} is ${show(v.pseudo)}: another pass.`; + if (step.kind === 'loop-end') return s.times ? 'The repeats are done.' : `${v.display} has reached the end of its range, so the loop is done.`; + return null; + } + case 'each': { + const v = s.var; + const what = s.mode === 'letters' ? 'letter' : s.mode === 'keys' ? 'key' : s.mode === 'members' ? 'member' : 'item'; + if (step.kind === 'loop-end' || step.outcome === false) return `No ${what}s are left, so the loop is done.`; + if (step.kind === 'loop-start') return `${v.display} is the first ${what} of ${said(s.coll)}: ${show(v.pseudo)}.`; + return `${v.display} is ${show(v.pseudo)}.`; + } + case 'next': { + if (s.kind === 'For') return `Moves on: ${s.var.display} becomes ${show(s.var.pseudo)}.`; + return `Moves on to the next ${s.mode === 'letters' ? 'letter' : 'item'}: ${s.var.display} is ${show(s.var.pseudo)}.`; + } + case 'filter': case 'if': case 'elseif': case 'while': { + if (step.kind !== 'test' && step.kind !== 'loop-test' && step.kind !== 'loop-end') return null; + const c = m.role === 'if' || m.role === 'elseif' ? (s.branches.find((b) => b.line === m.line && (m.role === 'if' ? b === s.branches[0] : b !== s.branches[0]))?.cond || s.branches[0].cond) : m.role === 'while' ? (s.until || s.cond) : s.filter; + let yes = step.outcome; + if (m.role === 'while' && s.until) yes = !yes; + if (step.kind === 'loop-end') yes = s.until ? true : false; + const detail = describeValues(c, vars, result); + const verdict = yes ? 'it holds' : 'it does not'; + const then = m.role === 'while' + ? ((s.until ? !yes : yes) ? 'the loop goes round again.' : 'the loop is done.') + : yes ? 'the steps under it run.' : m.role === 'filter' ? 'this pass does nothing.' : (s.elseBody || s.branches.length > 1) ? 'it moves on to the next case.' : 'the steps under it are skipped.'; + return `Checks whether ${said(c)}: ${detail ? `${detail}, so ${verdict}` : verdict}; ${then}`; + } + case 'else': return 'None of the conditions above held, so the "Otherwise" steps run.'; + case 'endif': case 'endwhile': case 'filter-end': case 'def-end': case 'end': return null; + case 'break': return 'Stops the loop.'; + case 'continue': return 'Skips the rest of this pass.'; + case 'output': return `Shows ${s.values.map((v) => valueOf(v) ?? said(v)).join(' ')}.`; + case 'assert': return `Checks that ${said(s.cond)}: it holds.`; + case 'swap1': return null; + case 'swap2': return null; + case 'swap3': return `Swaps ${said(s.a)} and ${said(s.b)}: ${said(s.a)} is now ${valueOf(s.a)}, and ${said(s.b)} is ${valueOf(s.b)}.`; + case 'remove': return `Removes from ${s.target.entry?.display}, which is now ${show(s.target.name)}.`; + case 'sort': return `Sorts ${s.target.entry?.display}: now ${show(s.target.name)}.`; + case 'call': return null; + default: return step.text || null; + } +} + +/** "the letter of T at i is 'G', and B is 'A'": the values a condition compared. */ +function describeValues(c, vars, result) { + if (!c) return ''; + const parts = []; + const seen = new Set(); + const look = (e) => { + if (!e || typeof e !== 'object' || parts.length >= 3) return; + if (e.kind === 'Binary' && ['==', '!=', '<', '<=', '>', '>='].includes(e.op)) { + for (const side of [e.left, e.right]) { + if (side.kind === 'Num' || side.kind === 'Str' || side.kind === 'Bool') continue; + const txt = side.text || englishExpr(side, result); + if (seen.has(txt)) continue; + seen.add(txt); + try { const v = evalNode(side, vars); parts.push(`${txt} is ${englishValue(v)}`); } catch { /* a value it cannot show */ } + } + return; + } + if (e.kind === 'Binary' && (e.op === 'AND' || e.op === 'OR')) { look(e.left); look(e.right); return; } + if (e.kind === 'Unary') look(e.arg); + }; + look(c); + return parts.join(', and '); +} + +/** Evaluate a simple expression against a frame's variables, for narration only. */ +function evalNode(e, vars) { + switch (e.kind) { + case 'Num': return e.value; + case 'Str': return e.value; + case 'Bool': return e.value; + case 'Name': { const v = vars[String(e.name).toUpperCase()]; if (!v || v.value === UNSET) throw new Error('unset'); return v.value; } + case 'Index': { + const t = evalNode(e.target, vars); const i = evalNode(e.indices[0], vars); + if (typeof t === 'string') { const cs = chars(t); if (!(i >= 0 && i < cs.length)) throw new Error('range'); return cs[i]; } + if (Array.isArray(t)) { if (!(i >= 0 && i < t.length)) throw new Error('range'); return t[i]; } + if (t instanceof Map) { if (!t.has(i)) throw new Error('key'); return t.get(i); } + throw new Error('index'); + } + case 'CallExpr': + if (e.builtin === 'LENGTH' && e.args[0]?.kind !== 'SetBuilder') { const x = evalNode(e.args[0], vars); return typeof x === 'string' ? chars(x).length : x instanceof Map || x instanceof Set ? x.size : x.length; } + if (e.builtin === 'SLICE') { const x = evalNode(e.args[0], vars); const a = evalNode(e.args[1], vars); const b = evalNode(e.args[2], vars); return typeof x === 'string' ? chars(x).slice(a, b).join('') : x.slice(a, b); } + throw new Error('call'); + case 'Binary': { + const l = evalNode(e.left, vars); const r = evalNode(e.right, vars); + switch (e.op) { case '+': return l + r; case '-': return l - r; case '*': return l * r; case '&': return String(l) + String(r); default: throw new Error('op'); } + } + default: throw new Error('kind'); + } +} + +/** An expression in plain words, for a node the student did not write (a count's hidden parts). */ +export function englishExpr(e, result) { + const d = (n) => result?.display?.get(String(n).toUpperCase()) || n; + const s = sourceText(e); + return s.replace(/\b[A-Za-z_][A-Za-z0-9_]*\b/g, (w) => d(w)); +} + +/* ------------------------------------------------------------------ */ +/* The sentences the tool reads, for the reference panel */ +/* ------------------------------------------------------------------ */ + +export const SENTENCE_FORMS = [ + { + group: 'Starting a routine', forms: [ + ['To :', 'To find every start of P in T:'], + ['The answer is .', 'The answer is Starts.'], + ['Stop and answer .', 'Stop and answer i.'], + ] + }, + { + group: 'Giving a value a name', forms: [ + ['Start with at .', 'Start with Count at 0.'], + ['Start with an empty list called .', 'Start with an empty list called Starts.'], + ['Start with an empty tally called .', 'Start with an empty tally called Counts.'], + ['Set to .', 'Set Best to i.'], + ] + }, + { + group: 'Changing a value', forms: [ + ['Add to .', 'Add 1 to Count. Add i to Starts.'], + ['Add to the end of .', 'Add the complement of b to the end of R.'], + ['Subtract from .', 'Subtract 1 from Left.'], + ['Count in .', 'Count the window in Counts.'], + ['Swap and .', 'Swap A and B.'], + ] + }, + { + group: 'Loops', forms: [ + ['For each position of :', 'For each position i of T:'], + ['For each letter of :', 'For each letter b of T:'], + ['For each from up to but not including :', 'For each i from 0 up to but not including the length of T minus 1:'], + ['For every third position of :', 'For every third position i of T:'], + ['Repeat while :', 'Repeat while i is less than the length of T:'], + ['Stop the loop. Skip to the next position.', ''], + ] + }, + { + group: 'Tests', forms: [ + ['If , .', 'If the letter of T at i is B, add 1 to Count.'], + ['If : ... Otherwise:', 'If Score is at least 30:'], + [' is , is not, is less than, is at most, is at least, is one of', "the letter is 'G' or 'C'"], + [' is neither nor ', "b is neither 'G' nor 'C'"], + [' is empty / contains ', 'T is empty'], + ['( or ) and ', "(b is 'A' or b is 'T') and i is more than 0"], + ['for every from up to but not including , ', 'for every j from 0 up to but not including the length of P, the letter of P at j is the letter of T at i + j'], + ] + }, + { + group: 'Values', forms: [ + ['the letter of at ', 'the letter of T at i + 1'], + ['the length of ', 'the length of T minus the length of P'], + ['the letters of starting at ', 'the 3 letters of T starting at i'], + ['the number of positions of where ', "the number of positions k of T where the letter of T at k is 'G'"], + ['the count for in ', 'the count for w in Counts'], + ['the larger of and ', 'the larger of Best and Run'], + ] + }, + { + group: 'Defining a phrase', forms: [ + [' when .', 'P matches at i when for every j from 0 up to but not including the length of P, the letter of P at j is the letter of T at i + j.'], + ['The is .', 'The GC count of S is the number of letters of S that are \'G\' or \'C\'.'], + ['The is when , ..., and otherwise.', "The complement of b is 'T' when b is 'A', 'A' when b is 'T', 'G' when b is 'C', and 'C' otherwise."], + ] + }, +]; + +/* ------------------------------------------------------------------ */ +/* Run-time errors, in the English program's words */ +/* ------------------------------------------------------------------ */ + +/** + * A run that stopped, told in the student's words: which English line, + * and the interpreter's explanation with pseudocode names swapped for the + * student's and T[i + 1] read as "position i + 1 of T". + */ +export function englishError(result, err) { + if (!err) return null; + const m = result.lineMap[(err.line ?? 0) - 1]; + const nameOut = (s) => String(s) + .replace(/\b([A-Za-z_][A-Za-z0-9_]*)\[([^\]]+)\]/g, (_, x, i) => `position ${i} of ${x}`) + .replace(/\b[A-Za-z_][A-Za-z0-9_]*\b/g, (w) => result.display.get(w.toUpperCase()) || w) + .replace(/\bTRUE\b/g, 'yes').replace(/\bFALSE\b/g, 'no') + .replace(/\bLENGTH\(([^)]+)\)/g, 'the length of $1'); + let message = nameOut(err.message); + if (err.kind === 'depth') message = 'The steps keep using a phrase that uses itself, more than 400 levels deep, so the run stopped. A phrase defined in terms of itself has to get smaller each time and stop at a simplest case.'; + if (err.kind === 'limit') message = `The run took more than ${err.limit ?? 'the allowed number of'} steps, so it stopped. A "Repeat while" loop ends only when its condition becomes false: check that some step inside it moves toward that.`; + return { line: m?.line ?? null, message, kind: err.kind }; +} + +/** One line of English steps as HTML, for the editor: step words, texts, numbers, comments. */ +export function englishLineHtml(line) { + const esc = (s) => String(s).replace(/&/g, '&').replace(//g, '>'); + const c = line.indexOf('//'); + const code = c >= 0 ? line.slice(0, c) : line; + const comment = c >= 0 ? line.slice(c) : ''; + const lead = /^(\s*)(to|if|otherwise|else|for each|for every|for all|repeat|while|until|start with|start|set|let|make|add|subtract|increase|decrease|count|stop and answer|stop|skip to|show|check that|swap|remove|sort|the answer is|example|put|insert)\b/i; + let html = ''; + const m = lead.exec(code); + let rest = code; + if (m) { html += esc(m[1]) + `${esc(m[2])}`; rest = code.slice(m[0].length); } + html += esc(rest) + .replace(/("|")([^"]*)("|$)/g, '"$2$3') + .replace(/(^|[^A-Za-z0-9])('[^']{1,2}')/g, '$1$2') + .replace(/\b(\d+(?:\.\d+)?)\b/g, '$1') + .replace(/\b(when|otherwise|up to but not including|up to and including|down to and including|down to but not including|for every|for some|there is)\b/gi, '$1') + .replace(/:\s*$/, ':'); + if (comment) html += `${esc(comment)}`; + return html; +} diff --git a/src/engines/pseudo/exercise.js b/src/engines/pseudo/exercise.js new file mode 100644 index 0000000..2b74b0f --- /dev/null +++ b/src/engines/pseudo/exercise.js @@ -0,0 +1,393 @@ +/** + * pseudo/exercise.js + * + * Practice: exercises checked against a model answer, and the activities + * the lessons already use (assemble the lines, predict, fill in the gap), + * rebuilt on the lab so that every answer is checked by running it. + * + * assess(source, exercise) -> { items, headline, study, failing } (no DOM; used by tests) + * exercise(spec) -> node + * parsons(spec) -> node + * predict(spec) -> node + * fillIn(spec) -> node + */ + +import { el, add, clear, button, plural, commas, guideLink } from './dom.js'; +import { lineHtml } from './highlight.js'; +import { editor } from './editor.js'; +import { study } from './study.js'; +import { mountWidget } from './views.js'; +import { parseTests, runTests, checkProperties } from './testing.js'; +import { jobsReport, PATTERN_NAMES } from './plan.js'; +import { verify } from './vcgen.js'; +import { run } from './interp.js'; +import { formatValue, UNSET } from './lang.js'; +import { resolveCall } from './views.js'; + +const up = (s) => String(s).toUpperCase(); + +/* ------------------------------------------------------------------ */ +/* Assessment */ +/* ------------------------------------------------------------------ */ + +/** + * Check an attempt at an exercise, in the order a student should fix + * things: well formed, named right, passes the examples, agrees with the + * model answer everywhere, and does each of the four jobs. The headline is + * the single most useful next step. + * + * item = { id, status: 'pass' | 'fail' | 'open', title, detail, line, call } + */ +export function assess(source, ex, options = {}) { + const items = []; + const s = study(source); + const push = (x) => { items.push(x); return x; }; + const errors = s.diags.filter((d) => d.severity === 'error' && d.code !== 'blank'); + if (s.blanks.length) push({ id: 'blanks', status: 'open', title: `Fill in ${plural(s.blanks.length, 'blank')}`, detail: `Start with <${s.blanks[0].text}> on line ${s.blanks[0].line}.`, line: s.blanks[0].line }); + if (errors.length) { + push({ id: 'form', status: 'fail', title: `Make it well formed: ${plural(errors.length, 'line needs', 'lines need')} attention`, detail: errors.slice(0, 3).map((d) => `Line ${d.line}: ${d.message}`).join(' '), line: errors[0].line, guide: errors[0].guide }); + } else if (!s.blanks.length) push({ id: 'form', status: 'pass', title: 'It is well formed' }); + const ok = s.ok; + const sub = ok ? s.program.subs.get(up(ex.name)) : null; + const ref = study(ex.reference, { coach: false }); + const refSub = ref.program.subs.get(up(ex.name)); + if (ok) { + if (!sub) push({ id: 'name', status: 'fail', title: `Define ${ex.name}`, detail: `The exercise asks for a subroutine called ${ex.name}, and the program defines ${[...s.program.subs.values()].map((x) => x.name).join(', ') || 'none'}.` }); + else if (refSub && sub.params.length !== refSub.params.length) push({ id: 'name', status: 'fail', title: `Give ${ex.name} the right parameters`, detail: `${ex.name} should take ${refSub.params.map((p) => `${p.name} : ${p.type.k}`).join(', ')}.`, line: sub.node.line }); + } + let failing = null; + if (ok && sub && (!refSub || sub.params.length === refSub.params.length)) { + // the examples + const results = runTests(s.program, parseTests(ex.tests)); + const bad = results.filter((r) => r.status !== 'pass'); + if (bad.length) { + const b = bad[0]; + failing = { call: b.source.split('->')[0].trim() }; + push({ id: 'examples', status: 'fail', title: `Pass the examples: ${results.length - bad.length} of ${results.length} pass`, detail: `${b.source.split('->')[0].trim()}: ${b.message}`, line: b.error?.line, call: failing.call }); + } else push({ id: 'examples', status: 'pass', title: `It passes all ${plural(results.length, 'example')}` }); + // the model answer, everywhere + if (!bad.length) { + const p = checkProperties(s.program, ex.name, { reference: ref.program, count: options.count ?? 400, generators: ex.generators }); + if (p.failures.length) { + const f = p.failures[0]; + failing = { call: f.call }; + const what = f.kind === 'mismatch' ? `${f.call} ${f.message}.` : `${f.call} stops with an error: ${f.message}`; + push({ id: 'model', status: 'fail', title: 'Agree with the model answer on every input', detail: `${what}${f.original !== f.call ? ` (The first input to go wrong was ${f.original}; this is the smallest version of it.)` : ''}`, line: f.line, call: f.call }); + } else push({ id: 'model', status: 'pass', title: `It agrees with the model answer on ${commas(p.checked)} generated inputs` }); + } + // contracts the student wrote + const wrote = sub.requires.length + sub.ensures.length; + if (wrote || source.includes('INVARIANT')) { + const v = verify(s.program); + const u = v.units.find((x) => up(x.name) === up(ex.name)); + const open = u.obligations.filter((o) => o.status !== 'proved'); + push({ id: 'proof', status: open.length ? 'open' : 'pass', title: open.length ? `The checker proves ${u.obligations.length - open.length} of ${u.obligations.length} claims` : `The checker proves all ${u.obligations.length} of its claims`, detail: open.length ? `Open: line ${open[0].line}, ${open[0].label}.` : '' }); + } + } + // the four jobs + const unit = s.analysis?.units.find((x) => up(x.name) === up(ex.name)); + if (unit) { + const jobs = jobsReport(unit, s.diags); + const missing = jobs.filter((j) => j.status !== 'ok'); + if (ex.pattern && unit.pattern && unit.pattern !== ex.pattern && !(ex.pattern === 'count' && unit.pattern === 'sum')) { + push({ id: 'pattern', status: 'open', title: `This task is a ${PATTERN_NAMES[ex.pattern]} scan`, detail: `The program reads as ${PATTERN_NAMES[unit.pattern] || 'something else'}. That can still be right; if a check above fails, compare the plan for ${PATTERN_NAMES[ex.pattern]}.` }); + } + if (missing.length) { + const m = missing[0]; + push({ id: 'jobs', status: 'open', title: `The four jobs: ${m.job === 'setup' ? 'set up' : m.job} ${m.status === 'missing' ? 'is missing' : 'needs a look'}`, detail: m.status === 'missing' ? m.message : `Line ${m.issues[0].line}: ${m.issues[0].message}`, line: m.issues[0]?.line }); + } else push({ id: 'jobs', status: 'pass', title: 'It does all four jobs: set up, visit, test, update, and then reports' }); + } + const coachNotes = s.diags.filter((d) => d.source === 'coach' && d.severity === 'warning'); + // The pattern and the four jobs are guidance: they matter while something fails. + if (!items.some((x) => x.status === 'fail') && !s.blanks.length) { + for (let i = items.length - 1; i >= 0; i--) if ((items[i].id === 'pattern' || items[i].id === 'jobs') && items[i].status === 'open') items.splice(i, 1); + } + const firstFail = items.find((x) => x.status === 'fail') || items.find((x) => x.status === 'open'); + let headline; + if (!firstFail || (firstFail.status === 'open' && firstFail.id === 'proof')) headline = 'Done: every check passes.'; + else if (firstFail.id === 'form') headline = firstFail.detail.split(' Line ')[0]; + else if ((firstFail.id === 'examples' || firstFail.id === 'model') && coachNotes.length) headline = `${firstFail.detail} The coach may know why. Line ${coachNotes[0].line}: ${coachNotes[0].message}`; + else headline = firstFail.detail || firstFail.title; + return { items, headline, study: s, failing, coach: coachNotes, done: !items.some((x) => x.status === 'fail') && !s.blanks.length && ok }; +} + +/* ------------------------------------------------------------------ */ +/* An exercise */ +/* ------------------------------------------------------------------ */ + +export function exercise(ex, options = {}) { + const root = el('div', { class: 'pl pl-exercise' }); + if (ex.title) add(root, el('h3', { class: 'pl-title', text: ex.title })); + add(root, el('div', { class: 'pl-ex-prompt' }, (ex.prompt || []).map((p) => el('p', { text: p })))); + const key = `pseudo-ex-${ex.name}`; + let saved = null; + try { saved = localStorage.getItem(key); } catch { /* fine */ } + const ed = editor({ value: saved || ex.starter, onChange: (v) => { live(v); try { localStorage.setItem(key, v); } catch { /* fine */ } }, onRun: () => check() }); + const result = el('div', { class: 'pl-ex-result' }); + const hintsEl = el('div', { class: 'pl-hints' }); + const stepHost = el('div'); + let hintN = 0; + const hintBtn = button('A hint', () => { + if (hintN < (ex.hints || []).length) { + add(hintsEl, el('details', { open: true }, [el('summary', { text: `Hint ${hintN + 1}` }), el('p', { text: ex.hints[hintN] })])); + hintN++; + if (hintN >= ex.hints.length) hintBtn.textContent = 'Show a model answer'; + return; + } + add(hintsEl, el('details', { open: true }, [el('summary', { text: 'A model answer' }), el('p', { class: 'pl-muted', text: 'One answer among several that work. Compare it with yours line by line; the four jobs are marked.' }), el('pre', { class: 'pl-console', html: ex.reference.split('\n').map(lineHtml).join('\n') })])); + hintBtn.disabled = true; + }); + const status = el('p', { class: 'pl-status', role: 'status' }); + function live(v) { const s = study(v); ed.setDiagnostics(s.diags); } + function check() { + const a = assess(ed.value, ex, options); + ed.setDiagnostics(a.study.diags); + clear(result); + status.textContent = a.headline; + status.className = `pl-status ${a.done ? 'is-ok' : ''}`; + const list = el('ul', { class: 'pl-checklist' }); + for (const it of a.items) { + add(list, el('li', { class: `is-${it.status}` }, [ + el('span', { text: it.status === 'pass' ? '✓' : it.status === 'fail' ? '✖' : '–', 'aria-label': it.status }), + el('div', {}, [el('strong', { text: it.title }), it.detail ? el('p', { class: 'pl-muted', text: it.detail }) : null, it.guide ? guideLink(it.guide, 'What the guide says') : null, + it.call ? button('Step through this call', () => stepThrough(it.call)) : null]), + ])); + } + add(result, list); + if (a.coach.length) add(result, el('details', {}, [el('summary', { text: `The coach noticed ${plural(a.coach.length, 'thing')}` }), ...a.coach.map((d) => el('p', { text: `Line ${d.line}: ${d.message}` }))])); + if (a.failing?.call) stepThrough(a.failing.call, true); + else clear(stepHost); + } + function stepThrough(call, quiet) { + clear(stepHost); + const s = study(ed.value); + if (!s.ok) return; + mountWidget(stepHost, { code: ed.value, study: s, call, title: `Stepping through ${call}`, views: ['code', 'variables', 'explain', 'trace'] }); + if (!quiet) stepHost.scrollIntoView?.({ behavior: 'smooth', block: 'nearest' }); + } + add(root, [ + ed.node, + el('div', { class: 'pl-row' }, [button('Check', check, { primary: true, title: 'Ctrl+Enter' }), hintBtn, button('Start again', () => { if (confirmReset()) { ed.value = ex.starter; live(ex.starter); clear(result); status.textContent = ''; } })]), + status, result, hintsEl, stepHost, + ]); + live(ed.value); + return root; +} +function confirmReset() { try { return typeof window === 'undefined' || window.confirm('Start again from the original code? Your changes will be lost.'); } catch { return true; } } + +/* ------------------------------------------------------------------ */ +/* Assemble the lines */ +/* ------------------------------------------------------------------ */ + +/** + * The lines of a working program, shuffled, with a few that do not belong. + * Click a line to move it between the pool and the program; drag, or use + * the arrow buttons, to reorder. Checking runs what was built, so any + * working order passes. + */ +export function parsons(spec) { + const root = el('div', { class: 'pl pl-parsons-widget' }); + if (spec.title) add(root, el('h3', { class: 'pl-title', text: spec.title })); + if (spec.prompt) add(root, el('div', { class: 'pl-ex-prompt' }, [el('p', { text: spec.prompt })])); + const all = [...spec.lines.map((t, i) => ({ id: `l${i}`, text: t })), ...(spec.distractors || []).map((t, i) => ({ id: `d${i}`, text: t, distractor: true }))]; + const seed = spec.seed ?? 7; + let x = seed; + const rnd = () => { x = (x * 16807) % 2147483647; return x / 2147483647; }; + const pool = all.slice().sort(() => rnd() - 0.5); + const placed = []; + const poolEl = el('div', { class: 'pl-parsons-list', 'aria-label': 'lines to use' }); + const placedEl = el('div', { class: 'pl-parsons-list', 'aria-label': 'your program' }); + const feedback = el('div'); + let dragging = null; + function lineButton(item, where) { + const b = el('button', { type: 'button', class: 'pl-parsons-line', draggable: 'true', title: where === 'pool' ? 'Add this line to your program' : 'Take this line out' }, [el('code', { html: lineHtml(item.text) })]); + b.addEventListener('click', () => { move(item, where === 'pool' ? 'placed' : 'pool'); }); + b.addEventListener('dragstart', () => { dragging = item; b.classList.add('is-dragging'); }); + b.addEventListener('dragend', () => { dragging = null; b.classList.remove('is-dragging'); }); + if (where === 'placed') { + b.addEventListener('dragover', (e) => e.preventDefault()); + b.addEventListener('drop', (e) => { e.preventDefault(); if (!dragging) return; const from = placed.indexOf(dragging); const to = placed.indexOf(item); if (from >= 0) placed.splice(from, 1); else pool.splice(pool.indexOf(dragging), 1); placed.splice(to, 0, dragging); draw(); }); + b.addEventListener('keydown', (e) => { + const i = placed.indexOf(item); + if (e.key === 'ArrowUp' && i > 0) { [placed[i - 1], placed[i]] = [placed[i], placed[i - 1]]; draw(); placedEl.children[i - 1]?.focus(); e.preventDefault(); } + if (e.key === 'ArrowDown' && i < placed.length - 1) { [placed[i + 1], placed[i]] = [placed[i], placed[i + 1]]; draw(); placedEl.children[i + 1]?.focus(); e.preventDefault(); } + }); + } + return b; + } + function move(item, to) { + if (to === 'placed') { pool.splice(pool.indexOf(item), 1); placed.push(item); } else { placed.splice(placed.indexOf(item), 1); pool.push(item); } + draw(); + } + placedEl.addEventListener('dragover', (e) => e.preventDefault()); + placedEl.addEventListener('drop', (e) => { if (e.target === placedEl && dragging && pool.includes(dragging)) { e.preventDefault(); move(dragging, 'placed'); } }); + function draw() { + clear(poolEl); clear(placedEl); + for (const it of pool) poolEl.appendChild(lineButton(it, 'pool')); + for (const it of placed) placedEl.appendChild(lineButton(it, 'placed')); + if (!placed.length) add(placedEl, el('p', { class: 'pl-muted', text: 'Click lines on the left to build the program here, in order. Drag, or use the up and down arrow keys, to reorder.' })); + } + function check() { + clear(feedback); + const code = placed.map((p) => p.text).join('\n'); + const s = study(code); + const out = el('div'); + if (!placed.length) { add(out, el('p', { class: 'pl-muted', text: 'Nothing placed yet.' })); add(feedback, out); return; } + if (!s.ok) { + const d = s.errors[0]; + add(out, el('p', { class: 'pl-warn-line', text: `Not a complete program yet. Line ${d.line}: ${d.message}` })); + } else { + const results = runTests(s.program, parseTests(spec.tests)); + const bad = results.filter((r) => r.status !== 'pass'); + let proper = !bad.length; + if (proper && spec.reference) { + const ref = study(spec.reference, { coach: false }); + const p = checkProperties(s.program, spec.name, { reference: ref.program, count: 200 }); + if (p.failures.length) { proper = false; add(out, el('p', { class: 'pl-warn-line', text: `It passes the examples, and ${p.failures[0].call} ${p.failures[0].kind === 'mismatch' ? p.failures[0].message : `stops: ${p.failures[0].message}`}.` })); } + } + if (bad.length) add(out, el('p', { class: 'pl-warn-line', text: `${results.length - bad.length} of ${results.length} tests pass. ${bad[0].source.split('->')[0].trim()}: ${bad[0].message}` })); + const used = placed.filter((p) => p.distractor); + if (proper) add(out, el('p', { class: 'pl-ok-line', text: `It works: every test passes${spec.reference ? ', and it agrees with the model answer on every generated input' : ''}. Read it once more top to bottom: each line depends only on lines above it.` })); + else { + const warn = s.diags.find((d) => d.source === 'coach' && d.severity === 'warning'); + if (warn) add(out, el('p', { class: 'pl-muted', text: `The coach suggests where to look: line ${warn.line}, ${warn.message}` })); + if (used.length) add(out, el('p', { class: 'pl-muted', text: `${used.length === 1 ? 'One line you used is' : 'Some lines you used are'} not needed in a working answer.` })); + } + } + add(feedback, out); + } + add(root, [ + el('div', { class: 'pl-parsons' }, [el('div', {}, [el('p', { class: 'pl-label', text: 'Lines' }), poolEl]), el('div', {}, [el('p', { class: 'pl-label', text: 'Your program' }), placedEl])]), + el('div', { class: 'pl-row' }, [button('Check', check, { primary: true }), button('Start again', () => { pool.push(...placed.splice(0)); draw(); clear(feedback); })]), + feedback, + ]); + draw(); + return root; +} + +/* ------------------------------------------------------------------ */ +/* Predict */ +/* ------------------------------------------------------------------ */ + +/** + * Two kinds of prediction. With `calls`, predict what each call returns. + * With `watch`, fill in a trace table by hand, the way the guide does, + * one row per step, then check each cell against the run. + */ +export function predict(spec) { + const root = el('div', { class: 'pl pl-predict' }); + if (spec.title) add(root, el('h3', { class: 'pl-title', text: spec.title })); + if (spec.prompt) add(root, el('div', { class: 'pl-ex-prompt' }, [el('p', { text: spec.prompt })])); + const s = study(spec.code); + add(root, el('pre', { class: 'pl-console', html: spec.code.split('\n').map((l, i) => `${String(i + 1).padStart(2)} ${lineHtml(l)}`).join('\n') })); + const feedback = el('div'); + const stepHost = el('div'); + if (!s.ok) { add(root, el('p', { class: 'pl-error', text: s.errors[0].message })); return root; } + if (spec.calls) { + const table = el('table', { class: 'pl-test-table pl-predict-table' }); + add(table, el('tr', {}, ['call', 'your prediction', 'it gives'].map((h) => el('th', { text: h })))); + const rows = spec.calls.map((c) => { + const input = el('input', { class: 'pl-input', type: 'text', 'aria-label': `prediction for ${c}` }); + const got = el('td'); + const tr = el('tr', {}, [el('td', {}, [el('code', { text: c })]), el('td', {}, [input]), got]); + add(table, tr); + return { c, input, got, tr }; + }); + const checkBtn = button('Check my predictions', () => { + let right = 0; + for (const r of rows) { + const call = resolveCall(s.program, r.c); + const t = run(s.program, { call, record: false }); + const sub = s.program.subs.get(up(call.name)); + const shown = t.error ? `an error: ${t.error.message}` : formatValue(t.result, sub.returns); + r.got.textContent = shown; + const guess = r.input.value.trim().replace(/^"(.*)"$/, '$1'); + const ok = !t.error && (guess === shown || guess === String(t.result) || (Array.isArray(t.result) && guess.replace(/\s+/g, '') === shown.replace(/\s+/g, ''))); + r.got.className = ok ? 'is-right' : 'is-wrong'; + if (ok) right++; + } + clear(feedback); + add(feedback, el('p', { class: right === rows.length ? 'pl-ok-line' : 'pl-warn-line', text: right === rows.length ? 'Every prediction was right.' : `${right} of ${rows.length} right. For the others, step through the call and find the first step where the run and your picture of it part ways.` })); + }, { primary: true }); + add(root, [table, el('div', { class: 'pl-row' }, [checkBtn, button('Step through the first call', () => mountWidget(stepHost, { code: spec.code, study: s, call: spec.calls[0], views: ['code', 'variables', 'explain', 'trace'] }))]), feedback, stepHost]); + return root; + } + // a trace table to fill in + const call = spec.call ? resolveCall(s.program, spec.call) : null; + const t = run(s.program, { call, input: spec.input || [], contracts: false }); + const kinds = new Set(spec.kinds || ['assign', 'next', 'loop-start']); + const watch = spec.watch.map(up); + const valueIn = (state, k) => { for (let i = state.frames.length - 1; i >= 0; i--) { const v = state.frames[i].vars[k]; if (v) return v; } return null; }; + const text = (v) => (!v || v.value === UNSET ? '' : formatValue(v.value, v.type, { bareChar: true })); + const rows = []; + t.steps.forEach((st, i) => { + if (!kinds.has(st.kind) || rows.length >= (spec.maxRows ?? 20)) return; + const before = t.states[i]; const after = t.states[i + 1]; + const cells = watch.map((w) => { const a = text(valueIn(after, w)); const b = text(valueIn(before, w)); return { expected: a, changed: a !== b }; }); + if (!cells.some((c) => c.changed)) return; + rows.push({ i, line: st.line, cells }); + }); + const table = el('table', { class: 'pl-trace pl-predict-table' }); + add(table, el('tr', {}, [el('th', { text: 'line' }), ...spec.watch.map((w) => el('th', {}, [el('code', { text: w })]))])); + const inputs = []; + for (const r of rows) { + const tr = el('tr', {}, [el('td', { text: String(r.line) })]); + r.cells.forEach((c) => { + const td = el('td'); + if (c.changed) { const inp = el('input', { class: 'pl-input', type: 'text', 'aria-label': `value after line ${r.line}` }); add(td, inp); inputs.push({ inp, td, c }); } + tr.appendChild(td); + }); + table.appendChild(tr); + } + const checkBtn = button('Check my table', () => { + let right = 0; + for (const x of inputs) { + const g = x.inp.value.trim(); + const ok = g === x.c.expected || `'${g}'` === x.c.expected || `"${g}"` === x.c.expected || g.toUpperCase() === x.c.expected; + x.td.className = ok ? 'is-right' : 'is-wrong'; + x.inp.title = ok ? 'right' : `the run says ${x.c.expected}`; + if (ok) right++; + } + clear(feedback); + add(feedback, el('p', { class: right === inputs.length ? 'pl-ok-line' : 'pl-warn-line', text: right === inputs.length ? 'Every cell is right.' : `${right} of ${inputs.length} cells right. Hover over a red cell to see the run's value, then step through to the first one.` })); + }, { primary: true }); + add(root, [el('p', { class: 'pl-muted', text: `Calling ${spec.call}. Each row is one step that changes a watched name; write the new value in the cell that changes.` }), table, + el('div', { class: 'pl-row' }, [checkBtn, button('Step through it', () => mountWidget(stepHost, { code: spec.code, study: s, call: spec.call, views: ['code', 'variables', 'explain', ['trace', { watch: spec.watch }]] }))]), feedback, stepHost]); + return root; +} + +/* ------------------------------------------------------------------ */ +/* Fill in the gap */ +/* ------------------------------------------------------------------ */ + +/** A program with blanks; each blank becomes a text box, and checking runs the result. */ +export function fillIn(spec) { + const root = el('div', { class: 'pl pl-fillin' }); + if (spec.title) add(root, el('h3', { class: 'pl-title', text: spec.title })); + if (spec.prompt) add(root, el('div', { class: 'pl-ex-prompt' }, [el('p', { text: spec.prompt })])); + const lines = spec.code.split('\n'); + const inputs = []; + const listing = el('pre', { class: 'pl-console' }); + lines.forEach((l, i) => { + const parts = l.split(/(<[A-Za-z][A-Za-z0-9 _'.-]*>)/); + const row = el('span', {}, [el('span', { class: 'pl-code-num', text: `${String(i + 1).padStart(2)} ` })]); + for (const p of parts) { + if (/^<.*>$/.test(p)) { const inp = el('input', { class: 'pl-blank-input', type: 'text', placeholder: p.slice(1, -1), 'aria-label': p.slice(1, -1), size: String(Math.max(8, p.length)) }); inputs.push(inp); add(row, inp); } + else add(row, el('span', { html: lineHtml(p) })); + } + add(listing, [row, '\n']); + }); + const feedback = el('div'); + const stepHost = el('div'); + const assemble = () => { let k = 0; return spec.code.replace(/<[A-Za-z][A-Za-z0-9 _'.-]*>/g, (m) => inputs[k++].value.trim() || m); }; + const check = () => { + clear(feedback); clear(stepHost); + const code = assemble(); + const a = assess(code, { name: spec.name, tests: spec.tests, reference: spec.reference }, { count: 250 }); + const first = a.items.find((x) => x.status !== 'pass'); + add(feedback, el('p', { class: a.done ? 'pl-ok-line' : 'pl-warn-line', text: a.done ? 'It works: every test passes and it agrees with the model answer everywhere.' : a.headline })); + if (first?.call) mountWidget(stepHost, { code, call: first.call, views: ['code', 'variables', 'explain'] }); + }; + add(root, [listing, el('div', { class: 'pl-row' }, [button('Check', check, { primary: true })]), feedback, stepHost]); + return root; +} diff --git a/src/engines/pseudo/highlight.js b/src/engines/pseudo/highlight.js new file mode 100644 index 0000000..b9f603f --- /dev/null +++ b/src/engines/pseudo/highlight.js @@ -0,0 +1,49 @@ +/** + * pseudo/highlight.js + * + * Color a line of pseudocode for display, using the same five classes the + * site's code blocks use (see css/syntax/pseudo.css): control flow, declarations, + * operators written as words, input and output, and types. + */ + +import { KEYWORD_CLASS, BUILTINS } from './lang.js'; + +const TOKEN = /(\/\/.*$)|("(?:[^"\\]|\\.)*"?)|('(?:[^'\\]|\\.)*'?)|(<-|←|==|!=|<=|>=)|(<[A-Za-z][A-Za-z0-9 _'.-]*>)|(\b[0-9]+(?:\.[0-9]+)?\b)|([A-Za-z_][A-Za-z0-9_]*)|(\s+)|(.)/g; + +/** [{ text, cls }] for one line. */ +export function tokens(line) { + const out = []; + let m; + TOKEN.lastIndex = 0; + let prevWord = ''; + while ((m = TOKEN.exec(line))) { + const [t, comment, str, chr, arrow, blank, number, word] = m; + let cls = ''; + if (comment) cls = 'co'; + else if (str) cls = 'st'; + else if (chr) cls = 'ch'; + else if (arrow) cls = arrow === '<-' || arrow === '←' ? 'ar' : 'op'; + else if (blank) cls = 'bl'; + else if (number) cls = 'nu'; + else if (word) { + const k = KEYWORD_CLASS[word]; + if (k) cls = `kw-${k}`; + else if ((word === 'EACH' && prevWord === 'FOR') || word === 'RESULT' || word === 'OLD' || word === 'SUM') cls = word === 'EACH' ? 'kw-flow' : 'kw-proof'; + else if (BUILTINS[word]) cls = 'bu'; + else if (/^[A-Z][A-Z0-9_]*$/.test(word) && word.length > 1 && /\(/.test(line.slice(m.index + word.length).trimStart()[0] || '')) cls = 'fn'; + else cls = 'nm'; + prevWord = word; + } + out.push({ text: t, cls }); + } + return out; +} + +/** HTML for one line, escaped. */ +export function lineHtml(line) { + return tokens(line).map((t) => (t.cls ? `${esc(t.text)}` : esc(t.text))).join(''); +} + +export function esc(s) { + return String(s).replace(/&/g, '&').replace(//g, '>'); +} diff --git a/src/engines/pseudo/interp.ts b/src/engines/pseudo/interp.ts new file mode 100644 index 0000000..8b5a9db --- /dev/null +++ b/src/engines/pseudo/interp.ts @@ -0,0 +1,2016 @@ +/** + * pseudo/interp.ts + * + * Runs a checked program and records what every step did. The record is + * the same thing a student builds on paper when tracing by hand: which line + * ran, what it read, what it changed, and every variable's value afterwards. + * + * run(program, options) -> trace + * + * options = { + * call: { name, args: [values] } call one subroutine instead of the main program + * input: ['7', 'GATTACA'] what INPUT reads, one line at a time + * maxSteps: 100000 stop a program that may never end + * record: true keep a snapshot after every step + * maxRecorded: 10000000 values the snapshots may copy (see below) + * contracts: true check REQUIRE, ENSURE, INVARIANT, DECREASES, ASSERT + * seed: 1 RAND and RANDINT give the same numbers every run + * shuffle: false visit SET members and HASHMAP keys in a random order + * } + * + * trace = { + * program, steps, states, output, result, resultType, error, count, + * truncated, work: { steps, compares, letterCompares, reads, writes, calls }, hits, + * unrecorded: null, or { after, values } when recording stopped + * } + * + * states[t] is the state before steps[t]; states[count] is the final state. + * A step is { line, kind, text, reads, writes, depth, sub, outcome }. + * + * A snapshot copies each variable a step changed, so a large collection that + * changes at every step is copied at every step. Once the copies come to more + * than `maxRecorded` values, the trace ends with a step of kind 'unrecorded' + * that says so, and the run goes on without recording: its output, result, + * stop and line counts are those of the whole run. + */ + +import { + T, UNSET, PArray, PRecord, cloneValue, chars, textLength, sortedKeys, compareValues, + valuesEqual, formatValue, outputText, typeName, isNumeric, + SIZED, isSizedInt, isFloat32, intRange, intRangeText, EXACT_MAX, knownDims, +} from './lang.js'; +import { sourceOf } from './parse.js'; +import { float32OfDecimal, fma, exactDecimal, nextAfter, ulp, copySign, gcd, bitOp, wrap, ieeePow } from './numeric.js'; +import type { + PType, IntType, Loc, Program, Sub, Value, RangeValue, ErrorKind, Access, Step, StepKind, Slot, State, VarSnapshot, RunCall, + RunOptions, Work, Expr, NameExpr, BinaryExpr, IndexExpr, CallExprNode, Stmt, LoopStmt, ForStmt, ForEachStmt, WhileStmt, + RepeatStmt, IfStmt, CaseStmt, InputStmt, ContractStmt, CallStmt, BuiltinCallStmt, SubStmt, IfBranch, CaseArm, +} from './ast.js'; + +declare module './lang.js' { + interface PArray { + /** Fill every position with what a fresh value of `of` holds (added to PArray below, by this module). */ + fillWith(of: PType | undefined, interp: { evaluate(e: Expr): Value }, at: { line?: number | null; col?: number | null } | null | undefined): PArray; + } +} + +/** A place a stop or a step is reported at: a node, or a line (and column) put together for it. */ +type Place = { line?: number | null; col?: number | null } | null | undefined; +/** + * Where a value is going, for the messages of fitNumber and fitInt: the statement or call (`target` names what + * receives the value, sometimes a name made up for the message), or the expression that computed it. + */ +type FitAt = { line?: number | null; col?: number | null; kind?: string; target?: Expr | { kind: string; name: string } | null } | null | undefined; +/** What checkPosition reports about: an index expression, a target, or a built-in's call. */ +type PositionAt = Loc & { kind: string; target?: Expr | null; indices?: Expr[]; args?: Expr[] }; +/** An expression read field by field, as the walkers below read any node (the prover passes its own nodes too). */ +interface LooseExpr { + kind: string; name?: string; var?: string; isOld?: boolean; builtin?: string; type?: PType; + left?: LooseExpr; right?: LooseExpr; arg?: LooseExpr; target?: LooseExpr | null; lo?: LooseExpr; hi?: LooseExpr; domain?: LooseExpr; body?: LooseExpr; + operands?: LooseExpr[]; indices?: LooseExpr[]; args?: LooseExpr[]; items?: LooseExpr[]; entries?: Array<{ key: LooseExpr; value: LooseExpr }>; +} +/** A statement read for the blocks it holds (PARALLEL FOR looks for the names its body declares). */ +interface LooseStmt { body?: Stmt[]; elseBody?: Stmt[] | null; branches?: IfBranch[]; arms?: CaseArm[] } +/** An argument on its way into a call: a BYREF parameter's slot, or a value. */ +type ArgSlot = { byref: Slot; value?: undefined } | { value: Value; byref?: undefined; name?: string; type?: PType }; +/** Where a call is made from: a CALL line, a call in an expression, or the routine itself when a run starts with it. */ +type CallSite = CallStmt | CallExprNode | SubStmt; +/** A loop in progress, and what the step limit's message says about it. */ +interface LoopState { node: LoopStmt; pass: number; same: number; stuck: string[] | null; lastKey: string | null } +/** A data race PARALLEL FOR found: a location two passes touched, one of them writing. */ +interface Race { key: string; pass: number; other: number; what: 'reads' | 'writes' } +interface RaceTrack { setPass(p: number): void; note(reads: Access[], writes: Access[]): void } +/** The options a run goes by, with every default filled in. */ +type Opts = RunOptions & { maxSteps: number; record: boolean; contracts: boolean; seed: number; shuffle: boolean; input: unknown[]; maxDepth: number; maxRecorded: number }; +/** What a run gives back: its steps and the state before each (when recorded), its output and answer, and how it stopped. */ +export interface Trace { + program: Program; + steps: Step[]; + states: State[]; + count: number; + output: string[]; + result: Value; + resultType: PType | null | undefined; + error: PseudoError | null; + truncated: boolean; + work: Work; + /** Positions visited by FORALL, EXISTS, SUM and sets written by rule (not steps of the program). */ + visits: number; + hits: Map; + call: RunCall | null; + unrecorded: { after: number; values: number } | null; +} + +/** What a PseudoError may carry besides its kind, message and place. */ +export interface PseudoErrorExtra { + /** The half of a contract that failed: an INVARIANT before the first pass or after one, a DECREASES below 0 or not smaller. */ + phase: 'init' | 'keep' | 'bound' | 'step'; + // typed-port: `name` is also Error's own field: an unset read's error is named after the variable (String(error) + // reads "Total: Total was declared ...", not "Error: ..."). + /** The variable a read found unset. */ + name: string; + index: Value; length: number; key: Value; type: string; result: Value; passes: number[]; location: string; library: string; libraryLine: number | null; +} + +/* ------------------------------------------------------------------ */ +/* Control flow and errors */ +/* ------------------------------------------------------------------ */ + +class BreakSignal { declare line: number; constructor(line: number) { this.line = line; } } +class ContinueSignal { declare line: number; constructor(line: number) { this.line = line; } } +class ReturnSignal { declare value: Value; declare line: number; constructor(value: Value, line: number) { this.value = value; this.line = line; } } +class StopSignal { } + +/** + * A runtime error that teaches. `kind` names the rule, for the coach and + * for tests: 'index', 'unset', 'key', 'divide', 'overflow', 'input', + * 'require', 'ensure', 'invariant', 'decreases', 'assert', 'limit', + * 'depth', 'value', 'return'. + */ +export class PseudoError extends Error { + declare kind: ErrorKind; + declare line: number | null; + declare col: number | null; + declare phase?: PseudoErrorExtra['phase']; + declare index?: Value; + declare length?: number; + declare key?: Value; + declare type?: string; + declare result?: Value; + declare passes?: number[]; + declare location?: string; + declare library?: string; + declare libraryLine?: number | null; + constructor(kind: ErrorKind, message: string, node?: Place, extra: Partial = {}) { + super(message); + this.kind = kind; + this.line = node?.line ?? null; + this.col = node?.col ?? null; + Object.assign(this, extra); + } +} + +const up = (s: unknown): string => String(s).toUpperCase(); +const plural = (n: number, w: string, many = `${w}s`): string => `${n} ${n === 1 ? w : many}`; + +/** + * The most values one collection may be given at once (a declared ARRAY, FILL, a range visited by FOR EACH or a + * quantifier), and the longest text. Past these the run stops with a sentence, where the engine underneath would + * run out of memory or throw an error of its own (tests/pseudo-robustness.test.mjs). + */ +const MAX_VALUES = 2000000; +/** Values a recording's snapshots may copy, about 80 MB of numbers (see `maxRecorded`). */ +const MAX_RECORDED = 10000000; + +/** + * The number a text spells in decimal: an optional sign, digits with an optional point, an optional exponent + * ("12", "-0.5", ".5", "6.02e23"); with USE IEEE also "Infinity" and "-Infinity". Anything else is null: + * JavaScript's Number() would also read "0x10", "0b11" and "" (as 0). + */ +function decimalText(s: string, ieee: boolean): number | null { + if (/^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/.test(s)) return Number(s); + if (ieee && /^[+-]?Infinity$/.test(s)) return Number(s); + return null; +} + +/** The FLOAT32 nearest a decimal text, rounded once (as a literal is), with its sign. */ +function float32OfText(s: string): number { + const neg = s.startsWith('-'); + const body = s.replace(/^[+-]/, '').replace(/^\./, '0.').replace(/\.(?=[eE]|$)/, ''); + return (neg ? -1 : 1) * float32OfDecimal(body); +} + +/** How many values a snapshot copies for `v`: one for a single value, and every value inside a collection. */ +function valueCount(v: Value): number { + if (v === null || typeof v !== 'object') return 1; + let n = 1; + const add = (x: Value) => { n += x !== null && typeof x === 'object' ? valueCount(x) : 1; }; + if (Array.isArray(v)) for (const x of v) add(x); + else if (v instanceof PArray) for (const x of v.data) add(x); + else if (v instanceof Map) for (const [k, x] of v) { add(k); add(x); } + else if (v instanceof Set) n += v.size; + else if (v instanceof PRecord) for (const x of v.fields.values()) add(x); + return n; +} +const MAX_TEXT = 10000000; + +/** A deterministic generator, so a run can be repeated exactly. */ +export function makeRandom(seed = 1): () => number { + let a = seed >>> 0 || 1; + return () => { + a = (a + 0x6D2B79F5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/* ------------------------------------------------------------------ */ +/* Values by type */ +/* ------------------------------------------------------------------ */ + +/** What a freshly declared name holds: collections start empty, everything else unset. */ +function initialValue(type: PType | undefined, interp: { evaluate(e: Expr): Value }, at: Place): Value { + if (!type) return UNSET; + switch (type.k) { + case 'LIST': return []; + case 'HASHMAP': return new Map(); + case 'SET': return new Set(); + case 'ARRAY': { + const dims = (type.dimExprs || []).map((d) => { + const n = interp.evaluate(d) as number; // the checker holds an ARRAY's lengths to whole numbers + if (!Number.isInteger(n) || n < 0) throw new PseudoError('value', `An ARRAY's length has to be a whole number of 0 or more, and ${sourceOf(d)} is ${formatValue(n)}.`, at); + return n; + }); + if (dims.reduce((p, n) => p * n, 1) > MAX_VALUES) { + throw new PseudoError('value', `An ARRAY of ${dims.join(' by ')} positions is more than this lab gives one collection (${MAX_VALUES.toLocaleString('en-US')}).`, at); + } + return new PArray(dims.length ? dims : [0], type.of && ['LIST', 'HASHMAP', 'SET', 'RECORD', 'ARRAY'].includes(type.of.k) ? null : UNSET) + .fillWith(type.of, interp, at); + } + case 'RECORD': { + const r = new PRecord(type.name, type.fieldNames); + for (const f of type.fieldNames) r.fields.set(up(f), initialValue(type.fields.get(up(f)), interp, at)); + return r; + } + default: return UNSET; + } +} +PArray.prototype.fillWith = function fillWith(this: PArray, of: PType | undefined, interp: { evaluate(e: Expr): Value }, at: Place): PArray { + if (of && ['LIST', 'HASHMAP', 'SET', 'RECORD'].includes(of.k)) for (let i = 0; i < this.data.length; i++) this.data[i] = initialValue(of, interp, at); + else this.data.fill(UNSET); + return this; +}; + +/** Does a runtime value belong to a type? Used for arguments supplied from outside. */ +/** + * Whether a value from outside the program (a call's argument) is one of `type`. With `finite`, a FLOAT is a + * finite number, as a run holds it without USE IEEE. + */ +export function valueFits(type: PType | null | undefined, v: unknown, finite = false): boolean { + if (!type || type.k === 'ANY') return true; + const fits = (t: PType, x: unknown) => valueFits(t, x, finite); + switch (type.k) { + case 'INTEGER': { if (!Number.isInteger(v)) return false; const [lo, hi] = intRange(type); return (v as number) >= lo && (v as number) <= hi; } + case 'FLOAT': return typeof v === 'number' && (!finite || Number.isFinite(v)) && (!isFloat32(type) || Math.fround(v) === v || !Number.isFinite(v)); + case 'CHAR': return typeof v === 'string' && textLength(v) === 1; + case 'STRING': return typeof v === 'string'; + case 'BOOL': return typeof v === 'boolean'; + case 'LIST': return Array.isArray(v) && v.every((x) => fits(type.of, x)); + case 'ARRAY': return (v instanceof PArray || Array.isArray(v)) && (v instanceof PArray ? v.data : v).every((x) => x === UNSET || fits(type.of, x)); + case 'SET': return v instanceof Set && [...v].every((x) => fits(type.of, x)); + case 'HASHMAP': return v instanceof Map && [...v].every(([k, x]) => fits(type.key, k) && fits(type.val, x)); + case 'RECORD': return v instanceof PRecord && up(v.typeName) === up(type.name); + default: return true; + } +} + +/* ------------------------------------------------------------------ */ +/* The interpreter */ +/* ------------------------------------------------------------------ */ + +let frameIds = 0; + +/** + * A frame's names. Besides being a map from name to slot, it keeps two things up to date so that a step costs + * the same however deep the calls are: `homes`, the interpreter's record of every frame and name a slot is + * reachable under (a BYREF parameter shares its caller's slot), and the frame's `touched` flag, set whenever a + * name is added, removed or pointed at another slot since the last snapshot. + */ +class Vars extends Map { + declare frame: Frame; + declare homes: WeakMap> | undefined; + constructor(frame: Frame, homes?: WeakMap>) { super(); this.frame = frame; this.homes = homes; } + override set(k: string, slot: Slot): this { + const prior = super.get(k); + if (prior !== slot) { + if (prior) this.unhome(prior, k); + if (this.homes) { const h = this.homes.get(slot); if (h) h.push([this.frame, k]); else this.homes.set(slot, [[this.frame, k]]); } + } + this.frame.touched = true; + return super.set(k, slot); + } + override delete(k: string): boolean { + const prior = super.get(k); + if (prior) this.unhome(prior, k); + this.frame.touched = true; + return super.delete(k); + } + unhome(slot: Slot, k: string): void { + const h = this.homes?.get(slot); + if (!h) return; + const i = h.findIndex(([f, key]) => f === this.frame && key === k); + if (i >= 0) h.splice(i, 1); + } + /** The frame has ended: its slots are no longer reachable through it. */ + release(): void { for (const [k, slot] of this) this.unhome(slot, k); } +} + +class Frame { + declare id: number; + declare name: string; + declare sub: Sub | null; + declare touched: boolean; + declare vars: Vars; + declare dirty: Set; + declare callLine: number | null; + declare old: Map | null; + /** The DECREASES measure on entry, for a routine that has one. */ + declare measure?: number; + constructor(name: string, sub: Sub | null, callLine: number | null, homes: WeakMap>) { + this.id = ++frameIds; + this.name = name; + this.sub = sub; + this.touched = true; // set when `vars` changes shape; cleared by a snapshot + this.vars = new Vars(this, homes); // UPPER -> slot { name, type, value, line } + this.dirty = new Set(); + this.callLine = callLine; + this.old = null; // parameter values on entry, for OLD() + } +} + +class Interpreter { + declare program: Program; + declare opts: Opts; + declare recorded: number; + declare unrecorded: { after: number; values: number } | null; + declare random: () => number; + /** How many random numbers the run has drawn: an explanation that would draw one is not given (explainFalse). */ + draws = 0; + declare shuffleRandom: () => number; + declare steps: Step[]; + declare states: State[]; + declare output: string[]; + declare input: string[]; + declare inputPos: number; + declare frames: Frame[]; + declare globals: Map; + declare hits: Map; + declare work: Work; + declare reads: Access[]; + declare writes: Access[]; + declare lastSnapshot: State | null; + declare loopStack: LoopState[]; + declare quiet: number; + declare ieee: boolean; + declare visited: number; + declare inLib: number; + declare homes: WeakMap>; + /** The statement the run is at, for a stop that has no node of its own. */ + declare at?: Stmt; + /** The ledger of the PARALLEL FOR being run, if any. */ + declare raceTrack?: RaceTrack; + constructor(program: Program, opts: RunOptions) { + this.program = program; + this.opts = { maxSteps: 100000, record: true, contracts: true, seed: 1, shuffle: false, input: [], maxDepth: 400, maxRecorded: MAX_RECORDED, ...opts }; + this.recorded = 0; // values the snapshots have copied + this.unrecorded = null; // { after, values } once recording has stopped + const draw = makeRandom(this.opts.seed); + this.random = () => { this.draws++; return draw(); }; + this.shuffleRandom = makeRandom((this.opts.seed * 7919) >>> 0); + this.steps = []; + this.states = []; + this.output = []; + this.input = (this.opts.input || []).map(String); + this.inputPos = 0; + this.frames = []; + this.globals = new Map(); + this.hits = new Map(); + this.work = { steps: 0, compares: 0, letterCompares: 0, reads: 0, writes: 0, calls: 0, contractChecks: 0 }; + this.reads = []; + this.writes = []; + this.lastSnapshot = null; + this.loopStack = []; + this.quiet = 0; // >0 while evaluating contracts: their reads are not the program's + this.ieee = !!program.ieee; // USE IEEE: NaN and infinities flow instead of stopping the run + this.visited = 0; // positions visited by quantifiers, counted apart from the program's steps + this.inLib = 0; // >0 while a library routine runs without being stepped into + this.homes = new WeakMap(); // slot -> [[frame, name]]: where markDirty finds it + } + + get frame(): Frame { return this.frames[this.frames.length - 1]; } + + /* ---------- recording ---------- */ + snapshot(): State { + const prev = this.lastSnapshot; + const frames = this.frames.map((f, i) => { + // a frame keeps its place in the stack while it lives, so its last picture is at the same index + const before = prev?.frames[i]?.id === f.id ? prev.frames[i] : undefined; + // nothing in this frame changed since that picture: it is the same picture + if (before && !f.touched && !f.dirty.size) return before; + f.touched = false; + const vars: Record = {}; + for (const [k, slot] of f.vars) { + if (before && k in before.vars && !f.dirty.has(k) && before.vars[k].slot === slot) vars[k] = before.vars[k]; + else { vars[k] = { name: slot.name, type: slot.type, value: cloneValue(slot.value), slot, line: slot.line, kind: slot.kind }; this.recorded += valueCount(slot.value); } + } + f.dirty.clear(); + return { id: f.id, name: f.name, vars, callLine: f.callLine }; + }); + const snap: State = { frames, outputLength: this.output.length, loops: this.loopStack.map((l) => ({ line: l.node.line, pass: l.pass, kind: l.node.kind })) }; + this.lastSnapshot = snap; + return snap; + } + + /** One recorded step. `node` places it on a line. */ + step(node: Place, kind: StepKind, text: string, extra: Partial = {}): void { + this.work.steps++; + if (this.inLib) { + // inside a library routine the run is not stepped into: its lines are not this program's + if (this.work.steps > this.opts.maxSteps) throw this.limitError(node); + this.reads = []; + this.writes = []; + return; + } + const line = extra.line ?? node?.line ?? null; + if (line != null) this.hits.set(line, (this.hits.get(line) || 0) + 1); + if (this.work.steps > this.opts.maxSteps) throw this.limitError(node); + if (this.raceTrack) this.raceTrack.note(this.reads, this.writes); + if (this.opts.record) { + this.steps.push({ + line, kind, text, + reads: this.reads, writes: this.writes, + depth: this.frames.length - 1, sub: this.frame?.name, + ...extra, + }); + const snap = this.snapshot(); + this.states.push(snap); + if (this.recorded > this.opts.maxRecorded) this.stopRecording(snap); + } + this.reads = []; + this.writes = []; + } + + /** The snapshots have copied more than the budget: say so in one last step, and go on without recording. */ + stopRecording(snap: State): void { + const after = this.steps.length; + this.steps.push({ + line: this.steps[after - 1]?.line ?? null, kind: 'unrecorded', + text: `The steps after this one are not recorded. Recording copies each variable a step changes, and this run has copied ${this.recorded.toLocaleString('en-US')} values, more than the ${this.opts.maxRecorded.toLocaleString('en-US')} a recording holds: a large collection changes at many steps. The run goes on; its output and how it ends are shown at the end.`, + reads: [], writes: [], depth: this.frames.length - 1, sub: this.frame?.name, + }); + this.states.push(snap); + this.unrecorded = { after, values: this.recorded }; + this.opts.record = false; + } + + limitError(node: Place): PseudoError { + const loop = this.loopStack[this.loopStack.length - 1]; + let msg = `Stopped after ${this.opts.maxSteps.toLocaleString('en-US')} steps.`; + if (loop) { + const word = loop.node.kind === 'For' ? 'FOR' : loop.node.kind === 'ForEach' ? 'FOR EACH' : loop.node.kind === 'While' ? 'WHILE' : 'REPEAT'; + msg += ` The ${word} loop on line ${loop.node.line} was on pass ${loop.pass.toLocaleString('en-US')}.`; + if (loop.stuck) msg += ` Its condition reads ${loop.stuck.join(' and ')}, and ${loop.stuck.length === 1 ? 'that value has not' : 'none of those values has'} changed in the last ${loop.same} passes, so the loop may never end. Something in the body has to move the condition toward ${loop.node.kind === 'Repeat' ? 'TRUE' : 'FALSE'}.`; + else msg += ' If the loop should end, check what moves its condition; if the input is just large, give the program a smaller one.'; + } + return new PseudoError('limit', msg, loop?.node || node); + } + + noteRead(name: string, path: Value[] | null): void { if (!this.quiet) { this.work.reads++; this.reads.push({ name, path, frame: this.frame?.id }); } } + noteWrite(name: string, path: Value[] | null, slotFrame?: number): void { this.work.writes++; if (!this.quiet) this.writes.push({ name, path, frame: slotFrame ?? this.frame?.id }); } + + /* ---------- names ---------- */ + slot(name: string, node: Place): Slot { + const k = up(name); + const s = this.frame.vars.get(k); + if (s) return s; + const g = this.globals.get(k); + if (g) return g; + throw new PseudoError('unset', `${name} does not exist at this point in the run.`, node); + } + + read(node: NameExpr): Value { + const s = this.slot(node.name, node); + if (s.value === UNSET) { + if (s.afterLoop) throw new PseudoError('unset', `${s.name} was the counter of the FOR loop that ended on line ${s.afterLoop}, and it has no value after the loop. Treat a counter as gone once its loop ends; if you need the position after the loop, keep it in a variable of your own.`, node, { name: s.name }); + throw new PseudoError('unset', `${s.name} was declared on line ${s.line} and has not been given a value yet, so there is nothing to read. Give it a starting value first.`, node, { name: s.name }); + } + this.noteRead(s.name, null); + return s.value; + } + + declareSlot(name: string, type: PType | undefined, value: Value, line: number, kind: Slot['kind'] = 'var'): Slot { + const k = up(name); + const slot: Slot = { name, type, value, line, kind }; + this.frame.vars.set(k, slot); + this.frame.dirty.add(k); + return slot; + } + + /** A slot's value changed: every live frame that reaches it under some name shows the change at the next step. */ + markDirty(slot: Slot): void { + for (const [f, k] of this.homes.get(slot) || []) f.dirty.add(k); + } + + /* ---------- program entry ---------- */ + start(): void { + const prog = this.program; + const main = new Frame('main', null, null, this.homes); + this.frames.push(main); + // constants and DEFINE sets + for (const c of prog.constants.values()) { + this.globals.set(up(c.name), { name: c.name, type: c.type, value: this.evaluate(c.node.value), line: c.line, kind: 'const' }); + } + for (const d of prog.defines.values()) { + const s = new Set(d.node.values.map((v) => this.evaluate(v))); + this.globals.set(up(d.name), { name: d.name, type: d.type, value: s, line: d.line, kind: 'define' }); + } + this.states.push(this.snapshot()); + } + + runMain(): void { + this.start(); + this.block(this.program.main.body); + } + + runCall(name: string, args: Value[]): Value { + this.start(); + const sub = this.program.subs.get(up(name)) || this.program.libs?.get(up(name)); + if (!sub) throw new PseudoError('value', `No FUNCTION or PROCEDURE is called ${name}.`, null); + if (args.length !== sub.params.length) throw new PseudoError('value', `${sub.name} takes ${plural(sub.params.length, 'value')}, and ${args.length} ${args.length === 1 ? 'was' : 'were'} given.`, sub.node); + sub.params.forEach((p, i) => { + if (!valueFits(p.type, args[i])) throw new PseudoError('value', `Parameter ${p.name} of ${sub.name} is ${typeName(p.type)}, and the value given was ${formatValue(args[i])}.`, sub.node); + // NaN and the infinities enter a run only where the program says USE IEEE, from outside as from inside + if (!this.ieee && !valueFits(p.type, args[i], true)) throw new PseudoError('value', `Parameter ${p.name} of ${sub.name} is ${typeName(p.type)}, and the value given was ${formatValue(args[i])}, which is not a finite number. A FLOAT holds NaN or an infinity only in a program that says USE IEEE.`, sub.node); + }); + const slots = args.map((v, i): ArgSlot => ({ name: `(argument ${i + 1})`, type: sub.params[i].type, value: cloneValue(v) })); + return this.invoke(sub, slots, sub.node, true); + } + + /* ---------- statements ---------- */ + block(body: Stmt[]): void { for (const s of body) this.statement(s); } + + statement(s: Stmt): void { + this.at = s; // where the run is, for a stop that has no node of its own + switch (s.kind) { + case 'Declare': { + const type = s.resolved!; // the checker resolved every DECLARE of a program that runs + for (const n of s.names) { + const slot = this.declareSlot(n.name, type, initialValue(type, this, s), s.line); + this.noteWrite(slot.name, null); + } + const names = s.names.map((n) => n.name).join(', '); + const what = ['LIST', 'HASHMAP', 'SET'].includes(type.k) ? 'empty' : type.k === 'ARRAY' ? 'with every position still unset' : type.k === 'RECORD' ? 'with every field still unset' : 'with no value yet'; + this.step(s, 'declare', `${names} ${s.names.length === 1 ? 'is' : 'are'} ${typeName(type)}, ${what}.`); + return; + } + case 'Assign': { + const v = this.evaluate(s.value); + const before = this.peekTarget(s.target); + this.assign(s.target, v, s); + const tt = s.target.type || s.value.type; + const shown = formatValue(v, tt, { bareChar: true }); + const was = before === UNSET || before === undefined ? '' : ` (it was ${formatValue(before, tt)})`; + this.step(s, 'assign', `${sourceOf(s.target)} becomes ${shown}${was}.`, { value: v, target: sourceOf(s.target) }); + return; + } + case 'Input': { + if (this.inputPos >= this.input.length) throw new PseudoError('input', `INPUT ${sourceOf(s.target)} asked for a value, and none was left. Add a line to the input box.`, s); + const raw = this.input[this.inputPos++]; + const t = s.target!.type || T.STRING; // an INPUT with no target does not parse + const v = this.parseInput(raw, t, s); + this.assign(s.target!, v, s); + this.step(s, 'input', `INPUT reads ${JSON.stringify(raw)} into ${sourceOf(s.target)}.`); + return; + } + case 'Output': { + const parts = s.values.map((e) => outputText(this.evaluate(e), e.type)); + const line = parts.join(''); + this.output.push(line); + this.step(s, 'output', `Shows: ${line}`, { output: line }); + return; + } + case 'If': return this.ifStmt(s); + case 'Case': return this.caseStmt(s); + case 'For': return this.forStmt(s); + case 'ForEach': return this.forEach(s); + case 'While': return this.whileStmt(s); + case 'Repeat': return this.repeatStmt(s); + case 'Break': this.step(s, 'break', 'BREAK leaves the innermost loop now.'); throw new BreakSignal(s.line); + case 'Continue': this.step(s, 'continue', 'CONTINUE skips the rest of this pass.'); throw new ContinueSignal(s.line); + case 'Return': { + const v = s.value ? this.evaluate(s.value) : undefined; + const f = this.frame; + this.step(s, 'return', s.value ? `${f.name} hands back ${formatValue(v, f.sub?.returns)}.` : `${f.name} ends here.`, { value: v }); + throw new ReturnSignal(v, s.line); + } + case 'Call': { + if (s.builtin) return this.builtinStmt({ ...s, name: up(s.name) }); + const sub = (s.sub || this.program.subs.get(up(s.name)))!; // the checker found the procedure + const slots = this.argSlots(sub, s.args); + this.invoke(sub, slots, s, false); + return; + } + case 'BuiltinCall': return this.builtinStmt(s); + case 'Assert': { + if (!this.opts.contracts) return; + this.checkCondition(s.expr, s, 'assert', () => `ASSERT on line ${s.line} is FALSE: ${this.explainFalse(s.expr)}.`); + this.step(s, 'check', `ASSERT holds: ${sourceOf(s.expr)}.`); + return; + } + case 'ExprStmt': this.evaluate(s.expr); this.step(s, 'expr', `${sourceOf(s.expr)} is computed and thrown away.`); return; + default: return; + } + } + + peekTarget(t: Expr): Value { + if (hasCall(t)) return UNSET; + try { this.quiet++; return this.evaluateTargetValue(t); } catch { return UNSET; } finally { this.quiet--; } + } + + evaluateTargetValue(t: Expr): Value { + if (t.kind === 'Name') { const s = this.frame.vars.get(up(t.name)) || this.globals.get(up(t.name)); return s ? s.value : UNSET; } + if (t.kind === 'Index') { + const c = this.evaluateTargetValue(t.target); + const idx = t.indices.map((i) => this.evaluate(i)); + if (Array.isArray(c)) return c[idx[0] as number]; + if (c instanceof PArray) return c.data[c.dims.length === 2 ? (idx[0] as number) * c.dims[1] + (idx[1] as number) : idx[0] as number]; + if (c instanceof Map) return c.has(idx[0]) ? c.get(idx[0]) : UNSET; + return UNSET; + } + if (t.kind === 'Field') { const r = this.evaluateTargetValue(t.target); return r instanceof PRecord ? r.fields.get(up(t.field)) : UNSET; } + return UNSET; + } + + parseInput(raw: string, t: PType, at: InputStmt): Value { + const s = raw.trim(); + switch (t.k) { + case 'INTEGER': + if (/^[+-]?\d+$/.test(s)) { + // a whole number this lab holds exactly, or the run stops here as it would on a sum that large + if (!Number.isSafeInteger(Number(s))) throw new PseudoError('overflow', `INPUT ${sourceOf(at.target)} read ${s}, which is past 9,007,199,254,740,991, the largest whole number this lab can hold exactly.`, at); + return Number(s); + } + break; + // a FLOAT is a decimal number, finite unless the program says USE IEEE; a FLOAT32 is rounded once, from the text + case 'FLOAT': { + const v = decimalText(s, this.ieee); + if (v !== null && (this.ieee || Number.isFinite(v))) return isFloat32(t) && Number.isFinite(v) ? float32OfText(s) : v; + if (v !== null) throw new PseudoError('input', `INPUT ${sourceOf(at.target)} read ${s}, which is past the largest FLOAT, about 1.8 × 10^308. Only a program that says USE IEEE holds an infinity.`, at); + break; + } + case 'BOOL': if (/^(TRUE|FALSE)$/i.test(s)) return /^TRUE$/i.test(s); break; + case 'CHAR': if (textLength(raw) === 1) return raw; break; + default: return raw; + } + throw new PseudoError('input', `INPUT ${sourceOf(at.target)} expected ${typeName(t)}, and the input line was ${JSON.stringify(raw)}.`, at); + } + + /** Store a value at a target: a name, a position, a key, or a field. */ + assign(target: Expr, value: Value, at: FitAt & Loc): void { + const v = cloneValue(value); + if (target.kind === 'Name') { + const slot = this.slot(target.name, target); + slot.value = this.fitNumber(slot.type, this.sameShape(slot, v, at), at); + this.markDirty(slot); + this.noteWrite(slot.name, null); + return; + } + // Walk down to the container, then set inside it. + const { root, container, key, type } = this.locate(target, at); + this.markDirty(root); + if (Array.isArray(container)) { + this.checkPosition(container.length, key, target, 'LIST'); + container[key] = this.fitNumber(type, v, at); + } else if (container instanceof PArray) { + const flat = this.flatIndex(container, key as Value[], target as IndexExpr); + container.data[flat] = this.fitNumber(type, v, at); + } else if (container instanceof Map) { + container.set(key, this.fitNumber(type, v, at)); + } else if (container instanceof PRecord) { + container.fields.set(key as string, this.fitNumber(type, v, at)); + } else { + throw new PseudoError('value', `${sourceOf((target as { target?: Expr }).target)} cannot hold values by position.`, at); + } + this.noteWrite(root.name, targetPath(target, key)); + } + + /** + * A whole collection stored under the name of a declared ARRAY: an ARRAY keeps the length (or the rows and + * columns) it was declared with, so a LIST or an ARRAY of another length stops the run, and a LIST of the right + * length becomes the ARRAY's positions (it used to replace the ARRAY, which then grew and shrank like a LIST). + */ + sameShape(slot: Slot, v: Value, at: FitAt): Value { + const was = slot.value; + if (slot.type?.k !== 'ARRAY' || !(was instanceof PArray)) return v; + const dims = Array.isArray(v) ? [v.length] : v instanceof PArray ? v.dims : null; + if (!dims) return v; + if (dims.length !== was.dims.length || dims.some((n, i) => n !== was.dims[i])) { + throw new PseudoError('value', `${slot.name} is an ARRAY of ${was.dims.join(' by ')} positions, and the ${Array.isArray(v) ? 'LIST' : 'ARRAY'} stored in it has ${dims.join(' by ')}. An ARRAY keeps the length it was declared with; a LIST is the collection that grows and shrinks.`, at); + } + if (!Array.isArray(v)) return v; + const a = new PArray(dims); + a.data = v; + return a; + } + + /** + * A value going into a name, a position, a parameter or a RETURN of a sized type: a whole number that does not + * fit stops the run, and a number going into a FLOAT32 is rounded to the nearest single. Collections of sized + * numbers are checked value by value. Everything else is stored as it is. + */ + fitNumber(type: PType | null | undefined, v: Value, at: FitAt): Value { + if (!type) return v; + if (type.k === 'ARRAY' && (Array.isArray(v) || v instanceof PArray)) v = this.fixedShape(type, v, at); + if (typeof v === 'number') { + if (isSizedInt(type)) return this.fitInt(type, v, at, 'store'); + // typed-port: `at` is the statement or call the value goes through, not an expression: a FLOAT32 that overflowed + // here would be reported as "?" with FLOAT's limit. Every way in is checked before it gets here (an INTEGER fits + // a FLOAT32, a FLOAT does not go into one unconverted), so no run reaches it. + if (isFloat32(type)) return this.finiteCheck(Math.fround(v), at as Expr); + return v; + } + if ((type.k === 'LIST' || type.k === 'ARRAY') && type.of && (isSizedInt(type.of) || isFloat32(type.of))) { + if (Array.isArray(v)) return v.map((x) => (x === UNSET ? x : this.fitNumber(type.of, x, at))); + if (v instanceof PArray) { v.data = v.data.map((x) => (x === UNSET ? x : this.fitNumber(type.of, x, at))); return v; } + } + return v; + } + + /** + * A collection going where an ARRAY declared with plain-number lengths is (a store, a parameter, a RETURN, a value + * APPENDed to a LIST OF ARRAY): it has those lengths or the run stops, and a LIST of the right length becomes an + * ARRAY. The checker reports what it can see before the run (`array-length`); the prover has an obligation for + * the rest, after which the lengths are a fact. + */ + fixedShape(type: PType, v: Value[] | PArray, at: FitAt): Value[] | PArray { + const want = knownDims(type); + if (!want) return v; + const have = Array.isArray(v) ? [v.length] : v.dims; + if (have.length !== want.length || have.some((n, i) => n !== want[i])) { + throw new PseudoError('value', `${at?.target ? sourceOf(at.target) : 'The value'} is an ARRAY of ${want.join(' by ')} positions, and the ${Array.isArray(v) ? 'LIST' : 'ARRAY'} given to it has ${have.join(' by ')}. An ARRAY keeps the length it was declared with; a LIST is the collection that grows and shrinks.`, at); + } + if (!Array.isArray(v)) return v; + const a = new PArray(want); + a.data = v; + return a; + } + + /** + * A whole number checked against a sized type. `how` is 'store' (a value put into a name or position), + * 'result' (what an operator computed) or 'convert' (INT32(X) and its kind). + */ + fitInt(type: IntType, v: number, at: FitAt, how: 'store' | 'result' | 'convert'): number { + if (!Number.isFinite(v) || !Number.isInteger(v)) throw new PseudoError('value', `${formatValue(v)} is not a whole number, and ${type.name} holds whole numbers only.`, at); + const [lo, hi] = intRange(type); + if (v >= lo && v <= hi) return v; + const big = Math.abs(v) > EXACT_MAX ? ' (past what this lab holds exactly)' : ''; + // `at` is the expression itself for a result or a conversion + const what = how === 'result' ? `${sourceOf(at as Expr)} comes to ${formatValue(v)}${big}` + : how === 'convert' ? `${sourceOf(at as Expr)} gets ${formatValue(v)}${big}` + : `${at?.target ? sourceOf(at.target) : 'The value'} would receive ${formatValue(v)}${big}`; + throw new PseudoError('overflow', `${what}, and ${type.name} holds ${intRangeText(type)}. A sized type stops the run instead of wrapping around; write WRAP(..., ${type.name}) when wrapping is what you mean, or use a wider type.`, at, { type: type.name }); + } + + /** What an arithmetic operator's result becomes in its type: checked for a sized whole number, rounded for FLOAT32. */ + sized(e: Expr, v: number): number { + const t = e.type; + if (isSizedInt(t)) return this.fitInt(t, v, e, 'result'); + if (isFloat32(t)) return this.finiteCheck(Math.fround(v), e); + return v; + } + + /** The container a target writes into, and the key within it. */ + locate(target: Expr, at: FitAt): { root: Slot; container: Value; key: Value; type: PType | undefined } { + if (target.kind === 'Index') { + const { value: container, root } = this.resolveContainer(target.target, at); + const idx = target.indices.map((i) => this.evaluate(i)); + const key = container instanceof PArray ? idx : idx[0]; + return { root, container, key, type: target.type }; + } + if (target.kind === 'Field') { + const { value: container, root } = this.resolveContainer(target.target, at); + if (!(container instanceof PRecord)) throw new PseudoError('value', `${sourceOf(target.target)} is not a record.`, at); + return { root, container, key: up(target.field), type: target.type }; + } + throw new PseudoError('value', `${sourceOf(target)} cannot receive a value.`, at); + } + + /** The live value a target expression names, for writing into. */ + resolveContainer(e: Expr, at: Place): { value: Value; root: Slot } { + if (e.kind === 'Name') { + const slot = this.slot(e.name, e); + if (slot.value === UNSET) this.read(e); + return { value: slot.value, root: slot }; + } + if (e.kind === 'Index') { + const { value: c, root } = this.resolveContainer(e.target, at); + const idx = e.indices.map((i) => this.evaluate(i)); + let v: Value; + if (Array.isArray(c)) { this.checkPosition(c.length, idx[0], e, 'LIST'); v = c[idx[0]]; } + else if (c instanceof PArray) v = c.data[this.flatIndex(c, idx, e)]; + else if (c instanceof Map) { if (!c.has(idx[0])) throw this.keyError(e, idx[0]); v = c.get(idx[0]); } + else throw new PseudoError('value', `${sourceOf(e.target)} has no positions.`, at); + return { value: v, root }; + } + if (e.kind === 'Field') { + const { value: r, root } = this.resolveContainer(e.target, at); + return { value: (r as PRecord).fields.get(up(e.field)), root }; + } + return { value: this.evaluate(e), root: { name: '?' } as Slot }; // a value that is no variable's: nothing to mark + } + + ifStmt(s: IfStmt): void { + for (let i = 0; i < s.branches.length; i++) { + const b = s.branches[i]; + const c = this.evaluate(b.cond); + const word = i === 0 ? 'IF' : 'ELSEIF'; + const next = c ? 'so its body runs' : i + 1 < s.branches.length ? 'so the next ELSEIF is tested' : s.elseBody ? 'so the ELSE body runs' : 'so the body is skipped'; + this.step(b.cond, 'test', `${word} ${this.describeCondition(b.cond)} is ${c ? 'TRUE' : 'FALSE'}, ${next}.`, { outcome: c, line: b.line }); + if (c) { this.block(b.body); return; } + } + if (s.elseBody) this.block(s.elseBody); + } + + caseStmt(s: CaseStmt): void { + const v = s.subject ? this.evaluate(s.subject) : UNSET; + const shown = formatValue(v, s.subject?.type, { bareChar: true }); + for (const arm of s.arms) { + for (const lab of arm.labels) { + let hit: boolean; + if (lab.value) hit = valuesEqual(v, this.evaluate(lab.value)); + else { + const lo = this.evaluate(lab.low); const hi = this.evaluate(lab.high); + hit = compareValues(v, lo) >= 0 && compareValues(v, hi) < 0; + } + if (hit) { + this.step(s, 'select', `${sourceOf(s.subject)} is ${shown}, which matches ${lab.value ? sourceOf(lab.value) : `${sourceOf(lab.low)} TO ${sourceOf(lab.high)}, the range [${sourceOf(lab.low)}, ${sourceOf(lab.high)})`}.`, { outcome: arm.line }); + this.block(arm.body); + return; + } + } + } + if (s.otherwise) { + this.step(s, 'select', `${sourceOf(s.subject)} is ${shown}, which matches no branch, so OTHERWISE runs.`, { outcome: s.otherwiseLine }); + this.block(s.otherwise); + } else { + this.step(s, 'select', `${sourceOf(s.subject)} is ${shown}, which matches no branch, and there is no OTHERWISE, so nothing happens.`, { outcome: null }); + } + } + + /* ---------- loops ---------- */ + enterLoop(node: LoopStmt): LoopState { const l: LoopState = { node, pass: 0, same: 0, stuck: null, lastKey: null }; this.loopStack.push(l); return l; } + leaveLoop(): void { this.loopStack.pop(); } + + /** Track whether a WHILE or REPEAT condition's inputs ever change. */ + watchCondition(loop: LoopState, cond: Expr): void { + const names = [...namesIn(cond)]; + const key = names.map((n) => { try { const s = this.frame.vars.get(up(n)) || this.globals.get(up(n)); return s ? formatValue(s.value, s.type, { max: 8 }) : '?'; } catch { return '?'; } }).join('|'); + if (key === loop.lastKey) { loop.same++; if (loop.same >= 50) loop.stuck = names; } else { loop.same = 0; loop.stuck = null; } + loop.lastKey = key; + } + + forStmt(s: ForStmt): void { + if (s.parallel) return this.parallelFor(s); + return this.sequentialFor(s); + } + + /** + * PARALLEL FOR: the passes are declared independent, as on a GPU where each runs on its own + * thread at the same time. The lab runs them one after another (any order would do) and keeps a + * ledger of what each pass reads and writes. A location written by one pass and touched by + * another is a data race, and the run stops with the two passes named: that is the whole lesson. + * Names declared inside the body and the counter are each pass's own and are exempt. + */ + parallelFor(s: ForStmt): void { + const outer = this.raceTrack; + const own = new Set([up(s.var)]); + const collectDecl = (body: Stmt[] | null | undefined): void => { for (const st of body || []) { if (st.kind === 'Declare') for (const n of st.names || []) own.add(up(n.name ?? n)); for (const k of ['body', 'elseBody'] as const) if ((st as LooseStmt)[k]) collectDecl((st as LooseStmt)[k]); if ((st as LooseStmt).branches) for (const b of (st as LooseStmt).branches!) collectDecl(b.body); if ((st as LooseStmt).arms) for (const a of (st as LooseStmt).arms!) collectDecl(a.body); } }; + collectDecl(s.body); + const ledger = new Map; writes: Map }>(); // key -> { reads: Map(pass -> line), writes: Map(pass -> line) } + const frameId = this.frame?.id; + let pass = 0; + let race: Race | null = null; + const keyOf = (r: Access) => `${r.name}${r.path && r.path.length ? '[' + r.path.join(', ') + ']' : ''}`; + this.raceTrack = { + setPass: (p: number) => { pass = p; }, + note: (reads: Access[], writes: Access[]) => { + if (race || !pass) return; + for (const [list, which] of [[reads, 'reads'], [writes, 'writes']] as Array<[Access[], 'reads' | 'writes']>) { + for (const r of list) { + if (r.frame !== frameId || own.has(up(r.name))) continue; + const key = keyOf(r); + const whole = ledger.get(r.name); // a write to the whole collection + const entry = ledger.get(key) || { reads: new Map(), writes: new Map() }; + ledger.set(key, entry); + const other = (m: Map) => [...m.keys()].find((q) => q !== pass); + if (which === 'writes') { + const q = other(entry.writes) ?? other(entry.reads) ?? (whole && whole !== entry ? (other(whole.writes) ?? other(whole.reads)) : undefined); + if (q !== undefined) race = { key, pass, other: q, what: 'writes' }; + } else { + const q = other(entry.writes) ?? (whole && whole !== entry ? other(whole.writes) : undefined); + if (q !== undefined) race = { key, pass, other: q, what: 'reads' }; + } + entry[which].set(pass, true); + } + } + }, + }; + try { + this.sequentialFor(s, (p) => { this.raceTrack!.setPass(p); if (race) this.raiseRace(s, race); }); + if (race) this.raiseRace(s, race); + } finally { + this.raceTrack = outer; + } + } + + raiseRace(s: ForStmt, race: Race): never { + const verb = race.what === 'writes' ? 'written' : 'read'; + const [a, b] = [race.other, race.pass].sort((p, q) => p - q); + throw new PseudoError('race', `${race.key} is ${verb} by pass ${race.pass} of the PARALLEL FOR on line ${s.line}, and pass ${race.other} ${race.what === 'writes' ? 'also touches' : 'writes'} it. In a parallel loop every pass runs at the same time, so passes ${a} and ${b} would be fighting over one location: a data race, whose result depends on which thread gets there first. Give each pass its own slot (indexed by ${s.var}) and combine the slots after the loop.`, s, { passes: [a, b], location: race.key }); + } + + sequentialFor(s: ForStmt, onPass: ((pass: number) => void) | null = null): void { + // a FOR that runs has a start and a stop, and the checker holds all three to whole numbers + const start = this.evaluate(s.start!) as number; + const stop = this.evaluate(s.stop!) as number; + const step = s.step ? this.evaluate(s.step) as number : 1; + if (step === 0) throw new PseudoError('value', 'STEP 0 never moves the counter, so the loop would never end.', s.step || s); + const own = !s.declaredCounter; + let slot: Slot & { value: number }; // the counter's value is a whole number while the loop runs + if (own) slot = this.declareSlot(s.var, T.INTEGER, start, s.line, 'counter') as Slot & { value: number }; + else { slot = this.slot(s.var, s) as Slot & { value: number }; slot.value = start; this.markDirty(slot); slot.afterLoop = null; } + this.noteWrite(slot.name, null); + const loop = this.enterLoop(s); + const cmp = step > 0 ? '<' : '>'; + const within = (i: number) => (step > 0 ? i < stop : i > stop); + const describe = step > 0 ? `the passes cover [${start}, ${stop})` : `counting down while above ${stop}`; + this.step(s, 'loop-start', `${s.var} starts at ${start}; ${start === stop || !within(start) ? 'that is already at the stop value, so the body never runs' : describe}.`); + let measure: number | null = null; + try { + for (; ;) { + this.checkInvariants(s, loop); + if (!within(slot.value)) { + this.step(s, 'loop-end', `${s.var} is ${slot.value}, which is not ${cmp} ${stop}, so the loop ends${loop.pass ? ` after ${plural(loop.pass, 'pass', 'passes')}` : ''}.`, { outcome: false }); + break; + } + measure = this.checkDecreases(s, measure); + // a counter declared with a sized type holds only what that type holds, pass by pass + if (!own && isSizedInt(slot.type)) this.fitInt(slot.type, slot.value, { ...s, target: { kind: 'Name', name: s.var } }, 'store'); + loop.pass++; + if (loop.pass > 1) this.step(s, 'loop-test', `${s.var} is ${slot.value}, still ${cmp} ${stop}: pass ${loop.pass}.`, { outcome: true, pass: loop.pass }); + else this.step(s, 'loop-test', `${s.var} is ${slot.value}, ${cmp} ${stop}: pass 1.`, { outcome: true, pass: 1 }); + if (onPass) onPass(loop.pass); + try { this.block(s.body); } catch (e) { + if (e instanceof BreakSignal) { break; } + if (!(e instanceof ContinueSignal)) throw e; + } + slot.value += step; + if (!Number.isSafeInteger(slot.value)) throw new PseudoError('overflow', `NEXT ${s.var} takes ${s.var} past 9,007,199,254,740,991, the largest whole number this lab can hold exactly.`, s); + this.markDirty(slot); + this.noteWrite(slot.name, null); + this.step(s, 'next', `NEXT ${s.var}: ${s.var} becomes ${slot.value}.`, { line: s.endLine ?? s.line }); + } + } finally { + this.leaveLoop(); + } + if (own) { this.frame.vars.delete(up(s.var)); this.frame.dirty.add(up(s.var)); } + else { (slot as Slot).value = UNSET; slot.afterLoop = s.endLine; this.markDirty(slot); } + } + + forEach(s: ForEachStmt): void { + const coll = this.evaluate(s.iterable!); // a FOR EACH that runs names its collection + let items = this.iterate(coll, s.iterable!); + const slot = this.declareSlot(s.var, s.eachSym?.type || T.ANY, UNSET, s.line, 'each'); + const loop = this.enterLoop(s); + const kind = coll instanceof Map ? 'key' : coll instanceof Set ? 'member' : 'value'; + this.step(s, 'loop-start', items.length ? `FOR EACH will visit ${plural(items.length, kind)} of ${sourceOf(s.iterable)}${s.unordered ? ', in an order the notation does not promise' : ', in order'}.` : `${sourceOf(s.iterable)} is empty, so the body never runs.`); + let broke = false; + try { + for (let i = 0; i < items.length; i++) { + slot.value = cloneValue(items[i]); + this.markDirty(slot); + this.noteWrite(slot.name, null); + this.checkInvariants(s, loop); + loop.pass++; + this.step(s, 'loop-test', `${s.var} is ${formatValue(items[i], slot.type, { bareChar: true })}${typeof coll === 'string' || Array.isArray(coll) || coll instanceof PArray ? `, from position ${i}` : ''}: pass ${loop.pass}.`, { outcome: true, pass: loop.pass }); + try { this.block(s.body); } catch (e) { + if (e instanceof BreakSignal) { broke = true; break; } + if (!(e instanceof ContinueSignal)) throw e; + } + } + // After the last pass (or instead of any, for an empty collection) the loop's variable has no value. An INVARIANT + // that does not name it describes the state between passes, and is checked once more: the prover relies on it + // after the loop (library/pseudo/prover/foreach-invariant-exit.pseudo). + if (!broke) this.checkInvariants(s, loop, loop.pass > 0, (inv) => ![...namesIn(inv.expr)].some((n) => up(n) === up(s.var))); + if (loop.pass === items.length) this.step(s, 'loop-end', `Every ${kind} has been visited, so the loop ends.`, { outcome: false, line: s.endLine ?? s.line }); + } finally { this.leaveLoop(); } + this.frame.vars.delete(up(s.var)); + this.frame.dirty.add(up(s.var)); + } + + iterate(coll: Value, at: Expr): Value[] { + if (typeof coll === 'string') return chars(coll); + if (Array.isArray(coll)) return coll.slice(); + if (coll instanceof PArray) return coll.data.slice(); + if (coll instanceof Set || coll instanceof Map) { + const keys = sortedKeys(coll instanceof Map ? coll.keys() : coll); + if (this.opts.shuffle) for (let i = keys.length - 1; i > 0; i--) { const j = Math.floor(this.shuffleRandom() * (i + 1));[keys[i], keys[j]] = [keys[j], keys[i]]; } + return keys; + } + if (coll && (coll as RangeValue).range) { + const r = coll as RangeValue; + if (r.hi - r.lo > MAX_VALUES) throw new PseudoError('value', `The range [${r.lo}, ${r.hi}) is too large to visit one position at a time (more than ${MAX_VALUES.toLocaleString('en-US')}).${r.hi - r.lo <= this.opts.maxSteps ? ` A counted loop, FOR I <- ${r.lo} TO ${r.hi}, visits the same positions without listing them first.` : ` That is also more passes than a run may take (${this.opts.maxSteps.toLocaleString('en-US')} steps); give the program a smaller input.`}`, at); + const out: Value[] = []; for (let i = r.lo; i < r.hi; i++) out.push(i); return out; + } + throw new PseudoError('value', `${sourceOf(at)} cannot be visited value by value.`, at); + } + + whileStmt(s: WhileStmt): void { + const loop = this.enterLoop(s); + let measure: number | null = null; + try { + for (; ;) { + this.checkInvariants(s, loop); + const c = this.evaluate(s.cond); + this.watchCondition(loop, s.cond); + if (!c) { this.step(s, 'loop-end', `WHILE ${this.describeCondition(s.cond)} is FALSE, so the loop ends${loop.pass ? ` after ${plural(loop.pass, 'pass', 'passes')}` : ' before its first pass'}.`, { outcome: false }); break; } + measure = this.checkDecreases(s, measure); + loop.pass++; + this.step(s, 'loop-test', `WHILE ${this.describeCondition(s.cond)} is TRUE: pass ${loop.pass}.`, { outcome: true, pass: loop.pass }); + try { this.block(s.body); } catch (e) { + if (e instanceof BreakSignal) break; + if (!(e instanceof ContinueSignal)) throw e; + } + } + } finally { this.leaveLoop(); } + } + + repeatStmt(s: RepeatStmt): void { + const loop = this.enterLoop(s); + let measure: number | null = null; + try { + for (; ;) { + this.checkInvariants(s, loop); + measure = this.checkDecreases(s, measure); + loop.pass++; + this.step(s, 'loop-test', `REPEAT: pass ${loop.pass} begins.`, { outcome: true, pass: loop.pass }); + let broke = false; + try { this.block(s.body); } catch (e) { + if (e instanceof BreakSignal) broke = true; + else if (!(e instanceof ContinueSignal)) throw e; + } + if (broke) break; + const c = this.evaluate(s.cond!); // a REPEAT that runs has its UNTIL + this.watchCondition(loop, s.cond!); + this.step(s.cond || s, 'loop-end', `UNTIL ${this.describeCondition(s.cond)} is ${c ? 'TRUE, so the loop ends' : 'FALSE, so the body runs again'}.`, { outcome: !c, line: s.endLine }); + if (c) { this.checkInvariants(s, loop, true); break; } + } + } finally { this.leaveLoop(); } + } + + /* ---------- contracts ---------- */ + checkInvariants(loop: LoopStmt, state: LoopState, atEnd = false, only: ((inv: ContractStmt) => boolean) | null = null): void { + if (!this.opts.contracts || !loop.invariants?.length) return; + for (const inv of loop.invariants) { + if (only && !only(inv)) continue; + const when = atEnd ? 'after the last pass' : state.pass === 0 ? 'before the first pass' : `after pass ${state.pass}`; + this.checkCondition(inv.expr, inv, 'invariant', () => `The INVARIANT on line ${inv.line} is FALSE ${when}: ${this.explainFalse(inv.expr)}.${state.pass === 0 ? ' It has to hold before the loop starts, so look at the set-up lines above the loop.' : ' It held at the start of this pass, so something in the body broke it.'}`, { phase: state.pass === 0 ? 'init' : 'keep' }); + this.step(inv, 'check', `INVARIANT holds ${when}: ${sourceOf(inv.expr)}.`); + } + } + + checkDecreases(loop: LoopStmt, previous: number | null): number | null { + if (!this.opts.contracts || !loop.decreases) return null; + const d = loop.decreases; + const v = this.quietly(() => this.evaluate(d.expr)) as number; // the checker holds a DECREASES to a whole number + this.work.contractChecks++; + if (v < 0) throw new PseudoError('decreases', `DECREASES ${sourceOf(d.expr)} is ${v} while the loop is still going. The measure has to stay at 0 or above for as long as the loop runs.`, d, { phase: 'bound' }); + if (previous != null && !(v < previous)) throw new PseudoError('decreases', `DECREASES ${sourceOf(d.expr)} was ${previous} at the start of the last pass and is ${v} now. It has to get strictly smaller on every pass, or nothing guarantees the loop ends.`, d, { phase: 'step' }); + return v; + } + + checkCondition(expr: Expr, node: Place, kind: ErrorKind, message: () => string, extra: Partial = {}): void { + this.work.contractChecks++; + const ok = this.quietly(() => this.evaluate(expr)); + if (!ok) throw new PseudoError(kind, message(), node, extra); + } + + quietly(fn: () => R): R { this.quiet++; try { return fn(); } finally { this.quiet--; } } + + /** + * For a FALSE condition, the values that made it so. It evaluates the parts again, and the run is about to stop, so + * what a routine called there would change no longer matters; but what it shows must not appear twice, and a value + * that came from a random number would not come out the same. So the lines shown while explaining are dropped, + * and an explanation that drew a random number is replaced by the condition's text alone. + */ + explainFalse(e: Expr): string { + const shown = this.output.length; + const drawn = this.draws; + const said = this.explainFalseOnce(e); + this.output.length = shown; + return this.draws === drawn ? said : sourceOf(e); + } + + explainFalseOnce(e: Expr): string { + return this.quietly(() => { + try { + if (e.kind === 'Binary' && e.op === 'AND') { + const l = this.evaluate(e.left); + return l ? this.explainFalseOnce(e.right) : this.explainFalseOnce(e.left); + } + if (e.kind === 'Binary' && ['==', '!=', '<', '<=', '>', '>='].includes(e.op)) { + const l = this.evaluate(e.left); const r = this.evaluate(e.right); + return `${sourceOf(e)}, where the left side is ${formatValue(l, e.left.type, { bareChar: true })} and the right side is ${formatValue(r, e.right.type, { bareChar: true })}`; + } + if (e.kind === 'Quant' && e.q === 'FORALL') { + const dom = this.domainValues(e.domain); + for (const x of dom) { + if (!this.withBound(e.var, x, () => this.evaluate(e.body))) return `${sourceOf(e)} fails at ${e.var} = ${formatValue(x, null, { bareChar: true })}`; + } + } + return sourceOf(e); + } catch (err) { return `${sourceOf(e)} (${(err as Error).message})`; } + }); + } + + /** Bind a quantifier's name for the length of one evaluation. */ + withBound(name: string, value: Value, fn: () => R): R { + const k = up(name); + const f = this.frame; + const prior = f.vars.get(k); + f.vars.set(k, { name, type: T.ANY, value, kind: 'bound' }); + try { return fn(); } finally { if (prior) f.vars.set(k, prior); else f.vars.delete(k); } + } + + domainValues(d: Expr): Value[] { + const v = this.evaluate(d); + if (v && (v as RangeValue).range) { + const r = v as RangeValue; + if (r.hi - r.lo > MAX_VALUES) throw new PseudoError('value', `The range [${r.lo}, ${r.hi}) is too large to check one position at a time.`, d); + const out: Value[] = []; for (let i = r.lo; i < r.hi; i++) out.push(i); return out; + } + return this.iterate(v, d); + } + + /* ---------- subroutines ---------- */ + argSlots(sub: Sub, args: Expr[]): ArgSlot[] { + return args.map((a, i): ArgSlot => { + const p = sub.params[i]; + if (p && p.mode === 'BYREF' && a.kind === 'Name') { + const slot = this.slot(a.name, a); + return { byref: slot }; + } + return { value: cloneValue(this.evaluate(a)) }; + }); + } + + invoke(sub: Sub, argSlots: ArgSlot[], at: CallSite, entry: boolean): Value { + if (sub.lib && !entry && !this.opts.stepIntoLibrary && !this.inLib) return this.invokeLibrary(sub, argSlots, at); + if (this.frames.length > this.opts.maxDepth) throw new PseudoError('depth', `${sub.name} has been called ${this.opts.maxDepth} levels deep without returning. A recursive subroutine needs a case that stops calling itself, and every call has to move toward it.`, at); + this.work.calls++; + const caller = this.frame; + const f = new Frame(sub.name, sub, at.line, this.homes); + sub.params.forEach((p, i) => { + const a = argSlots[i]; + if (a.byref) { this.fitNumber(p.type, a.byref.value, { line: at.line, col: at.col, target: { kind: 'Name', name: `Parameter ${p.name} of ${sub.name}` } }); f.vars.set(up(p.name), a.byref); f.dirty.add(up(p.name)); } + else f.vars.set(up(p.name), { name: p.name, type: p.type, value: this.fitNumber(p.type, a.value, { line: at.line, col: at.col, target: { kind: 'Name', name: `Parameter ${p.name} of ${sub.name}` } }), line: sub.node.line, kind: 'param' }); + }); + const shown = sub.params.map((p) => `${p.name} = ${formatValue(f.vars.get(up(p.name))!.value, p.type, { bareChar: true, max: 10 })}`).join(', '); + this.frames.push(f); + let result: Value; + try { + this.step(at, 'call', `${entry ? '' : `${sourceOf(at.kind === 'Call' ? { kind: 'CallExpr', name: sub.name, args: at.args } : at)} `}enters ${sub.name}${shown ? ` with ${shown}` : ''}.`, { line: entry ? sub.node.line : at.line, callee: sub.name }); + if (this.opts.contracts) { + for (const r of sub.requires) { + this.checkCondition(r.expr, r, 'require', () => `${sub.name} was called with ${shown || 'no values'}, and its REQUIRE on line ${r.line} is FALSE: ${this.explainFalse(r.expr)}. A REQUIRE is the caller's promise, so the call is what needs fixing.`); + this.step(r, 'check', `REQUIRE holds: ${sourceOf(r.expr)}.`); + } + if (sub.ensures.length) f.old = new Map(sub.params.map((p) => [up(p.name), cloneValue(f.vars.get(up(p.name))!.value)])); + if (sub.decreases) { + const m = this.quietly(() => this.evaluate(sub.decreases!.expr)) as number; + const outer = [...this.frames].reverse().slice(1).find((x) => x.sub === sub); + if (m < 0) throw new PseudoError('decreases', `The DECREASES measure of ${sub.name}, ${sourceOf(sub.decreases.expr)}, is ${m} on this call; it has to stay at 0 or above.`, sub.decreases, { phase: 'bound' }); + if (outer && !(m < outer.measure!)) throw new PseudoError('decreases', `${sub.name} called itself with DECREASES ${sourceOf(sub.decreases.expr)} at ${m}, and the call it came from had ${outer.measure}. Each recursive call has to make the measure smaller.`, sub.decreases, { phase: 'step' }); + f.measure = m; + } + } + try { + this.block(sub.node.body); + if (sub.kind === 'Function') throw new PseudoError('return', `${sub.name} reached ENDFUNCTION without a RETURN, and a FUNCTION has to hand back a value on every path.`, { line: sub.node.endLine, col: 1 }); + this.step({ line: sub.node.endLine }, 'return', `${sub.name} ends and control goes back to line ${at.line}.`); + } catch (e) { + if (!(e instanceof ReturnSignal)) throw e; + result = sub.returns ? this.fitNumber(sub.returns, e.value, { line: e.line, col: 1, target: { kind: 'Name', name: `the RESULT of ${sub.name}` } }) : e.value; + } + if (this.opts.contracts && sub.ensures.length) { + for (const en of sub.ensures) { + const ok = this.quietly(() => this.withResult(f, result, () => this.evaluate(en.expr))); + this.work.contractChecks++; + if (!ok) { + const why = this.withResult(f, result, () => this.explainFalse(en.expr)); + throw new PseudoError('ensure', `${sub.name} was called with ${shown || 'no values'}${sub.kind === 'Function' ? ` and returned ${formatValue(result, sub.returns)}` : ''}, and its ENSURE on line ${en.line} is FALSE: ${why}.`, en, { result }); + } + this.step(en, 'check', `ENSURE holds: ${sourceOf(en.expr)}.`, { line: en.line }); + } + } + } finally { + this.frames.pop(); + f.vars.release(); + if (caller) for (const k of caller.vars.keys()) if (argSlots.some((a) => a.byref === caller.vars.get(k))) caller.dirty.add(k); + } + return result; + } + + /** + * A routine from the library, run without stepping into it: one step where it is called, one where it hands + * back, and nothing of its own lines in this program's trace (they belong to another file). Its contracts are + * still checked; a failure is reported at the call, naming the routine. + */ + invokeLibrary(sub: Sub, argSlots: ArgSlot[], at: CallSite): Value { + const origin = sub.lib ? `${sub.lib.module}.${sub.name}` : sub.name; + const shown = sub.params.map((p, i) => `${p.name} = ${formatValue(argSlots[i].byref ? argSlots[i].byref.value : argSlots[i].value, p.type, { bareChar: true, max: 10 })}`).join(', '); + this.step(at, 'call', `${sub.name} from the library (${origin}) runs with ${shown || 'no values'}; open it in the library to step through it.`, { callee: sub.name, library: origin }); + this.inLib++; + let result: Value; + try { + result = this.invoke(sub, argSlots, at, false); + } catch (e) { + if (e instanceof PseudoError) { + const where = e.line != null ? ` (line ${e.line} of ${origin})` : ''; + throw new PseudoError(e.kind, `In the library routine ${origin}${where}: ${e.message}`, at, { library: origin, libraryLine: e.line }); + } + throw e; + } finally { + this.inLib--; + } + if (sub.kind === 'Function') this.step(at, 'return', `${origin} hands back ${formatValue(result, sub.returns)}.`, { value: result, library: origin }); + return result; + } + + /** + * Evaluate an ENSURE: RESULT is the returned value, and a by-value + * parameter means the value it was called with, even if the body changed + * its own copy. A BYREF parameter means its value now; OLD() reaches back. + */ + withResult(frame: Frame, result: Value, fn: () => R): R { + const k = 'RESULT'; + const prior = frame.vars.get(k); + const swapped: Array<[string, Slot]> = []; + if (frame.old && frame.sub) { + for (const p of frame.sub.params) { + if (p.mode === 'BYREF') continue; + const pk = up(p.name); + const slot = frame.vars.get(pk); + if (slot && frame.old.has(pk)) { swapped.push([pk, slot]); frame.vars.set(pk, { ...slot, value: frame.old.get(pk) }); } + } + } + frame.vars.set(k, { name: 'RESULT', type: frame.sub?.returns, value: result === undefined ? UNSET : result, kind: 'result' }); + try { return fn(); } finally { + if (prior) frame.vars.set(k, prior); else frame.vars.delete(k); + for (const [pk, slot] of swapped) frame.vars.set(pk, slot); + } + } + + /* ---------- built-in procedures ---------- */ + builtinStmt(s: BuiltinCallStmt | CallStmt): void { + const [first, ...rest] = s.args; + const vals = rest.map((a) => this.evaluate(a)); + // the collection the procedure changes: the checker has held it to the type each procedure works on (a LIST for + // APPEND, a SET for ADD, ...), which each case below uses as such + const { value: coll, root } = this.resolveContainer(first, s) as { value: any; root: Slot }; + this.markDirty(root); + const name = sourceOf(first); + let text: string; + switch (s.name) { + case 'APPEND': + // stored as the list's own type: a sized whole number is checked, a FLOAT32 rounded + coll.push(this.fitNumber(first.type?.of, cloneValue(vals[0]), s)); + text = `${formatValue(vals[0], rest[0].type, { bareChar: true })} goes on the end of ${name}, which now holds ${plural(coll.length, 'value')}.`; + this.noteWrite(root.name, [coll.length - 1]); + break; + case 'INSERT': { + const i = vals[0] as number; + if (!Number.isInteger(i) || i < 0 || i > coll.length) throw new PseudoError('index', `INSERT puts a value at a position from 0 to ${coll.length}, the length of ${name}, and ${i} is outside that.`, s); + coll.splice(i, 0, this.fitNumber(first.type?.of, cloneValue(vals[1]), s)); + text = `${formatValue(vals[1], rest[1].type, { bareChar: true })} goes in at position ${i} of ${name}.`; + this.noteWrite(root.name, [i]); + break; + } + case 'REMOVE': { + const i = vals[0]; + this.checkPosition(coll.length, i, { ...s, kind: 'CallExpr', args: s.args }, 'LIST', name); + const [gone] = coll.splice(i, 1); + text = `${formatValue(gone, first.type?.of, { bareChar: true })} comes out of position ${i} of ${name}.`; + this.noteWrite(root.name, null); + break; + } + case 'ADD': { + vals[0] = this.fitNumber(first.type?.of, vals[0], s); + const had = coll.has(vals[0]); + coll.add(vals[0]); + text = had ? `${formatValue(vals[0], rest[0].type, { bareChar: true })} is already in ${name}, so nothing changes.` : `${formatValue(vals[0], rest[0].type, { bareChar: true })} joins ${name}.`; + this.noteWrite(root.name, null); + break; + } + case 'DELETE': { + const had = coll.has(vals[0]); + coll.delete(vals[0]); + text = had ? `${formatValue(vals[0], rest[0].type, { bareChar: true })} leaves ${name}.` : `${formatValue(vals[0], rest[0].type, { bareChar: true })} was not in ${name}, so nothing changes.`; + this.noteWrite(root.name, null); + break; + } + case 'SORT': + if (coll instanceof PArray) this.allSet(coll.data, 'SORT', first, s).sort(compareValues); else coll.sort(compareValues); + text = `${name} is now in order, smallest first.`; + this.noteWrite(root.name, null); + break; + default: text = s.name; + } + this.step(s, 'builtin', text); + } + + /* ---------- expressions ---------- */ + evaluate(e: Expr): Value { + switch (e.kind) { + case 'Num': { + if (!isFloat32(e.type)) return this.finiteCheck(e.float ? e.value : e.value + 0, e); // -0 written as a whole number is 0 + // a decimal that becomes a FLOAT32 is rounded once, from its text (kept on the node: the value is fixed) + if (e.single === undefined) e.single = e.text ? (e.value < 0 || Object.is(e.value, -0) ? -1 : 1) * float32OfDecimal(e.text) : Math.fround(e.value); + return this.finiteCheck(e.single, e); + } + case 'Str': case 'Char': case 'Bool': return e.value; + case 'Name': { + if (up(e.name) === 'RESULT' && e.sym?.kind === 'result') { + const s = this.frame.vars.get('RESULT'); + if (!s || s.value === UNSET) throw new PseudoError('value', 'RESULT has a value only after the function returns.', e); + return s.value; + } + return this.read(e); + } + case 'Blank': throw new PseudoError('value', `<${e.text}> is a blank still to fill in.`, e); + case 'Unary': { + const v = this.evaluate(e.arg); + if (e.op === 'NOT') return !v; + return this.sized(e, this.intCheck(-(v as number), e)); + } + case 'Binary': return this.binary(e); + case 'Chain': { + let left = this.evaluate(e.operands[0]); + for (let i = 0; i < e.ops.length; i++) { + const right = e.ops[i] === 'IN' ? this.evaluate(e.operands[i + 1]) : this.evaluate(e.operands[i + 1]); + if (!this.compare(e.ops[i], left, right, e, e.operands[i], e.operands[i + 1])) return false; + left = right; + } + return true; + } + case 'Index': return this.index(e); + case 'Field': { + const r = this.evaluate(e.target); + const v = (r as PRecord).fields.get(up(e.field)); // the checker held the target to a record + if (v === UNSET) throw new PseudoError('unset', `Field ${e.field} of ${sourceOf(e.target)} has not been given a value yet.`, e); + return v; + } + case 'CallExpr': return this.callExpr(e); + case 'ListLit': return e.items.map((x) => cloneValue(this.evaluate(x))); + case 'SetLit': return new Set(e.items.map((x) => this.evaluate(x))); + case 'MapLit': return new Map(e.entries.map((x) => [this.evaluate(x.key), cloneValue(this.evaluate(x.value))])); + case 'Range': { + const lo = this.evaluate(e.lo); const hi = this.evaluate(e.hi); + return { range: true, lo, hi } as RangeValue; // the checker held both ends to whole numbers + } + case 'SetBuilder': { + const out = new Set(); + for (const x of this.domainValues(e.domain)) if (this.visit(e) && this.withBound(e.var, x, () => this.evaluate(e.body))) out.add(x); + return out; + } + case 'Quant': { + const dom = this.domainValues(e.domain); + if (e.q === 'FORALL') { for (const x of dom) if (this.visit(e) && !this.withBound(e.var, x, () => this.evaluate(e.body))) return false; return true; } + if (e.q === 'EXISTS') { for (const x of dom) if (this.visit(e) && this.withBound(e.var, x, () => this.evaluate(e.body))) return true; return false; } + // each partial sum is checked as + checks its result: a whole number past 2^53 - 1, or a FLOAT that is not + // finite, stops the run (library/pseudo/prover/sum-past-exact.pseudo, sum-not-finite.pseudo) + let sum = 0; + for (const x of dom) sum = this.visit(e) && this.intCheck(sum + (this.withBound(e.var, x, () => this.evaluate(e.body)) as number), e); + return sum; + } + default: throw new PseudoError('value', 'This expression could not be evaluated.', e); + } + } + + /** + * One position visited by FORALL, EXISTS, SUM or a set written by rule. These are not steps of the program, so + * the step limit does not see them; nested, they can still run without end (SUM I IN [0, N) : SUM J IN [0, N) : 1). + * A run may visit 100 positions for every step it is allowed (or `maxVisits`), and stops past that. Always answers + * true. + */ + visit(e: Expr): true { + if (++this.visited > (this.opts.maxVisits ?? this.opts.maxSteps * 100)) { + throw new PseudoError('limit', `Stopped after ${this.visited.toLocaleString('en-US')} positions visited by FORALL, EXISTS, SUM and sets written by rule. One inside another multiplies the positions; check the ranges, or give the program a smaller input.`, e); + } + return true; + } + + intCheck(v: number, e: Expr): number { + this.finiteCheck(v, e); + if (v === 0 && e.type?.k === 'INTEGER') return 0; // 0 * -1 and -0 are the whole number 0, with no sign + if (typeof v === 'number' && Number.isInteger(v) && !Number.isSafeInteger(v) && e.type?.k === 'INTEGER') { + throw new PseudoError('overflow', v > 0 + ? `${sourceOf(e)} comes to more than 9,007,199,254,740,991, the largest whole number this lab can hold exactly.` + : `${sourceOf(e)} comes to less than -9,007,199,254,740,991, the most negative whole number this lab can hold exactly.`, e); + } + return v; + } + + /** + * A FLOAT result that is not a finite number stops the run, as a division by + * zero does. The prover reasons about FLOAT values as numbers (X == X, and + * X < Y or X >= Y), which infinities and NaN are not; stopping here keeps every + * run inside what a proof covers (library/pseudo/prover/float-overflow.pseudo). + */ + finiteCheck(v: V, e: Expr | null | undefined): V { + if (typeof v === 'number' && !Number.isFinite(v) && !this.ieee) { + // a FLOAT32 leaves its range long before a FLOAT does, and the message names the limit that was passed + const [limit, what] = isFloat32(e?.type) ? ['3.4 × 10^38', 'FLOAT32'] : ['1.8 × 10^308', 'FLOAT']; + throw new PseudoError('overflow', Number.isNaN(v) + ? `${sourceOf(e)} has no number as its answer. That happens when a negative number is raised to a power that is not whole, or when a step on the way went past the largest ${what} this lab can hold, about ${limit}.` + : `${sourceOf(e)} comes to ${v > 0 ? 'more than' : 'less than minus'} about ${limit}, beyond the largest ${what} this lab can hold.`, e); + } + return v; + } + + binary(e: BinaryExpr): Value { + const op = e.op; + if (op === 'AND') { const l = this.evaluate(e.left); return l ? Boolean(this.evaluate(e.right)) : false; } + if (op === 'OR') { const l = this.evaluate(e.left); return l ? true : Boolean(this.evaluate(e.right)); } + // the operands are what the checker typed them: numbers for the arithmetic, text for &, any two values to compare + let l: any = this.evaluate(e.left); + let r: any = this.evaluate(e.right); + // FLOAT32 arithmetic: both sides are singles (a whole number is converted first), and the result is rounded + if (isFloat32(e.type) && ['+', '-', '*', '/', '^'].includes(op)) { l = Math.fround(l); r = Math.fround(r); } + switch (op) { + case '+': return this.sized(e, this.intCheck(l + r, e)); + case '-': return this.sized(e, this.intCheck(l - r, e)); + case '*': return this.sized(e, this.intCheck(l * r, e)); + case '/': + if (r === 0 && !(this.ieee && e.type?.k === 'FLOAT')) throw this.zeroError(e); + return this.sized(e, this.finiteCheck(l / r, e)); + case 'DIV': + if (r === 0) throw this.zeroError(e); + return this.sized(e, Math.floor(l / r) + 0); // + 0: a whole number is never minus zero + case 'MOD': + if (r === 0) throw this.zeroError(e); + // The remainder with the divisor's sign, exactly: l % r is exact for any two whole numbers, where + // l - r * FLOOR(l / r) rounds once the product passes 2^53 (9007199254740991 MOD -3 is -2, not -1). + { const m = l % r; return this.sized(e, (m !== 0 && (m < 0) !== (r < 0) ? m + r : m) + 0); } + case '^': { + if (e.type?.k === 'FLOAT' && !this.ieee) { + // say why there is no answer, as POW does, rather than call it an overflow + if (l === 0 && r < 0) throw new PseudoError('value', `${sourceOf(e)} has no answer: zero raised to a negative power divides by zero.`, e); + if (l < 0 && Number.isFinite(r) && !Number.isInteger(r)) throw new PseudoError('value', `${sourceOf(e)} has no real answer: a negative number is raised to a power that is not whole.`, e); + } + if (e.type?.k === 'INTEGER' && r < 0) throw new PseudoError('value', `${sourceOf(e)} raises a whole number to the power ${r}, which is not whole. Write ${sourceOf(e.left)} as a FLOAT, such as ${l}.0, for a fractional answer.`, e); + return this.sized(e, this.intCheck(e.type?.k === 'FLOAT' ? ieeePow(l, r) : l ** r, e)); + } + case '&': { + const joined = String(l) + String(r); + if (joined.length > MAX_TEXT) throw new PseudoError('value', `${sourceOf(e)} makes a text of more than ${MAX_TEXT.toLocaleString('en-US')} characters, longer than this lab holds.`, e); + return joined; + } + default: return this.compare(op, l, r, e, e.left, e.right); + } + } + + zeroError(e: BinaryExpr): PseudoError { + const names = [...namesIn(e.right)]; + const which = names.length ? ` (${names.map((n) => { const s = this.frame.vars.get(up(n)); return s ? `${n} is ${formatValue(s.value, s.type)}` : n; }).join(', ')})` : ''; + return new PseudoError('divide', `${sourceOf(e.right)} is 0${which}, and dividing by zero has no answer. Guard the division with an IF, or state with REQUIRE that it cannot happen.`, e); + } + + compare(op: string, l: Value, r: Value, e: BinaryExpr | Expr, le?: Expr, re?: Expr): boolean { + if (op === 'IN') return this.member(l, r, e); + this.work.compares++; + if (typeof l === 'string' && typeof r === 'string' && textLength(l) === 1 && textLength(r) === 1) this.work.letterCompares++; + if (typeof l === 'number' && typeof r === 'number') { + switch (op) { case '<': return l < r; case '<=': return l <= r; case '>': return l > r; case '>=': return l >= r; default: } + } + switch (op) { + case '==': return valuesEqual(l, r); + case '!=': return !valuesEqual(l, r); + case '<': return compareValues(l, r) < 0; + case '<=': return compareValues(l, r) <= 0; + case '>': return compareValues(l, r) > 0; + case '>=': return compareValues(l, r) >= 0; + default: throw new PseudoError('value', `Unknown comparison ${op}.`, e); + } + } + + member(x: Value, coll: Value, e: BinaryExpr | Expr): boolean { + if (coll && (coll as RangeValue).range) return Number.isInteger(x) && (x as number) >= (coll as RangeValue).lo && (x as number) < (coll as RangeValue).hi; + if (coll instanceof Set) return coll.has(x); + if (coll instanceof Map) return coll.has(x); + if (Array.isArray(coll)) return coll.some((v) => valuesEqual(v, x)); + if (coll instanceof PArray) return coll.data.some((v) => valuesEqual(v, x)); + if (typeof coll === 'string') return chars(coll).includes(x as string); + // typed-port: `e` is the comparison chain for 0 <= I IN S, which has no `right`, so this would say "inside ?"; the + // checker turns away IN on what is not a collection, so no run gets here. + throw new PseudoError('value', `IN cannot look inside ${sourceOf((e as BinaryExpr).right)}.`, e); + } + + checkPosition(length: number, i: Value, e: PositionAt, what: string, name?: string): asserts i is number { + if (Number.isInteger(i) && (i as number) >= 0 && (i as number) < length) return; + const tg = e.target || e; + const nm = name || (['Binary', 'Chain', 'Unary'].includes(tg.kind) ? `(${sourceOf(tg)})` : sourceOf(tg)); + const where = this.valuesNote(e.indices?.[0]); + if (length === 0) throw new PseudoError('index', `${sourceOf(e)} asks for position ${i as number}, and ${nm} is empty, so it has no positions at all.${where}`, e, { index: i, length }); + if (i === length) throw new PseudoError('index', `${sourceOf(e)} asks for position ${i as number}, and ${nm} holds ${plural(length, what === 'STRING' ? 'character' : 'value')}, at positions 0 to ${length - 1}. Position ${length} is one past the end: a length used where a position belongs.${where}`, e, { index: i, length }); + throw new PseudoError('index', `${sourceOf(e)} asks for position ${i as number}, and ${nm} has positions 0 to ${length - 1} only.${where}`, e, { index: i, length }); + } + + /** " Here I is 6 and J is 1." for the names inside an index expression. */ + valuesNote(expr: Expr | undefined): string { + if (!expr) return ''; + const names = [...namesIn(expr)]; + const parts: string[] = []; + for (const n of names) { const s = this.frame.vars.get(up(n)); if (s && s.value !== UNSET) parts.push(`${s.name} is ${formatValue(s.value, s.type)}`); } + return parts.length ? ` Here ${parts.join(' and ')}.` : ''; + } + + flatIndex(arr: PArray, idx: Value[], e: IndexExpr): number { + if (arr.dims.length === 2) { + const [r, c] = idx as number[]; + if (!Number.isInteger(r) || r < 0 || r >= arr.dims[0]) throw new PseudoError('index', `${sourceOf(e)} asks for row ${r}, and ${sourceOf(e.target)} has rows 0 to ${arr.dims[0] - 1}.${this.valuesNote(e.indices[0])}`, e); + if (!Number.isInteger(c) || c < 0 || c >= arr.dims[1]) throw new PseudoError('index', `${sourceOf(e)} asks for column ${c}, and ${sourceOf(e.target)} has columns 0 to ${arr.dims[1] - 1}.${this.valuesNote(e.indices[1])}`, e); + return r * arr.dims[1] + c; + } + this.checkPosition(arr.dims[0], idx[0], e, 'ARRAY'); + return idx[0]; + } + + keyError(e: IndexExpr, key: Value): PseudoError { + const kt = e.target.type?.key; + const shown = formatValue(key, kt, { bareChar: true }); + return new PseudoError('key', `${sourceOf(e.target)} has no key ${shown}. Asking for a key that is not there has no answer: test with ${shown} IN ${sourceOf(e.target)} first.`, e, { key }); + } + + index(e: IndexExpr): Value { + const mark = this.reads.length; + const c = this.evaluate(e.target); + const idx = e.indices.map((i) => this.evaluate(i)); + if (!this.quiet) { + let root = e.target; + while (root.kind === 'Index' || root.kind === 'Field') root = root.target; + if (root.kind === 'Name' && e.target.kind === 'Name') { + const name = this.slot(root.name, root).name; + // L[I] reads one position, not the whole of L: replace the whole-name read that evaluating L recorded + const whole = this.reads.findIndex((r, k) => k >= mark && r.name === name && !r.path); + if (whole >= 0) this.reads.splice(whole, 1); + this.reads.push({ name, path: idx, frame: this.frame.id }); + } + } + if (typeof c === 'string') { + const cs = chars(c); + this.checkPosition(cs.length, idx[0], e, 'STRING'); + return cs[idx[0]]; + } + if (Array.isArray(c)) { + this.checkPosition(c.length, idx[0], e, 'LIST'); + const v = c[idx[0]]; + if (v === UNSET) throw new PseudoError('unset', `Position ${idx[0]} of ${sourceOf(e.target)} has not been given a value yet.`, e); + return v; + } + if (c instanceof PArray) { + const v = c.data[this.flatIndex(c, idx, e)]; + if (v === UNSET) throw new PseudoError('unset', `${sourceOf(e)} has not been given a value yet. An ARRAY arrives holding nothing you can rely on, so set every position before reading it, usually with a loop.`, e, { index: idx }); + return v; + } + if (c instanceof Map) { + if (!c.has(idx[0])) throw this.keyError(e, idx[0]); + return c.get(idx[0]); + } + throw new PseudoError('value', `${sourceOf(e.target)} has no positions.`, e); + } + + callExpr(e: CallExprNode): Value { + if (e.isOld) { + const n = e.args[0]; + const old = this.frame.old; + if (old && n.kind === 'Name' && old.has(up(n.name))) return old.get(up(n.name)); + return this.evaluate(n); + } + if (e.sub || this.program.subs.get(up(e.name)) || this.program.libs?.get(up(e.name))) { + const sub = (e.sub || this.program.subs.get(up(e.name)) || this.program.libs.get(up(e.name)))!; + const slots = this.argSlots(sub, e.args); + return this.invoke(sub, slots, e, false); + } + if (e.convert) return this.convert(e, this.evaluate(e.args[0])); + const args = e.args.map((a): Value | PType => (a.kind === 'TypeRef' ? T[a.name] : this.evaluate(a))); + return this.finiteCheck(this.builtin(up(e.name), args, e), e); + } + + /** INT32(X), FLOAT32(X), ...: a FLOAT loses its fraction toward zero for a whole-number type; out of range stops. */ + convert(e: CallExprNode, v: Value): Value { + const to = SIZED[up(e.name)]; + if (to.k === 'INTEGER') { + if (!Number.isFinite(v)) throw new PseudoError('value', `${sourceOf(e)} has no answer: ${formatValue(v)} is not a number that has a whole part.`, e); + return this.fitInt(to, Math.trunc(v as number) || 0, e, 'convert'); + } + return isFloat32(to) ? this.finiteCheck(Math.fround(v as number), e) : v; + } + + /** A whole-number result that must still fit exactly. */ + checkInt(v: number, e: Expr): number { + if (!Number.isSafeInteger(v)) { + if (Number.isNaN(v)) throw new PseudoError('overflow', `${sourceOf(e)} has no whole number as its answer: the value is not a number.`, e); + throw new PseudoError('overflow', v < 0 + ? `${sourceOf(e)} comes to less than -9,007,199,254,740,991, the most negative whole number this lab can hold exactly.` + : `${sourceOf(e)} comes to more than 9,007,199,254,740,991, the largest whole number this lab can hold exactly.`, e); + } + return v; + } + + /** The INTEGER a FLOAT was turned into (INT, ROUND, FLOOR, CEIL): one this lab holds exactly, with zero for minus zero. */ + wholeOf(v: number, e: CallExprNode): number { + if (typeof e.args?.[0]?.type?.k === 'string' && e.args[0].type.k !== 'FLOAT') return v; // a whole number is already its own + return this.checkInt(v, e) === 0 ? 0 : v; + } + + /** The values of a collection, once every position has one: an ARRAY position never given a value stops the run. */ + allSet(xs: X[], name: string, arg: Expr, at: Place): X[] { + for (const v of xs) if (v === UNSET) throw new PseudoError('unset', `${name}(${sourceOf(arg)}): a position of ${sourceOf(arg)} has not been given a value yet.`, at); + return xs; + } + + /** A whole-number result checked against its type: a sized type's range, or what INTEGER holds exactly. */ + intResult(e: CallExprNode, v: number): number { return isSizedInt(e.type) ? this.fitInt(e.type, v, e, 'result') : this.checkInt(v, e); } + + builtin(name: string, a: Array, e: CallExprNode): Value { + // each built-in's arguments are what the checker held them to (numbers, text, a collection, a type for + // MAX_VALUE and WRAP), and each case below uses them as such + const [x, y, z] = a as any[]; + const a4 = (): any => a[3]; + switch (name) { + case 'LENGTH': + if (typeof x === 'string') return textLength(x); + if (Array.isArray(x)) return x.length; + if (x instanceof PArray) return x.dims[0]; + if (x instanceof Set || x instanceof Map) return x.size; + throw new PseudoError('value', `LENGTH cannot measure ${formatValue(x)}.`, e); + case 'ROWS': return x.dims[0]; + case 'COLUMNS': return x.dims[1]; + // a FLOAT becoming a whole number: the result is an INTEGER, so it must be one this lab holds exactly (INT(6.02e23) stops the + // run, as 2 ^ 80 does), and it is never minus zero + case 'INT': return this.wholeOf(Math.trunc(x), e); + // halves away from zero. Math.round is exact where Math.floor(x + 0.5) rounds twice: ROUND(0.49999999999999994) is 0, and + // ROUND(4503599627370497.0) is itself + case 'ROUND': return this.wholeOf(Math.sign(x) * Math.round(Math.abs(x)), e); + case 'ABS': return this.sized(e, Math.abs(x)); // ABS of the lowest INT8 is 128, which no INT8 holds + case 'MIN': case 'MAX': { + let list = a as Value[]; // two or more numbers, or (below) the values of one collection + if (a.length === 1) list = Array.isArray(x) ? x : x instanceof PArray ? x.data : [...x]; + if (!list.length) throw new PseudoError('value', `${name} of an empty collection has no answer.`, e); + if (a.length === 1) this.allSet(list, name, e.args[0], e); + // with USE IEEE a NaN among the values makes the answer NaN, whatever its place (IEEE 754-2019 minimum and + // maximum); comparing in turn kept or dropped it depending on where it stood + if (list.some((v) => Number.isNaN(v))) return NaN; + const r = list.reduce((m, v) => ((name === 'MIN' ? compareValues(v, m) < 0 : compareValues(v, m) > 0) ? v : m)); + // a FLOAT32 answer is a FLOAT32: a whole number among the values is rounded to one + return isFloat32(e.type) && typeof r === 'number' ? Math.fround(r) : r; + } + case 'SQRT': if (x < 0 && !this.ieee) throw new PseudoError('value', `SQRT(${formatValue(x)}) has no real answer: the number is negative.`, e); return Math.sqrt(x); + case 'EXP': return Math.exp(x); + case 'LOG': case 'LOG2': + if (x <= 0 && !this.ieee) throw new PseudoError('value', `${name}(${formatValue(x)}) has no answer: a logarithm needs a number above 0.${x === 0 ? ' A probability of 0 often means a pseudocount is missing.' : ''}`, e); + return name === 'LOG' ? Math.log(x) : Math.log2(x); + // a number in [0, X), X itself never: for the smallest FLOATs the product can round up to X, and then it is 0 + case 'RAND': { const r = this.random() * x; return r === x && x !== 0 ? 0 : r; } + case 'RANDINT': + if (!(x < y)) throw new PseudoError('value', `RANDINT(${x}, ${y}) picks from [${x}, ${y}), which is empty.`, e); + return x + Math.floor(this.random() * (y - x)); + case 'SLICE': { + const len = typeof x === 'string' ? textLength(x) : x.length; + if (!(Number.isInteger(y) && Number.isInteger(z) && y >= 0 && y <= z && z <= len)) { + throw new PseudoError('index', `SLICE(${sourceOf(e.args[0])}, ${y}, ${z}) asks for positions [${y}, ${z}), and ${sourceOf(e.args[0])} has positions [0, ${len}). Both ends have to lie in 0 to ${len}, with the first no larger than the second.`, e); + } + return typeof x === 'string' ? chars(x).slice(y, z).join('') : x.slice(y, z).map(cloneValue); + } + // a CHAR stays one character: a letter whose capital is two letters ('ß', "SS") is left as it is + case 'UPPER': case 'LOWER': { + const r = name === 'UPPER' ? x.toUpperCase() : x.toLowerCase(); + return e.args[0].type?.k === 'CHAR' && textLength(r) !== 1 ? x : r; + } + case 'SPLIT': { + if (!String(y).length) throw new PseudoError('value', 'SPLIT needs a separator with at least one character.', e); + return String(x).split(String(y)); + } + case 'JOIN': return x.map((v: Value) => String(v)).join(String(y)); + case 'INDEX_OF': { + if (typeof x === 'string') { const i = x.indexOf(String(y)); return i < 0 ? -1 : textLength(x.slice(0, i)); } + const i = x.findIndex((v: Value) => valuesEqual(v, y)); + return i; + } + case 'REPLACE': { + if (!String(y).length) throw new PseudoError('value', 'REPLACE needs something to look for: the piece to replace cannot be empty.', e); + return String(x).split(String(y)).join(String(z)); + } + case 'ORD': return x.codePointAt(0); + case 'CHR': + // the halves of a surrogate pair are not characters: two of them joined would count as one + if (!Number.isInteger(x) || x < 0 || x > 0x10FFFF || (x >= 0xD800 && x <= 0xDFFF)) throw new PseudoError('value', `CHR(${x}) names no character.`, e); + return String.fromCodePoint(x); + case 'TO_STRING': return outputText(x, e.args[0].type); + case 'TO_INTEGER': { + const s = String(x).trim(); + if (!/^[+-]?\d+$/.test(s)) throw new PseudoError('value', `TO_INTEGER(${formatValue(x)}) has no answer: the text is not a whole number.`, e); + return this.checkInt(Number(s), e); + } + case 'TO_FLOAT': { + if (typeof x === 'number') return x; + const v = decimalText(String(x).trim(), this.ieee); + if (v === null) throw new PseudoError('value', `TO_FLOAT(${formatValue(x)}) has no answer: the text is not a decimal number.`, e); + return this.finiteCheck(v, e); + } + case 'KEYS': return sortedKeys(x.keys()); + case 'SORTED': return this.allSet(x instanceof Set ? [...x] : x instanceof PArray ? x.data.slice() : x.slice(), name, e.args[0], e).sort(compareValues); + // ---- numbers + case 'FLOOR': return this.wholeOf(Math.floor(x), e); + case 'CEIL': return this.wholeOf(Math.ceil(x), e); + case 'LOG10': + if (x <= 0 && !this.ieee) throw new PseudoError('value', `LOG10(${formatValue(x)}) has no answer: a logarithm needs a number above 0.`, e); + return Math.log10(x); + case 'SIN': return Math.sin(x); + case 'COS': return Math.cos(x); + case 'TAN': return Math.tan(x); + case 'ATAN2': return Math.atan2(x, y); + case 'PI': return Math.PI; + case 'CHOOSE': { + if (y < 0 || y > x) return 0; + // Exact whole-number arithmetic: the product of this step can pass 2^53 while the count itself still fits + // (CHOOSE(56, 23)). The count only grows from step to step, so the loop ends at the first one that cannot fit, + // however large K is. + const k = Math.min(y, x - y); const base = BigInt(x - k); + let r = 1n; + for (let i = 1; i <= k; i++) { + r = r * (base + BigInt(i)) / BigInt(i); + if (r > BigInt(EXACT_MAX)) throw new PseudoError('overflow', `CHOOSE(${x}, ${y}) comes to more than 9,007,199,254,740,991, the largest whole number this lab can hold exactly.`, e); + } + return Number(r); + } + case 'RANDN': { const u = 1 - this.random(), v = this.random(); return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v); } + // ---- statistics over a LIST or ARRAY of numbers + case 'SUM_OF': case 'MEAN_OF': case 'VARIANCE_OF': case 'STDEV_OF': case 'MEDIAN_OF': case 'NORM_OF': { + const xs = x instanceof PArray ? x.data : x; + for (const v of xs) if (v === UNSET) throw new PseudoError('unset', `${name}(${sourceOf(e.args[0])}): a position of ${sourceOf(e.args[0])} has not been given a value yet.`, e); + const n = xs.length; + const el = e.args[0].type?.of; + const isFloat = el?.k === 'FLOAT' || xs.some((v: Value) => !Number.isInteger(v)); + if (name === 'SUM_OF' && (isFloat32(el) || isSizedInt(el))) { + // a running total kept in the values' own type, as a loop with a total of that type would keep it + let t = 0; + for (const v of xs) t = isFloat32(el) ? Math.fround(t + v) : this.fitInt(el, t + v, e, 'result'); + return this.finiteCheck(t, e); + } + // a sum of whole numbers is checked at every step, as + checks its result: a partial sum past 2^53 - 1 is + // rounded, and the total can come back inside the range wrong (SUM_OF([9007199254740991, 2, -3])) + const total = () => (isFloat ? xs.reduce((m: number, v: number) => m + v, 0) : xs.reduce((m: number, v: number) => this.checkInt(m + v, e), 0)); + if (name === 'SUM_OF') return total(); + if (name === 'NORM_OF') return Math.sqrt(xs.reduce((m: number, v: number) => m + v * v, 0)); + if (n === 0) throw new PseudoError('value', `${name} of an empty collection has no answer.`, e); + const mean = total() / n; + if (name === 'MEAN_OF') return mean; + if (name === 'MEDIAN_OF') { const srt = xs.slice().sort((p: number, q: number) => p - q); return n % 2 ? srt[(n - 1) / 2] : (srt[n / 2 - 1] + srt[n / 2]) / 2; } + if (n < 2) throw new PseudoError('value', `${name} needs at least two values; ${sourceOf(e.args[0])} holds one.`, e); + const varc = xs.reduce((m: number, v: number) => m + (v - mean) * (v - mean), 0) / (n - 1); + return name === 'VARIANCE_OF' ? varc : Math.sqrt(varc); + } + case 'DOT': { + const xs = x instanceof PArray ? x.data : x; const ys = y instanceof PArray ? y.data : y; + if (xs.length !== ys.length) throw new PseudoError('value', `DOT needs two collections of the same length, and these hold ${xs.length} and ${ys.length} values.`, e); + this.allSet(xs, name, e.args[0], e); this.allSet(ys, name, e.args[1], e); + if (isFloat32(e.type) || isSizedInt(e.type)) { + let t = 0; + for (let i = 0; i < xs.length; i++) t = isFloat32(e.type) ? Math.fround(t + Math.fround(xs[i] * ys[i])) : this.fitInt(e.type, t + this.fitInt(e.type, xs[i] * ys[i], e, 'result'), e, 'result'); + return t; + } + // whole numbers: every product and every partial sum is checked, as * and + check theirs (a product past + // 2^53 - 1 is rounded, and the total can come back inside the range wrong); FLOATs: the left-to-right sum + // of the rounded products a loop would compute + let t = 0; + if (e.type?.k === 'INTEGER') for (let i = 0; i < xs.length; i++) t = this.checkInt(t + this.checkInt(xs[i] * ys[i], e), e); + else for (let i = 0; i < xs.length; i++) t += xs[i] * ys[i]; + return t; + } + case 'FILL': { + if (!Number.isInteger(x) || x < 0) throw new PseudoError('value', `FILL(${formatValue(x)}, ...) needs a whole number of copies, 0 or more.`, e); + if (x > MAX_VALUES) throw new PseudoError('value', `FILL(${x}, ...) asks for more values than this lab gives one collection (${MAX_VALUES.toLocaleString('en-US')}).`, e); + const out: Value[] = []; for (let i = 0; i < x; i++) out.push(cloneValue(y)); return out; + } + // ---- numerical work + case 'ASIN': case 'ACOS': + if ((x < -1 || x > 1) && !this.ieee) throw new PseudoError('value', `${name}(${formatValue(x)}) has no answer: ${name === 'ASIN' ? 'a sine' : 'a cosine'} lies between -1 and 1.`, e); + return name === 'ASIN' ? Math.asin(x) : Math.acos(x); + case 'ATAN': return Math.atan(x); + case 'SINH': return Math.sinh(x); + case 'COSH': return Math.cosh(x); + case 'TANH': return Math.tanh(x); + case 'HYPOT': return Math.hypot(x, y); + case 'POW': + if (x < 0 && !Number.isInteger(y) && !this.ieee) throw new PseudoError('value', `POW(${formatValue(x)}, ${formatValue(y)}) has no real answer: a negative number raised to a power that is not whole.`, e); + if (x === 0 && y < 0 && !this.ieee) throw new PseudoError('value', `POW(0, ${formatValue(y)}) has no answer: it divides by zero.`, e); + return ieeePow(x, y); + case 'CBRT': return Math.cbrt(x); + case 'EXPM1': return Math.expm1(x); + case 'LOG1P': + if (x <= -1 && !this.ieee) throw new PseudoError('value', `LOG1P(${formatValue(x)}) has no answer: 1 + X has to be above 0.`, e); + return Math.log1p(x); + case 'E': return Math.E; + case 'TRUNC': return Math.trunc(x); + case 'SIGN': return x > 0 ? 1 : x < 0 ? -1 : 0; + case 'COPYSIGN': return copySign(x, y); + case 'CLAMP': + if (y > z) throw new PseudoError('value', `CLAMP(${formatValue(x)}, ${formatValue(y)}, ${formatValue(z)}) has no answer: the low end is above the high end.`, e); + return isFloat32(e.type) ? Math.fround(x < y ? y : x > z ? z : x) : x < y ? y : x > z ? z : x; + case 'FMA': return fma(x, y, z, isFloat32(e.type)); + case 'IS_NAN': return Number.isNaN(x); + case 'IS_INF': return x === Infinity || x === -Infinity; + case 'IS_FINITE': return Number.isFinite(x); + case 'INF': return Infinity; + case 'NAN': return NaN; + case 'ULP': return ulp(x, isFloat32(e.type)); + case 'NEXT_AFTER': return nextAfter(isFloat32(e.type) ? Math.fround(x) : x, y, isFloat32(e.type)); + case 'IS_CLOSE': { + const rel = z; const tol = a4(); + if (rel < 0 || tol < 0) throw new PseudoError('value', 'IS_CLOSE needs tolerances of 0 or more.', e); + if (x === y) return true; + if (!Number.isFinite(x) || !Number.isFinite(y)) return false; + return Math.abs(x - y) <= Math.max(rel * Math.max(Math.abs(x), Math.abs(y)), tol); + } + case 'EXACT': return exactDecimal(x); + case 'MAX_VALUE': case 'MIN_VALUE': { + const ty = x; + if (ty.k === 'INTEGER') { const [lo, hi] = intRange(ty); return name === 'MAX_VALUE' ? hi : lo; } + const m = isFloat32(ty) ? 3.4028234663852886e38 : Number.MAX_VALUE; + return name === 'MAX_VALUE' ? m : -m; + } + case 'EPSILON': return isFloat32(x) ? 2 ** -23 : Number.EPSILON; + case 'GCD': return this.intResult(e, gcd(x, y)); // GCD(-128, 0) is 128 + case 'LCM': return x === 0 || y === 0 ? 0 : this.intResult(e, Math.abs((x / gcd(x, y)) * y)); + case 'BIT_AND': return this.intResult(e, bitOp('AND', x, y)); + case 'BIT_OR': return this.intResult(e, bitOp('OR', x, y)); + case 'BIT_XOR': return this.intResult(e, bitOp('XOR', x, y)); + case 'BIT_NOT': { + const ty = e.args[0].type; + // every bit of a UINT64 flipped is 2^64 - 1 - A, always past what this lab holds exactly + if (isSizedInt(ty) && !ty.signed && ty.bits >= 64) { + // say the exact answer (a double near 2^64 would print as 18446744073709552000) + const exact = (2n ** 64n - 1n - BigInt(x)).toLocaleString('en-US'); + throw new PseudoError('overflow', `${sourceOf(e)} comes to ${exact} (every bit of ${formatValue(x)} flipped in 64 bits), and this lab holds a ${ty.name} exactly only from 0 to 9,007,199,254,740,991. A sized type stops the run instead of wrapping around; write WRAP(..., ${ty.name}) when wrapping is what you mean.`, e, { type: ty.name }); + } + return isSizedInt(ty) && !ty.signed ? intRange(ty)[1] - x : -x - 1; + } + case 'SHIFT_LEFT': case 'SHIFT_RIGHT': + if (!Number.isInteger(y) || y < 0) throw new PseudoError('value', `${name} moves bits by 0 or more places, and ${formatValue(y)} is not that.`, e); + // a nonzero number moved up more than 64 places fits no type: say so without building a number that long + // (SHIFT_LEFT(1, 2147483648) asked the engine for a whole number of two billion bits) + if (name === 'SHIFT_LEFT' && x !== 0 && y > 64) return this.intResult(e, x * 2 ** 70); + return this.intResult(e, bitOp(name === 'SHIFT_LEFT' ? 'SHL' : 'SHR', x, y)); + case 'WRAP': return wrap(x, y.bits, y.signed); + default: throw new PseudoError('value', `${name} is not a built-in function.`, e); + } + } + + /** No routine is called and no random number drawn anywhere in it, so evaluating it again changes nothing and gives the same value. */ + replayable(e: LooseExpr | null | undefined): boolean { + let ok = true; + const seen = new WeakSet(); + const visit = (x: unknown): void => { + if (!ok || !x || typeof x !== 'object' || seen.has(x)) return; + seen.add(x); + if (Array.isArray(x)) { x.forEach(visit); return; } + const n = x as LooseExpr; + if (n.kind === 'CallExpr' && !n.isOld) { + const routine = (n as { sub?: unknown }).sub || this.program.subs.get(up(n.name)) || this.program.libs?.get(up(n.name)); + if (routine || !n.builtin || ['RAND', 'RANDINT', 'RANDN'].includes(n.builtin)) { ok = false; return; } + } + for (const [k, v] of Object.entries(n)) if (k !== 'type' && k !== 'sub' && v && typeof v === 'object') visit(v); + }; + visit(e); + return ok; + } + + /** A condition in words, with the values its names hold right now. */ + describeCondition(e: Expr | null): string { + const src = sourceOf(e); + return this.quietly(() => { + try { + const parts: string[] = []; + const visit = (x: LooseExpr | null | undefined): void => { + if (!x || parts.length >= 4) return; + if (x.kind === 'Index' || x.kind === 'Field' || (x.kind === 'CallExpr' && x.builtin === 'LENGTH')) { + // the condition has been evaluated once, and the run goes on: a part that calls a routine or draws a + // random number is not evaluated a second time to describe it (IF LENGTH(F(1)) > 0 ran F twice) + if (!this.replayable(x)) return; + try { parts.push(`${sourceOf(x)} is ${formatValue(this.evaluate(x as Expr), x.type, { bareChar: true, max: 6 })}`); } catch { /* not readable yet */ } + return; + } + if (x.kind === 'Name') { + const s = this.frame.vars.get(up(x.name)) || this.globals.get(up(x.name)); + if (s && s.value !== UNSET && !parts.some((p) => p.startsWith(`${s.name} is`))) parts.push(`${s.name} is ${formatValue(s.value, s.type, { bareChar: true, max: 6 })}`); + return; + } + if (x.kind === 'Quant' || x.kind === 'SetBuilder') return; + for (const k of ['left', 'right', 'arg'] as const) if (x[k]) visit(x[k]); + if (x.operands) x.operands.forEach(visit); + if (x.args) x.args.forEach(visit); + }; + visit(e); + return parts.length ? `${src} (${parts.join(', ')})` : src; + } catch { return src; } + }); + } +} + +/** Every name an expression reads, not counting a quantifier's own. */ +export function namesIn(e: LooseExpr | null | undefined, out: Set = new Set(), bound: Set = new Set()): Set { + if (!e || typeof e !== 'object') return out; + if (e.kind === 'Name') { if (!bound.has(up(e.name))) out.add(e.name!); return out; } + if (e.kind === 'Quant' || e.kind === 'SetBuilder') { + namesIn(e.domain, out, bound); + const b = new Set(bound); b.add(up(e.var)); + namesIn(e.body, out, b); + return out; + } + for (const k of ['left', 'right', 'arg', 'target', 'lo', 'hi', 'domain', 'body'] as const) if (e[k]) namesIn(e[k], out, bound); + for (const k of ['operands', 'indices', 'args', 'items'] as const) if (Array.isArray(e[k])) for (const x of e[k]) namesIn(x, out, bound); + if (Array.isArray(e.entries)) for (const x of e.entries) { namesIn(x.key, out, bound); namesIn(x.value, out, bound); } + return out; +} + +function hasCall(e: LooseExpr | null | undefined): boolean { + if (!e || typeof e !== 'object') return false; + if (e.kind === 'CallExpr') return true; + return (['target', 'left', 'right', 'arg'] as const).some((k) => hasCall(e[k])) || (e.indices || []).some(hasCall); +} + +function usesOld(ens: ContractStmt): boolean { + let found = false; + const visit = (x: LooseExpr | null | undefined): void => { + if (!x || typeof x !== 'object' || found) return; + if (x.kind === 'CallExpr' && x.isOld) { found = true; return; } + for (const v of Object.values(x)) if (v && typeof v === 'object') { if (Array.isArray(v)) v.forEach(visit); else if ((v as LooseExpr).kind) visit(v as LooseExpr); } + }; + visit(ens.expr); + return found; +} + +function targetPath(target: Expr, key: Value): Value[] { return Array.isArray(key) ? key : [key]; } + +/* ------------------------------------------------------------------ */ +/* Entry point */ +/* ------------------------------------------------------------------ */ + +export function run(program: Program, options: RunOptions = {}): Trace { + const it = new Interpreter(program, options); + let error: PseudoError | null = null; + let result: Value; + let truncated = false; + try { + if (options.call) result = it.runCall(options.call.name, options.call.args || []); + else it.runMain(); + } catch (e) { + if (e instanceof PseudoError) { + error = e; + truncated = e.kind === 'limit'; + if (it.opts.record) { + it.steps.push({ line: e.line, kind: 'error', text: e.message, reads: it.reads, writes: [], depth: Math.max(0, it.frames.length - 1), sub: it.frame?.name, errorKind: e.kind }); + it.states.push(it.snapshot()); + } + } else if (e instanceof BreakSignal || e instanceof ContinueSignal || e instanceof ReturnSignal || e instanceof StopSignal) { + error = new PseudoError('value', 'A BREAK, CONTINUE, or RETURN was reached outside the block it belongs to.', { line: (e as { line?: number }).line }); + } else if (e instanceof RangeError && /call stack/i.test(e.message)) { + error = new PseudoError('depth', 'The calls went too deep for the lab to follow. A recursive subroutine needs a case that stops calling itself.', it.at); + } else if (e instanceof RangeError) { + // the engine underneath refused a size (a text or a collection too long for it) + error = new PseudoError('value', 'A value grew larger than this lab can hold. Check what makes it grow, or give the program a smaller input.', it.at); + } else { + // Anything else is a fault in the lab, not in the program: say so and stop, so that no caller has to expect an + // exception from a run (tests/pseudo-robustness.test.mjs). + error = new PseudoError('internal', `The lab could not carry out this line (${(e as Error)?.message || e}). That is a fault in the lab, not in your program.`, it.at); + } + } + const sub = options.call ? (program.subs.get(up(options.call.name)) || program.libs?.get(up(options.call.name))) : null; + return { + program, + steps: it.steps, + states: it.states, + count: it.steps.length, + output: it.output, + result, + resultType: sub?.returns, + error, + truncated, + work: it.work, + visits: it.visited, + hits: it.hits, + call: options.call || null, + unrecorded: it.unrecorded, + }; +} + +/** Evaluate a stand-alone expression, such as a test's arguments, against a program's constants. */ +export function evaluateConstant(program: Program, expr: Expr): Value { + const it = new Interpreter(program, { record: false, contracts: false }); + it.start(); + return it.evaluate(expr); +} diff --git a/src/engines/pseudo/lab.js b/src/engines/pseudo/lab.js new file mode 100644 index 0000000..51019f7 --- /dev/null +++ b/src/engines/pseudo/lab.js @@ -0,0 +1,226 @@ +/** + * pseudo/lab.js + * + * The lab: write pseudocode, see what the checker and the coach make of it + * as you type, run it on a call or on input, step through the run with any + * set of views, test it, try to prove it, and read it in Mojo. + * + * lab({ code, call, input, tests, examples, viewSet, useHash, remember }) -> node + * + * "Copy link" puts the lab's whole state in the page address. "Spec" prints + * the dictionary a lesson passes to pseudo.widget, so an experiment that + * works here becomes a widget on a page by pasting it into an {ojs} cell. + */ + +import { el, add, clear, panel, button, plural } from './dom.js'; +import { encodeState, decodeState } from '../shared/share.js'; +import { editor, SNIPPETS } from './editor.js'; +import { study, applyFixes, severityCounts } from './study.js'; +import { mountWidget, defaultCall, resolveCall } from './views.js'; +import { diagnosticsView, jobsView, proofView, testsView, checksView, flowView, translateView, cfgView } from './views-analysis.js'; +import { EXAMPLES } from './specs.js'; +import { formatValue } from './lang.js'; + +export const VIEW_SETS = { + 'step through': ['code', 'variables', 'explain', 'trace', 'output'], + 'data flow': ['code', 'variables', 'defuse', 'flow'], + flowchart: ['code', 'cfg', 'explain', 'variables'], + 'four jobs': [['code', { roles: true }], 'jobs', 'variables'], + 'prove it': ['code', ['proof', { open: true }], ['trace', { checks: true }]], + 'Mojo side by side': ['translate', 'variables', 'explain'], + 'how the work grows': [['growth', { span: 2 }]], + everything: ['code', 'variables', 'explain', 'output', 'trace', 'flow', 'jobs', 'cfg', 'translate'], +}; + +const DEFAULT_CODE = EXAMPLES.count.code; + +/** A spec as JavaScript source a lesson can paste. */ +export function specSource(spec) { + const js = (v) => (typeof v === 'string' ? (v.includes('\n') ? `\`${v.replace(/\\/g, '\\\\').replace(/`/g, '\\`').replace(/\$\{/g, '\\${')}\`` : JSON.stringify(v)) : JSON.stringify(v)); + const lines = Object.entries(spec).filter(([, v]) => v !== undefined && v !== '' && !(Array.isArray(v) && !v.length)).map(([k, v]) => ` ${k}: ${js(v)},`); + return `pseudo.widget({\n${lines.join('\n')}\n})`; +} + +export function lab(options = {}) { + const opts = { useHash: true, remember: true, examples: EXAMPLES, viewSet: 'step through', ...options }; + const root = el('div', { class: 'pl pl-lab' }); + let state = { code: opts.code ?? DEFAULT_CODE, call: opts.call ?? '', input: (opts.input || []).join('\n'), tests: opts.tests ?? EXAMPLES.count.tests, viewSet: opts.viewSet, title: '' }; + const hash = opts.useHash && typeof location !== 'undefined' ? /#lab=([\w-]+)/.exec(location.hash) : null; + if (hash) { try { state = { ...state, ...decodeState(hash[1]) }; } catch { /* a broken link falls back to the default */ } } + else if (opts.remember && !options.code) { try { const saved = localStorage.getItem('pseudo-lab'); if (saved) state = { ...state, ...JSON.parse(saved) }; } catch { /* storage may be unavailable */ } } + + /* ---------- controls ---------- */ + const exampleSel = el('select', { class: 'pl-select', 'aria-label': 'load an example' }, [ + el('option', { value: '', text: 'Load an example\u2026' }), + ...Object.entries(opts.examples).map(([k, s]) => el('option', { value: k, text: s.title || k })), + ]); + const ed = editor({ value: state.code, onChange: () => { state.code = ed.value; recheck(); persist(); }, onRun: () => runNow() }); + const snippetBar = el('div', { class: 'pl-snippets' }, [el('span', { class: 'pl-label', text: 'Insert: ' }), ...SNIPPETS.map(([name, text]) => button(name, () => ed.insert(text), { class: 'pl-snippet', title: text }))]); + const callInput = el('input', { class: 'pl-input', type: 'text', 'aria-label': 'call', placeholder: 'COUNT_BASE(\'A\', "GATTACA")', spellcheck: 'false' }); + const inputBox = el('textarea', { class: 'pl-input pl-lab-tests', rows: '2', 'aria-label': 'input lines', placeholder: 'what INPUT reads, one value per line', spellcheck: 'false' }); + const callRow = el('div', { class: 'pl-lab-callrow' }, [el('label', { class: 'pl-label', text: 'Call' }), callInput]); + const inputRow = el('div', { class: 'pl-lab-callrow' }, [el('label', { class: 'pl-label', text: 'Input' }), inputBox]); + const runBtn = button('Run', () => runNow(), { primary: true, title: 'Ctrl+Enter' }); + const viewSel = el('select', { class: 'pl-select', 'aria-label': 'views' }, Object.keys(VIEW_SETS).map((k) => el('option', { value: k, text: `show: ${k}` }))); + const fixBtn = button('Apply fixes', () => { + const r = applyFixes(ed.value, current.diags); + if (r.applied) { ed.value = r.source; state.code = r.source; recheck(); status(`${plural(r.applied, 'mechanical fix', 'mechanical fixes')} applied.`, 'ok'); } + }, { title: 'Fix keyword case, = for <-, quotes on a CHAR, and the other purely mechanical mistakes' }); + const linkBtn = button('Copy link', async () => { + persist(true); + try { await navigator.clipboard.writeText(location.href); status('Link copied: it holds the program, the call, the tests, and the views.', 'ok'); } catch { status('The address bar now holds a link to exactly this.', 'ok'); } + }); + const specBtn = button('Spec', () => { + specOut.textContent = `// paste into an {ojs} cell, after pseudo = await import("/static/js/pseudo/views.js")\n${specSource({ title: state.title || undefined, code: state.code, call: currentCall() || undefined, input: inputLines(), tests: state.tests || undefined, views: VIEW_SETS[state.viewSet] })}`; + specOut.hidden = !specOut.hidden; + }); + const clearBtn = button('New', () => { ed.value = ''; state = { ...state, code: '', call: '', tests: '', title: '' }; callInput.value = ''; recheck(); ed.focus(); }); + const statusEl = el('span', { class: 'pl-status', role: 'status' }); + const specOut = el('pre', { class: 'pl-spec-out', hidden: true }); + const status = (t, kind = '') => { statusEl.textContent = t; statusEl.className = `pl-status${kind ? ` is-${kind}` : ''}`; }; + + /* ---------- the side panel: tabs ---------- */ + const TABS = [['coach', 'Coach'], ['jobs', 'Four jobs'], ['tests', 'Tests'], ['proof', 'Prove'], ['checks', 'Generated checks'], ['flow', 'Data flow'], ['mojo', 'Mojo']]; + const tabBar = el('div', { class: 'pl-tabs', role: 'tablist' }); + const tabBody = el('div', { class: 'pl-lab-side', role: 'tabpanel' }); + const tabBtns = new Map(); + let tab = 'coach'; + for (const [id, label] of TABS) { + const b = el('button', { class: 'pl-tab', role: 'tab', type: 'button', 'aria-selected': id === tab ? 'true' : 'false' }, [label, el('span', { class: 'pl-count', hidden: true })]); + b.addEventListener('click', () => { tab = id; for (const [k, x] of tabBtns) x.setAttribute('aria-selected', k === id ? 'true' : 'false'); drawTab(); }); + tabBtns.set(id, b); + tabBar.appendChild(b); + } + const testsBox = el('textarea', { class: 'pl-input pl-lab-tests', rows: '4', 'aria-label': 'tests', spellcheck: 'false', placeholder: 'COUNT_BASE(\'A\', "GATTACA") -> 3\nCOUNT_BASE(\'A\', "") -> 0\nMATCH("ATT", "GATTACA") -> ?' }); + testsBox.addEventListener('input', () => { state.tests = testsBox.value; persist(); }); + + const out = el('div', { class: 'pl-lab-out' }); + + add(root, [ + el('div', { class: 'pl-lab-toolbar' }, [exampleSel, viewSel, runBtn, fixBtn, linkBtn, specBtn, clearBtn]), + el('div', { class: 'pl-lab-top' }, [ + el('div', {}, [ed.node, snippetBar, callRow, inputRow, statusEl]), + el('div', {}, [tabBar, tabBody]), + ]), + specOut, + out, + ]); + + /* ---------- checking as you type ---------- */ + let current = study(state.code); + function recheck() { + current = study(ed.value); + ed.setDiagnostics(current.diags); + const c = severityCounts(current.diags); + const badge = tabBtns.get('coach').querySelector('.pl-count'); + badge.hidden = !(c.error + c.warning + c.note); + badge.textContent = String(c.error + c.warning + c.note); + badge.className = `pl-count${c.error ? ' is-error' : c.warning ? ' is-warning' : ''}`; + fixBtn.disabled = !current.diags.some((d) => d.fix && d.fix.endCol != null); + const hasSubs = current.program.subs.size > 0; + const hasMain = current.program.main.body.length > 0; + callRow.hidden = !hasSubs || (hasMain && !callInput.value); + inputRow.hidden = !hasMain || !/\bINPUT\b/.test(ed.value); + if (hasSubs && !hasMain && !callInput.value.trim()) callInput.placeholder = defaultCall(current.program) || callInput.placeholder; + if (current.blanks.length) status(`${plural(current.blanks.length, 'blank')} left to fill in, such as <${current.blanks[0].text}> on line ${current.blanks[0].line}.`); + else if (!current.ok) status(`${plural(current.errors.length, 'line needs', 'lines need')} attention before this can run. The Coach tab says what each rule is.`, 'error'); + else status(c.warning ? `Ready to run. The coach has ${plural(c.warning, 'warning')}.` : 'Ready to run. Ctrl+Enter runs it.', c.warning ? '' : 'ok'); + if (['coach', 'jobs'].includes(tab)) drawTab(); + } + + function ctxFor() { return { study: current, program: current.program, analysis: current.analysis, spec: { tests: state.tests }, call: null, widget: null }; } + + let tabTimer = null; + function drawTab() { + clear(tabBody); + const ctx = ctxFor(); + const select = (line) => ed.goToLine(line); + const staticOk = current.ok || current.errors.every((d) => d.code === 'blank'); + try { + switch (tab) { + case 'coach': add(tabBody, diagnosticsView(ctx, { onSelect: select, title: '' }).node); break; + case 'jobs': if (current.analysis) add(tabBody, jobsView(ctx, { title: '' }).node); else add(tabBody, el('p', { class: 'pl-muted', text: 'Fix the structure first: the jobs are found once every block is closed.' })); break; + case 'tests': + testsBox.value = state.tests || ''; + add(tabBody, [el('p', { class: 'pl-muted', text: 'One test per line: a call, ->, and the answer. Write ? for the answer to predict it first.' }), testsBox, el('div', { class: 'pl-row' }, [button('Run the tests', () => drawTab(), { primary: true })])]); + if (current.ok && state.tests?.trim()) add(tabBody, testsView(ctx, { title: '', tests: state.tests, onRun: (r) => { callInput.value = r.source.split('->')[0].trim(); runNow(); } }).node); + break; + case 'proof': case 'checks': { + if (!current.ok) { add(tabBody, el('p', { class: 'pl-muted', text: 'The program has to be free of errors before anything can be proved or tested.' })); break; } + const wait = el('p', { class: 'pl-muted', text: tab === 'proof' ? 'Working out what can be proved\u2026' : 'Running generated inputs\u2026' }); + add(tabBody, wait); + clearTimeout(tabTimer); + const which = tab; + tabTimer = setTimeout(() => { + if (tab !== which) return; + clear(tabBody); + const v = which === 'proof' ? proofView(ctx, { title: '', onSelect: select }) : checksView(ctx, { title: '', onRun: (f) => { callInput.value = f.call; runNow(); } }); + add(tabBody, v.node); + }, 30); + break; + } + case 'flow': if (staticOk && current.analysis) { const v = flowView(ctx, { title: '' }); v.update(null); add(tabBody, v.node); } else add(tabBody, el('p', { class: 'pl-muted', text: 'Fix the errors first.' })); break; + case 'mojo': if (current.ok) { const v = translateView(ctx, { title: '' }); v.update(null); add(tabBody, v.node); } else add(tabBody, el('p', { class: 'pl-muted', text: 'Fix the errors first: only a well-formed program can be translated.' })); break; + default: break; + } + } catch (err) { + add(tabBody, el('p', { class: 'pl-error', text: `This panel failed: ${err.message}` })); + console.error(err); // eslint-disable-line no-console + } + } + + function currentCall() { + const c = callInput.value.trim(); + if (c) return c; + if (current.program.subs.size && !current.program.main.body.length) return defaultCall(current.program); + return null; + } + const inputLines = () => inputBox.value.split('\n').filter((l, i, a) => l.length || i < a.length - 1); + + let widgetCtl = null; + function runNow() { + state.code = ed.value; + recheck(); + clear(out); + if (!current.ok) { status(current.blanks.length ? 'Fill in the blanks first.' : 'Fix the errors first: see the Coach tab.', 'error'); tab = 'coach'; for (const [k, x] of tabBtns) x.setAttribute('aria-selected', k === tab ? 'true' : 'false'); drawTab(); return; } + const call = currentCall(); + if (call) { try { resolveCall(current.program, call); } catch (e) { status(e.message, 'error'); return; } } + const host = el('div'); + add(out, host); + widgetCtl = mountWidget(host, { code: state.code, study: current, call, input: inputLines(), views: VIEW_SETS[viewSel.value] || VIEW_SETS['step through'], tests: state.tests }); + if (widgetCtl.trace) { + const t = widgetCtl.trace; + const sub = call ? current.program.subs.get(call.split('(')[0].trim().toUpperCase()) : null; + status(t.error ? `Ran ${plural(t.count, 'step')}, then stopped at line ${t.error.line}: ${t.error.message}` : call ? `${call} ${t.result === undefined ? 'finished' : `returned ${formatResult(t, sub)}`} after ${plural(t.count, 'step')}.` : `Ran ${plural(t.count, 'step')}.`, t.error ? 'error' : 'ok'); + widgetCtl.onChange((frame) => ed.setCurrentLine(frame.next?.line ?? null)); + host.querySelector('.pl')?.focus?.(); + } + persist(false); + } + + function persist(toHash) { + state.call = callInput.value; state.input = inputBox.value; state.viewSet = viewSel.value; + if (opts.remember) { try { localStorage.setItem('pseudo-lab', JSON.stringify(state)); } catch { /* fine without */ } } + if (toHash && opts.useHash && typeof history !== 'undefined') history.replaceState(null, '', `#lab=${encodeState(state)}`); + } + + exampleSel.addEventListener('change', () => { + const ex = opts.examples[exampleSel.value]; + exampleSel.value = ''; + if (!ex) return; + state = { ...state, code: ex.code, call: ex.call || '', input: (ex.input || []).join('\n'), tests: ex.tests || '', title: ex.title || '' }; + ed.value = ex.code; callInput.value = state.call; inputBox.value = state.input; + recheck(); runNow(); + }); + viewSel.addEventListener('change', () => { state.viewSet = viewSel.value; runNow(); }); + + callInput.value = state.call || ''; + inputBox.value = state.input || ''; + viewSel.value = VIEW_SETS[state.viewSet] ? state.viewSet : 'step through'; + recheck(); + drawTab(); + runNow(); + return root; +} + +function formatResult(t, sub) { return formatValue(t.result, sub?.returns); } diff --git a/src/engines/pseudo/lang.ts b/src/engines/pseudo/lang.ts new file mode 100644 index 0000000..4df5d95 --- /dev/null +++ b/src/engines/pseudo/lang.ts @@ -0,0 +1,593 @@ +/** + * pseudo/lang.ts + * + * The fixed vocabulary of the course's pseudocode: keywords, types, built-in + * functions and procedures, and the way values are written back out. Every + * other file in the library reads its facts about the notation from here, so + * a change to the notation is a change to this file first. + * + * Nothing here depends on the DOM. + */ + +import type { PType, IntType, FloatType, ListType, ArrayType, MapType, SetType, RangeType, SimpleType, AnyType, RecordType, LooseType, Value, Expr } from './ast.js'; + +/* ------------------------------------------------------------------ */ +/* Keywords */ +/* ------------------------------------------------------------------ */ + +/** Words that belong to the notation and can never be names. */ +export const KEYWORDS = new Set([ + 'PARALLEL', + // libraries + 'USE', + // declarations + 'DECLARE', 'CONSTANT', 'DEFINE', 'TYPE', 'ENDTYPE', 'OF', + // types + 'INTEGER', 'FLOAT', 'CHAR', 'STRING', 'BOOL', 'BOOLEAN', 'ARRAY', 'LIST', 'HASHMAP', 'SET', + 'INT8', 'INT16', 'INT32', 'INT64', 'UINT8', 'UINT16', 'UINT32', 'UINT64', 'FLOAT32', 'FLOAT64', + // selection + 'IF', 'THEN', 'ELSEIF', 'ELSE', 'ENDIF', 'CASE', 'OTHERWISE', 'ENDCASE', + // iteration + 'FOR', 'TO', 'STEP', 'NEXT', 'WHILE', 'ENDWHILE', 'REPEAT', 'UNTIL', 'BREAK', 'CONTINUE', + // subroutines + 'FUNCTION', 'ENDFUNCTION', 'PROCEDURE', 'ENDPROCEDURE', 'RETURNS', 'RETURN', 'CALL', 'BYREF', 'BYVAL', + // input and output + 'INPUT', 'OUTPUT', + // operators + 'AND', 'OR', 'NOT', 'DIV', 'MOD', 'IN', + // constants + 'TRUE', 'FALSE', + // reasoning + 'REQUIRE', 'ENSURE', 'INVARIANT', 'DECREASES', 'ASSERT', 'FORALL', 'EXISTS', +]); + +/** + * Words that other pseudocode dialects use and this one does not. Each maps + * to the sentence a student sees on meeting it. + */ +export const FOREIGN_WORDS: Record = { + NULL: 'This notation has no NULL. An empty LIST, or a BOOL flag saying whether a value was found, says the same thing more plainly.', + CLASS: 'Classes are outside this course\'s notation. A TYPE ... ENDTYPE record groups fields without them.', + ENDCLASS: 'Classes are outside this course\'s notation.', + INHERITS: 'Inheritance is outside this course\'s notation.', + NEW: 'This notation has no NEW. DECLARE makes a variable of any type, including a record.', + PUBLIC: 'This notation has no PUBLIC or PRIVATE.', + PRIVATE: 'This notation has no PUBLIC or PRIVATE.', + SUPER: 'Inheritance is outside this course\'s notation.', + ENDFOR: 'A FOR loop closes with NEXT and its counter\'s name, as in NEXT I.', + END: 'Blocks close with one word: ENDIF, ENDWHILE, ENDCASE, ENDFUNCTION, ENDPROCEDURE, ENDTYPE, or NEXT for a FOR loop.', + DO: 'WHILE takes no DO in this notation. Write WHILE on a line of its own.', + ELIF: 'Write ELSEIF, as one word.', + PRINT: 'Output is written OUTPUT in this notation.', + READ: 'Input is written INPUT in this notation.', + SWITCH: 'Selecting on a value is written CASE OF ... ENDCASE.', + DEFAULT: 'The catch-all branch of a CASE is written OTHERWISE.', + XOR: 'This notation has no XOR. A != B says the same thing for two BOOL values.', +}; + +/** + * Words that act as keywords only in one position, so they stay available + * as names everywhere else. COUNT, for one, is the name of a function in the + * counting lesson. + */ +export const CONTEXTUAL: Record = { + EACH: 'after FOR, as in FOR EACH X IN Collection', + SUM: 'before a name and IN, as in SUM I IN [0, N) : Q[I]', + RESULT: 'inside an ENSURE, for the value the function returns', + OLD: 'inside an ENSURE, for the value a BYREF parameter held on entry', +}; + +/** How the highlighter and the editor color each keyword. */ +export const KEYWORD_CLASS: Record = (() => { + const m: Record = {}; + const put = (cls: string, words: string) => { for (const w of words.split(' ')) m[w] = cls; }; + put('flow', 'IF THEN ELSEIF ELSE ENDIF CASE OTHERWISE ENDCASE FOR TO STEP NEXT WHILE ENDWHILE REPEAT UNTIL RETURN BREAK CONTINUE EACH'); + put('decl', 'DECLARE CONSTANT DEFINE TYPE ENDTYPE FUNCTION ENDFUNCTION PROCEDURE ENDPROCEDURE RETURNS USE'); + put('kw', 'AND OR NOT DIV MOD OF CALL IN BYREF BYVAL'); + put('io', 'INPUT OUTPUT'); + put('type', 'INTEGER FLOAT CHAR STRING BOOL BOOLEAN ARRAY LIST HASHMAP SET INT8 INT16 INT32 INT64 UINT8 UINT16 UINT32 UINT64 FLOAT32 FLOAT64'); + put('const', 'TRUE FALSE'); + put('proof', 'REQUIRE ENSURE INVARIANT DECREASES ASSERT FORALL EXISTS SUM RESULT OLD'); + return m; +})(); + +/** Which keyword closes which opener. */ +export const CLOSERS: Record = { + IF: 'ENDIF', CASE: 'ENDCASE', FOR: 'NEXT', WHILE: 'ENDWHILE', REPEAT: 'UNTIL', + FUNCTION: 'ENDFUNCTION', PROCEDURE: 'ENDPROCEDURE', TYPE: 'ENDTYPE', +}; + +/* ------------------------------------------------------------------ */ +/* Types */ +/* ------------------------------------------------------------------ */ + +/** + * The sized numeric types. Each is an INTEGER or a FLOAT with a width, so every rule that asks for "a whole + * number" (a position, a FOR counter, DIV) accepts INT32 as readily as INTEGER, and only the rules about + * range, rounding and mixing look at the width. + * + * INTEGER a whole number of any size (the teaching integer; a run holds it exactly to 2^53 - 1) + * INT8 ... INT64 two's complement, n bits; a result outside the range stops the run (obligation 'overflow') + * UINT8 ... UINT64 0 to 2^n - 1, the same way + * FLOAT, FLOAT64 IEEE 754 double precision (the two names are one type) + * FLOAT32 IEEE 754 single precision: every result is rounded to the nearest single + * + * INT64 and UINT64 are held exactly only to 2^53 - 1 in this lab (a JavaScript number); a result past that + * stops the run with a message that says so, and the prover checks against the same limit. + */ +export const SIZED: Record & { + INT8: IntType; INT16: IntType; INT32: IntType; INT64: IntType; UINT8: IntType; UINT16: IntType; UINT32: IntType; UINT64: IntType; + FLOAT32: FloatType; FLOAT64: FloatType; +} = { + INT8: { k: 'INTEGER', bits: 8, signed: true, name: 'INT8' }, + INT16: { k: 'INTEGER', bits: 16, signed: true, name: 'INT16' }, + INT32: { k: 'INTEGER', bits: 32, signed: true, name: 'INT32' }, + INT64: { k: 'INTEGER', bits: 64, signed: true, name: 'INT64' }, + UINT8: { k: 'INTEGER', bits: 8, signed: false, name: 'UINT8' }, + UINT16: { k: 'INTEGER', bits: 16, signed: false, name: 'UINT16' }, + UINT32: { k: 'INTEGER', bits: 32, signed: false, name: 'UINT32' }, + UINT64: { k: 'INTEGER', bits: 64, signed: false, name: 'UINT64' }, + FLOAT32: { k: 'FLOAT', bits: 32, name: 'FLOAT32' }, + FLOAT64: { k: 'FLOAT', bits: 64, name: 'FLOAT64' }, +}; +export const SIZED_NAMES = new Set(Object.keys(SIZED)); +/** The largest whole number a run holds exactly. */ +export const EXACT_MAX = Number.MAX_SAFE_INTEGER; + +/** A whole-number type with a width (INT32, UINT8, ...), as opposed to INTEGER. */ +// typed-port: the prover (vcgen.ts, logic.ts) calls these with its own loosely typed nodes (LooseType), read here as a PType. +export const isSizedInt = (t: PType | LooseType | null | undefined): t is IntType & { bits: number } => !!t && t.k === 'INTEGER' && !!(t as PType).bits; +/** FLOAT32, the one FLOAT that rounds to fewer bits than FLOAT. */ +export const isFloat32 = (t: PType | LooseType | null | undefined): t is FloatType & { bits: 32 } => !!t && t.k === 'FLOAT' && (t as PType).bits === 32; + +/** + * The values a sized whole-number type holds in a run: [lowest, highest]. For INT64 and UINT64 the run's range + * is cut at 2^53 - 1, the largest whole number it holds exactly; `full` gives the type's own range as text. + */ +export function intRange(t: PType | LooseType | null | undefined): [number, number] { + if (!isSizedInt(t)) return [-EXACT_MAX, EXACT_MAX]; + if (t.bits >= 64) return t.signed ? [-EXACT_MAX, EXACT_MAX] : [0, EXACT_MAX]; + return t.signed ? [-(2 ** (t.bits - 1)), 2 ** (t.bits - 1) - 1] : [0, 2 ** t.bits - 1]; +} +export function intRangeText(t: IntType): string { + const full = ({ INT64: ['-9,223,372,036,854,775,808', '9,223,372,036,854,775,807'], UINT64: ['0', '18,446,744,073,709,551,615'] } as Record)[t.name as string]; + if (full) return `${full[0]} to ${full[1]}, held exactly in this lab only from ${t.signed ? '-9,007,199,254,740,991' : '0'} to 9,007,199,254,740,991`; + const [lo, hi] = intRange(t); + return `${lo.toLocaleString('en-US')} to ${hi.toLocaleString('en-US')}`; +} + +export const T: { + INTEGER: IntType; FLOAT: FloatType; CHAR: SimpleType; STRING: SimpleType; BOOL: SimpleType; VOID: SimpleType; ANY: AnyType; + INT8: IntType; INT16: IntType; INT32: IntType; INT64: IntType; UINT8: IntType; UINT16: IntType; UINT32: IntType; UINT64: IntType; + FLOAT32: FloatType; FLOAT64: FloatType; + list(of: PType): ListType; array(of: PType, dims?: number): ArrayType; map(key: PType, val: PType): MapType; set(of: PType): SetType; range(): RangeType; +} = { + INTEGER: { k: 'INTEGER' }, + FLOAT: { k: 'FLOAT' }, + ...SIZED, + CHAR: { k: 'CHAR' }, + STRING: { k: 'STRING' }, + BOOL: { k: 'BOOL' }, + VOID: { k: 'VOID' }, + ANY: { k: 'ANY' }, + list: (of: PType): ListType => ({ k: 'LIST', of }), + array: (of: PType, dims = 1): ArrayType => ({ k: 'ARRAY', of, dims }), + map: (key: PType, val: PType): MapType => ({ k: 'HASHMAP', key, val }), + set: (of: PType): SetType => ({ k: 'SET', of }), + range: () => ({ k: 'RANGE', of: { k: 'INTEGER' } }), +}; + +export const SCALARS = new Set(['INTEGER', 'FLOAT', 'CHAR', 'STRING', 'BOOL']); +// typed-port: these three answer the type itself (not a boolean) for null or undefined, as the JavaScript did; every +// caller only tests the answer for truth. +export const isNumeric = (t: PType | null | undefined): boolean => (t && (t.k === 'INTEGER' || t.k === 'FLOAT')) as boolean; +export const isAny = (t: PType | null | undefined): t is AnyType | null | undefined => !t || t.k === 'ANY'; +export const isText = (t: PType | null | undefined): boolean => (t && (t.k === 'STRING' || t.k === 'CHAR')) as boolean; +export const isCollection = (t: PType | null | undefined): boolean => (t && ['LIST', 'ARRAY', 'HASHMAP', 'SET', 'STRING', 'RANGE'].includes(t.k)) as boolean; + +/** INTEGER, LIST OF INTEGER, HASHMAP OF STRING : CHAR, and so on. */ +export function typeName(t0: PType | LooseType | null | undefined): string { + const t = t0 as PType | null | undefined; + if (!t) return 'an unknown type'; + switch (t.k) { + case 'LIST': return `LIST OF ${typeName(t.of)}`; + case 'ARRAY': return `ARRAY${t.dims === 2 ? ' (two-dimensional)' : ''} OF ${typeName(t.of)}`; + case 'HASHMAP': return `HASHMAP OF ${typeName(t.key)} : ${typeName(t.val)}`; + case 'SET': return t.name ? t.name : `SET OF ${typeName(t.of)}`; + case 'RECORD': return t.name; + case 'RANGE': return 'a range of positions'; + case 'ANY': return 'any type'; + case 'VOID': return 'nothing'; + case 'INTEGER': case 'FLOAT': return t.name || t.k; + default: return t.k; + } +} + +/** Two types that describe the same values. ANY matches everything. */ +export function sameType(a: PType | null | undefined, b: PType | null | undefined): boolean { + if (isAny(a) || isAny(b)) return true; + if (a.k !== b.k) return false; + switch (a.k) { + case 'LIST': case 'SET': return sameType(a.of, b.of); + case 'ARRAY': return a.dims === b.dims && sameType(a.of, b.of); + case 'HASHMAP': return sameType(a.key, b.key) && sameType(a.val, b.val); + case 'RECORD': return a.name.toUpperCase() === (b as RecordType).name.toUpperCase(); + case 'INTEGER': return (a.bits || 0) === (b.bits || 0) && (!a.bits || a.signed === b.signed); + case 'FLOAT': return isFloat32(a) === isFloat32(b); + default: return true; + } +} + +/** + * Can a value of type `from` be stored under a name of type `to`? The one + * widening allowed is INTEGER into FLOAT: a whole number is a perfectly good + * measurement. Nothing narrows silently. + */ +/** The lengths an ARRAY type was declared with, when each is a plain number: ARRAY[2, 3] gives [2, 3]. */ +export function knownDims(t0: PType | LooseType | null | undefined): number[] | null { + const t = t0 as PType | null | undefined; + if (!t || t.k !== 'ARRAY' || !t.dimExprs?.length || !t.dimExprs.every((d) => d.kind === 'Num')) return null; + return t.dimExprs.map((d) => (d as Expr & { kind: 'Num' }).value); +} + +export function assignable(to: PType | null | undefined, from: PType | null | undefined): boolean { + if (isAny(to) || isAny(from)) return true; + if (to.k === 'FLOAT' && from.k === 'INTEGER') return true; + // INTEGER into a sized type is checked when it happens (the run stops if it does not fit, and the prover has + // an obligation for it); a sized value always fits INTEGER; two different widths need a conversion: INT32(X). + if (to.k === 'INTEGER' && from.k === 'INTEGER') return !to.bits || !from.bits || sameType(to, from); + // FLOAT32 widens into FLOAT exactly; the other way rounds, so it is written FLOAT32(X). + if (to.k === 'FLOAT' && from.k === 'FLOAT') return !isFloat32(to) || isFloat32(from); + // a LIST fills a one-dimensional ARRAY; a table is filled position by position, since [1, 2, 3, 4] has no rows + if (to.k === 'ARRAY' && from.k === 'LIST') return to.dims !== 2 && assignable(to.of, from.of); + if (to.k === 'LIST' && from.k === 'LIST') return assignable(to.of, from.of) && (isAny(from.of) || sameType(to.of, from.of)); + if (to.k === 'SET' && from.k === 'SET') return isAny(from.of) || sameType(to.of, from.of); + if (to.k === 'HASHMAP' && from.k === 'HASHMAP') return (isAny(from.key) || sameType(to.key, from.key)) && (isAny(from.val) || assignable(to.val, from.val)); + return sameType(to, from); +} + +/* ------------------------------------------------------------------ */ +/* Runtime values */ +/* ------------------------------------------------------------------ */ + +/** The contents of a name that has been declared and never given a value. */ +export const UNSET: unique symbol = Symbol('unset'); + +/** + * A fixed-length array. One and two dimensions share a flat store. + * `frozen` marks a value that a recorded step still refers to, so the next + * change copies it first rather than rewriting history. + */ +export class PArray { + declare dims: number[]; + declare data: Value[]; + declare frozen: boolean; + constructor(dims: number[], fill: Value | null = UNSET) { + this.dims = dims.slice(); + this.data = new Array(dims.reduce((a, b) => a * b, 1)).fill(fill); + this.frozen = false; + } + get length(): number { return this.dims[0]; } + clone(): PArray { const a = new PArray(this.dims); a.data = this.data.map(cloneValue); return a; } +} + +/** A value of a TYPE ... ENDTYPE record. */ +export class PRecord { + declare typeName: string; + declare fields: Map; + declare names: Map; + declare frozen: boolean; + constructor(typeName: string, fieldNames: string[]) { + this.typeName = typeName; + this.fields = new Map(fieldNames.map((f) => [f.toUpperCase(), UNSET])); + this.names = new Map(fieldNames.map((f) => [f.toUpperCase(), f])); + this.frozen = false; + } + clone(): PRecord { + const r = new PRecord(this.typeName, []); + r.names = new Map(this.names); + for (const [k, v] of this.fields) r.fields.set(k, cloneValue(v)); + return r; + } +} + +/** LIST is a JavaScript array, HASHMAP a Map, SET a Set. Each can be frozen. */ +export function cloneValue(v: V): V; +/** LIST is a JavaScript array, HASHMAP a Map, SET a Set. Each can be frozen. */ +export function cloneValue(v: Value | null): Value | null { + if (v === UNSET || v == null) return v; + if (Array.isArray(v)) { const c = v.map(cloneValue); return c; } + if (v instanceof Map) return new Map([...v].map(([k, x]) => [k, cloneValue(x)])); + if (v instanceof Set) return new Set(v); + if (v instanceof PArray || v instanceof PRecord) return v.clone(); + return v; +} + +/** Characters, not bytes: 'ñ' is one position, as the guide promises. */ +export function chars(s: string): string[] { + // eslint-disable-next-line no-control-regex + return /^[\x00-\x7f]*$/.test(s) ? s.split('') : Array.from(s); +} +export function textLength(s: string): number { return /^[\x00-\x7f]*$/.test(s) ? s.length : Array.from(s).length; } + +/** Keys of a HASHMAP and members of a SET, in the one order the lab uses. */ +export function sortedKeys(iterable: Iterable): V[] { + return [...iterable].sort(compareValues); +} + +export function compareValues(a: Value, b: Value): number { + if (typeof a === 'number' && typeof b === 'number') { + // not a - b: infinity minus infinity is not a number, which no sort can use. Not-a-number sorts after every + // number, so SORT and SORTED still put the numbers in order (USE IEEE). + if (a < b) return -1; + if (a > b) return 1; + if (a === b) return 0; + return Number.isNaN(a) ? (Number.isNaN(b) ? 0 : 1) : -1; + } + if (typeof a === 'string' && typeof b === 'string') return a < b ? -1 : a > b ? 1 : 0; + if (typeof a === 'boolean' && typeof b === 'boolean') return Number(a) - Number(b); + // lists (and one-dimensional ARRAYs) in dictionary order: the first values that differ decide, and a list that + // runs out first comes first, so [2] comes before [10] and [1, 5] before [2] + const items = (v: Value): Value[] | null => (Array.isArray(v) ? v : v instanceof PArray && v.dims.length === 1 ? v.data : null); + const la = items(a), lb = items(b); + if (la && lb) { + for (let i = 0; i < Math.min(la.length, lb.length); i++) { const c = compareValues(la[i], lb[i]); if (c) return c; } + return la.length - lb.length; + } + return String(a) < String(b) ? -1 : String(a) > String(b) ? 1 : 0; +} + +/** Deep equality, the meaning of == on every type. */ +export function valuesEqual(a: Value, b: Value): boolean { + if (a === b) return true; + if (typeof a === 'number' && typeof b === 'number') return a === b; + if (Array.isArray(a) && Array.isArray(b)) return a.length === b.length && a.every((x, i) => valuesEqual(x, b[i])); + if (a instanceof PArray && b instanceof PArray) return a.data.length === b.data.length && a.data.every((x, i) => valuesEqual(x, b.data[i])); + if (Array.isArray(a) && b instanceof PArray) return valuesEqual(a, b.data); + if (a instanceof PArray && Array.isArray(b)) return valuesEqual(a.data, b); + if (a instanceof Set && b instanceof Set) return a.size === b.size && [...a].every((x) => b.has(x)); + if (a instanceof Map && b instanceof Map) return a.size === b.size && [...a].every(([k, v]) => b.has(k) && valuesEqual(v, b.get(k))); + if (a instanceof PRecord && b instanceof PRecord) { + return a.typeName === b.typeName && [...a.fields].every(([k, v]) => valuesEqual(v, b.fields.get(k))); + } + return false; +} + +/* ------------------------------------------------------------------ */ +/* Writing values back out */ +/* ------------------------------------------------------------------ */ + +/** A FLOAT always shows its point, so 2.0 never reads as the INTEGER 2. */ +export function formatFloat(x: number): string { + if (!Number.isFinite(x)) return Number.isNaN(x) ? 'not a number' : (x > 0 ? 'infinity' : '-infinity'); + if (Object.is(x, -0)) return '-0.0'; // minus zero is a FLOAT of its own (1.0 / -0.0 is minus infinity), and Mojo prints it so + const s = String(x); + // Large and tiny values come out as 1e+21 or 5e-7; they keep their point + // too, and lose the +, so that what is shown can be typed back in. + const m = /^(-?[\d.]+)e([+-]\d+)$/.exec(s); + if (m) return `${m[1].includes('.') ? m[1] : `${m[1]}.0`}e${m[2].replace('+', '')}`; + return s.includes('.') ? s : `${s}.0`; +} + +/** + * A value as a literal a student could type back in: 'A' with its quotes, + * "GATTACA" with its quotes, [4, 17] for a list. `type` decides INTEGER + * versus FLOAT, since both are JavaScript numbers underneath. + */ +/** + * A FLOAT32 as the shortest decimal that rounds back to the same single: FLOAT32(0.1) shows as 0.1, though the + * value it holds is 0.100000001490116119384765625 (EXACT shows that). + */ +export function formatFloat32(x: number): string { + if (!Number.isFinite(x)) return formatFloat(x); + for (let p = 1; p <= 9; p++) { + const s = x.toPrecision(p); + if (Math.fround(Number(s)) === x) return formatFloat(Number(s)); + } + return formatFloat(x); +} + +export function formatValue(v: Value | null, type?: PType | null, opts: { max?: number; unset?: string; bareChar?: boolean } = {}): string { + const max = opts.max ?? 40; + if (v === UNSET) return opts.unset ?? '(no value)'; + if (v === undefined || v === null) return '(nothing)'; + if (typeof v === 'boolean') return v ? 'TRUE' : 'FALSE'; + if (typeof v === 'number') { + if (type && type.k === 'FLOAT') return isFloat32(type) ? formatFloat32(v) : formatFloat(v); + if (type && type.k === 'INTEGER') return String(v); + return Number.isInteger(v) ? String(v) : formatFloat(v); + } + if (typeof v === 'string') { + if (type && type.k === 'CHAR') return `'${escapeText(v, "'")}'`; + if (type && type.k === 'STRING') return `"${escapeText(v, '"')}"`; + return textLength(v) === 1 && opts.bareChar ? `'${v}'` : `"${escapeText(v, '"')}"`; + } + const el = (x: Value, t: PType | null | undefined) => formatValue(x, t, { ...opts, max: 12 }); + const cut = (items: string[]) => (items.length > max ? [...items.slice(0, max), `… ${items.length - max} more`] : items); + if (Array.isArray(v)) return `[${cut(v.map((x) => el(x, type?.of))).join(', ')}]`; + if (v instanceof PArray) { + if (v.dims.length === 2) { + const rows = []; + for (let r = 0; r < v.dims[0]; r++) rows.push(`[${v.data.slice(r * v.dims[1], (r + 1) * v.dims[1]).map((x) => el(x, type?.of)).join(', ')}]`); + return `[${cut(rows).join(', ')}]`; + } + return `[${cut(v.data.map((x) => el(x, type?.of))).join(', ')}]`; + } + if (v instanceof Set) return `{${cut(sortedKeys(v).map((x) => el(x, type?.of))).join(', ')}}`; + if (v instanceof Map) return `{${cut(sortedKeys(v.keys()).map((k) => `${el(k, type?.key)}: ${el(v.get(k), type?.val)}`)).join(', ')}}`; + if (v instanceof PRecord) { + const parts = [...v.fields].map(([k, x]) => `${v.names.get(k)}: ${el(x, type?.fields?.get(k))}`); + return `${v.typeName}(${parts.join(', ')})`; + } + return String(v); +} + +/** What OUTPUT prints: text without its quotes, everything else as a literal. */ +export function outputText(v: Value, type?: PType | null): string { + if (typeof v === 'string') return v; + return formatValue(v, type, { max: 1000 }); +} + +function escapeText(s: string, q: '"' | "'"): string { + return s.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/\t/g, '\\t').replace(q === '"' ? /"/g : /'/g, `\\${q}`); +} + +/* ------------------------------------------------------------------ */ +/* Built-ins */ +/* ------------------------------------------------------------------ */ + +/** + * Every built-in, in one table. `kind` is 'function' (stands where a value + * stands) or 'procedure' (a line of its own, no CALL). `sig` is what the + * reference prints; `doc` is its one-sentence meaning; `mojo` is the + * translation hint the translator uses. + */ +export const BUILTINS: Record = { + LENGTH: { kind: 'function', sig: 'LENGTH(X)', doc: 'How many characters a STRING holds, or how many values a LIST, one-dimensional ARRAY, SET, or HASHMAP holds.', group: 'Measuring' }, + ROWS: { kind: 'function', sig: 'ROWS(A)', doc: 'The first length of a two-dimensional ARRAY.', group: 'Measuring' }, + COLUMNS: { kind: 'function', sig: 'COLUMNS(A)', doc: 'The second length of a two-dimensional ARRAY.', group: 'Measuring' }, + INT: { kind: 'function', sig: 'INT(X)', doc: 'The whole-number part of a FLOAT, dropping the fraction toward zero: INT(27.54) is 27 and INT(-2.5) is -2.', group: 'Numbers' }, + ROUND: { kind: 'function', sig: 'ROUND(X)', doc: 'The nearest whole number, with halves rounded away from zero: ROUND(2.5) is 3.', group: 'Numbers' }, + ABS: { kind: 'function', sig: 'ABS(X)', doc: 'The size of a number, ignoring its sign.', group: 'Numbers' }, + MIN: { kind: 'function', sig: 'MIN(A, B, ...)', doc: 'The smallest of two or more numbers, or of the values in one LIST. With USE IEEE, a NaN among them makes the answer NaN.', group: 'Numbers' }, + MAX: { kind: 'function', sig: 'MAX(A, B, ...)', doc: 'The largest of two or more numbers, or of the values in one LIST. With USE IEEE, a NaN among them makes the answer NaN.', group: 'Numbers' }, + SQRT: { kind: 'function', sig: 'SQRT(X)', doc: 'The square root, as a FLOAT.', group: 'Numbers' }, + EXP: { kind: 'function', sig: 'EXP(X)', doc: 'e raised to the power X, as a FLOAT.', group: 'Numbers' }, + LOG: { kind: 'function', sig: 'LOG(X)', doc: 'The natural logarithm, as a FLOAT.', group: 'Numbers' }, + LOG2: { kind: 'function', sig: 'LOG2(X)', doc: 'The base-2 logarithm, as a FLOAT; information in bits.', group: 'Numbers' }, + RAND: { kind: 'function', sig: 'RAND(X)', doc: 'A random FLOAT between 0 and X, never X itself: in [0, X) for X above 0, in (X, 0] for X below 0, and 0.0 for X = 0.', group: 'Numbers' }, + RANDINT: { kind: 'function', sig: 'RANDINT(A, B)', doc: 'A random INTEGER in [A, B).', group: 'Numbers' }, + FLOOR: { kind: 'function', sig: 'FLOOR(X)', doc: 'The largest whole number at or below X: FLOOR(2.7) is 2 and FLOOR(-2.3) is -3.', group: 'Numbers' }, + CEIL: { kind: 'function', sig: 'CEIL(X)', doc: 'The smallest whole number at or above X: CEIL(2.1) is 3.', group: 'Numbers' }, + LOG10: { kind: 'function', sig: 'LOG10(X)', doc: 'The base-10 logarithm, as a FLOAT; pH and decibels live here.', group: 'Numbers' }, + SIN: { kind: 'function', sig: 'SIN(X)', doc: 'The sine of X radians, as a FLOAT.', group: 'Numbers' }, + COS: { kind: 'function', sig: 'COS(X)', doc: 'The cosine of X radians, as a FLOAT.', group: 'Numbers' }, + TAN: { kind: 'function', sig: 'TAN(X)', doc: 'The tangent of X radians, as a FLOAT.', group: 'Numbers' }, + ATAN2: { kind: 'function', sig: 'ATAN2(Y, X)', doc: 'The angle in radians of the point (X, Y) from the positive x axis, in [-PI(), PI()]: -PI() only when Y is -0.0 and X is negative.', group: 'Numbers' }, + PI: { kind: 'function', sig: 'PI()', doc: 'The number pi, 3.14159…, as a FLOAT.', group: 'Numbers' }, + CHOOSE: { kind: 'function', sig: 'CHOOSE(N, K)', doc: 'The binomial coefficient: how many ways to pick K of N things, and the counts behind the binomial distribution.', group: 'Numbers' }, + RANDN: { kind: 'function', sig: 'RANDN()', doc: 'A random FLOAT from the standard normal distribution (mean 0, standard deviation 1).', group: 'Numbers' }, + SUM_OF: { kind: 'function', sig: 'SUM_OF(L)', doc: 'The sum of the numbers in a LIST or ARRAY; 0 for an empty one. A FLOAT when any value is a FLOAT.', group: 'Statistics' }, + MEAN_OF: { kind: 'function', sig: 'MEAN_OF(L)', doc: 'The arithmetic mean of the numbers in a LIST or ARRAY, as a FLOAT; stops on an empty one.', group: 'Statistics' }, + VARIANCE_OF: { kind: 'function', sig: 'VARIANCE_OF(L)', doc: 'The sample variance (dividing by N - 1) of the numbers in a LIST or ARRAY, as a FLOAT; needs at least two values.', group: 'Statistics' }, + STDEV_OF: { kind: 'function', sig: 'STDEV_OF(L)', doc: 'The sample standard deviation, SQRT(VARIANCE_OF(L)), as a FLOAT.', group: 'Statistics' }, + MEDIAN_OF: { kind: 'function', sig: 'MEDIAN_OF(L)', doc: 'The middle value of the numbers in a LIST or ARRAY, as a FLOAT; the mean of the two middle values when there is an even number.', group: 'Statistics' }, + DOT: { kind: 'function', sig: 'DOT(A, B)', doc: 'The dot product of two LISTs or ARRAYs of numbers of the same length: the sum of A[I] * B[I].', group: 'Vectors' }, + NORM_OF: { kind: 'function', sig: 'NORM_OF(A)', doc: 'The Euclidean length of a LIST or ARRAY of numbers, SQRT(DOT(A, A)), as a FLOAT.', group: 'Vectors' }, + FILL: { kind: 'function', sig: 'FILL(N, V)', doc: 'A new LIST of N copies of the value V: FILL(3, 0.0) is [0.0, 0.0, 0.0]. The way to make a buffer before a loop fills it.', group: 'Vectors' }, + SLICE: { kind: 'function', sig: 'SLICE(X, A, B)', doc: 'The characters of a STRING, or the values of a LIST, at positions [A, B), as a new value of the same type.', group: 'Text and lists' }, + UPPER: { kind: 'function', sig: 'UPPER(X)', doc: 'A STRING or CHAR with every letter in capitals. A STRING can grow (UPPER("straße") is "STRASSE"); a CHAR whose capital is two characters, such as \'ß\', stays as it is.', group: 'Text and lists' }, + LOWER: { kind: 'function', sig: 'LOWER(X)', doc: 'A STRING or CHAR with every letter in small letters. A STRING can grow; a CHAR whose small letter is two characters stays as it is.', group: 'Text and lists' }, + SPLIT: { kind: 'function', sig: 'SPLIT(S, Sep)', doc: 'The pieces of the STRING S between occurrences of the STRING or CHAR Sep, as a LIST OF STRING. Empty pieces are kept, so SPLIT("a,,b", ",") is ["a", "", "b"]; split on a space and drop the empty pieces yourself for words.', group: 'Text and lists' }, + JOIN: { kind: 'function', sig: 'JOIN(L, Sep)', doc: 'One STRING made of the values of the LIST OF STRING L with Sep between each pair: JOIN(["a", "b"], ", ") is "a, b".', group: 'Text and lists' }, + INDEX_OF: { kind: 'function', sig: 'INDEX_OF(X, Piece)', doc: 'The first position at which Piece (a STRING or CHAR) occurs in the STRING X, or at which the value Piece occurs in the LIST X; -1 when it does not occur.', group: 'Text and lists' }, + REPLACE: { kind: 'function', sig: 'REPLACE(S, A, B)', doc: 'The STRING S with every occurrence of A replaced by B, left to right, without overlapping.', group: 'Text and lists' }, + ORD: { kind: 'function', sig: 'ORD(C)', doc: 'The number Unicode gives a CHAR: ORD(\'A\') is 65.', group: 'Text and lists' }, + CHR: { kind: 'function', sig: 'CHR(N)', doc: 'The CHAR Unicode numbers N: CHR(65) is \'A\'.', group: 'Text and lists' }, + TO_STRING: { kind: 'function', sig: 'TO_STRING(X)', doc: 'Any value written as a STRING, the way OUTPUT would print it.', group: 'Converting' }, + TO_INTEGER: { kind: 'function', sig: 'TO_INTEGER(S)', doc: 'The INTEGER a STRING of digits spells, or the digit a CHAR is: TO_INTEGER(\'7\') is 7.', group: 'Converting' }, + TO_FLOAT: { kind: 'function', sig: 'TO_FLOAT(X)', doc: 'The FLOAT a STRING of decimal digits spells ("2.5", "-1e3"), or an INTEGER as a FLOAT.', group: 'Converting' }, + // ---- more numbers, for numerical work + ASIN: { kind: 'function', sig: 'ASIN(X)', doc: 'The angle in radians whose sine is X, in [-PI()/2, PI()/2]; X must lie in [-1, 1].', group: 'Numbers' }, + ACOS: { kind: 'function', sig: 'ACOS(X)', doc: 'The angle in radians whose cosine is X, in [0, PI()]; X must lie in [-1, 1].', group: 'Numbers' }, + ATAN: { kind: 'function', sig: 'ATAN(X)', doc: 'The angle in radians whose tangent is X, in [-PI()/2, PI()/2] (the ends for the largest FLOATs, where the exact angle rounds to them).', group: 'Numbers' }, + SINH: { kind: 'function', sig: 'SINH(X)', doc: 'The hyperbolic sine of X, as a FLOAT.', group: 'Numbers' }, + COSH: { kind: 'function', sig: 'COSH(X)', doc: 'The hyperbolic cosine of X, as a FLOAT.', group: 'Numbers' }, + TANH: { kind: 'function', sig: 'TANH(X)', doc: 'The hyperbolic tangent of X, as a FLOAT, in [-1, 1]: exactly 1.0 or -1.0 once X is past about 19 either way.', group: 'Numbers' }, + HYPOT: { kind: 'function', sig: 'HYPOT(X, Y)', doc: 'SQRT(X * X + Y * Y), computed without overflowing on the way when X or Y is very large.', group: 'Numbers' }, + POW: { kind: 'function', sig: 'POW(X, Y)', doc: 'X raised to the power Y, as a FLOAT; X must not be negative unless Y is whole, nor 0 when Y is negative. With USE IEEE it follows IEEE 754 pow: POW(1.0, NAN()) is 1.0.', group: 'Numbers' }, + CBRT: { kind: 'function', sig: 'CBRT(X)', doc: 'The cube root of X, as a FLOAT; defined for negative X too.', group: 'Numbers' }, + EXPM1: { kind: 'function', sig: 'EXPM1(X)', doc: 'EXP(X) - 1, accurate when X is close to 0, where the subtraction would lose every digit.', group: 'Numbers' }, + LOG1P: { kind: 'function', sig: 'LOG1P(X)', doc: 'LOG(1 + X), accurate when X is close to 0; X must be above -1.', group: 'Numbers' }, + E: { kind: 'function', sig: 'E()', doc: 'The number e, 2.71828…, as a FLOAT.', group: 'Numbers' }, + TRUNC: { kind: 'function', sig: 'TRUNC(X)', doc: 'X with its fraction dropped toward zero, kept as a FLOAT: TRUNC(-2.7) is -2.0. INT does the same and gives an INTEGER.', group: 'Numbers' }, + SIGN: { kind: 'function', sig: 'SIGN(X)', doc: '-1, 0 or 1, as X is negative, zero or positive, in the type of X.', group: 'Numbers' }, + COPYSIGN: { kind: 'function', sig: 'COPYSIGN(X, Y)', doc: 'The size of X with the sign of Y, as a FLOAT.', group: 'Numbers' }, + CLAMP: { kind: 'function', sig: 'CLAMP(X, Lo, Hi)', doc: 'X held inside [Lo, Hi]: Lo when X is below it, Hi when X is above it, X otherwise; Lo must not exceed Hi.', group: 'Numbers' }, + FMA: { kind: 'function', sig: 'FMA(A, B, C)', doc: 'A * B + C with a single rounding at the end, as the processor\'s fused multiply-add computes it.', group: 'Precision' }, + // ---- floating point itself + IS_NAN: { kind: 'function', sig: 'IS_NAN(X)', doc: 'TRUE when X is not a number. Only a program that says USE IEEE can hold one.', group: 'Precision' }, + IS_INF: { kind: 'function', sig: 'IS_INF(X)', doc: 'TRUE when X is infinity or minus infinity. Only a program that says USE IEEE can hold one.', group: 'Precision' }, + IS_FINITE: { kind: 'function', sig: 'IS_FINITE(X)', doc: 'TRUE when X is an ordinary number, neither infinite nor not a number.', group: 'Precision' }, + INF: { kind: 'function', sig: 'INF()', doc: 'Positive infinity. A program has to say USE IEEE to hold it; otherwise the run stops.', group: 'Precision' }, + NAN: { kind: 'function', sig: 'NAN()', doc: 'Not a number. A program has to say USE IEEE to hold it; otherwise the run stops.', group: 'Precision' }, + ULP: { kind: 'function', sig: 'ULP(X)', doc: 'The gap between X and the next FLOAT away from zero: one unit in the last place, the size of X\'s rounding step. For a FLOAT32 X, the gap between singles. With USE IEEE, ULP of an infinity is infinity.', group: 'Precision' }, + NEXT_AFTER: { kind: 'function', sig: 'NEXT_AFTER(X, Y)', doc: 'The FLOAT next to X in the direction of Y (a FLOAT32 next to X, for a FLOAT32 X).', group: 'Precision' }, + IS_CLOSE: { kind: 'function', sig: 'IS_CLOSE(A, B, RelTol, AbsTol)', doc: 'TRUE when ABS(A - B) <= MAX(RelTol * MAX(ABS(A), ABS(B)), AbsTol): equality for measured values, with the tolerance said out loud.', group: 'Precision' }, + EXACT: { kind: 'function', sig: 'EXACT(X)', doc: 'The exact decimal value a FLOAT holds, as a STRING: EXACT(0.1) is "0.1000000000000000055511151231257827021181583404541015625".', group: 'Precision' }, + // ---- the limits of a type + MAX_VALUE: { kind: 'function', sig: 'MAX_VALUE(T)', doc: 'The largest value of the numeric type T: MAX_VALUE(INT8) is 127, MAX_VALUE(FLOAT64) about 1.8 × 10^308.', group: 'Precision' }, + MIN_VALUE: { kind: 'function', sig: 'MIN_VALUE(T)', doc: 'The lowest value of the numeric type T: MIN_VALUE(INT8) is -128, MIN_VALUE(FLOAT64) is -MAX_VALUE(FLOAT64).', group: 'Precision' }, + EPSILON: { kind: 'function', sig: 'EPSILON(T)', doc: 'For FLOAT32 or FLOAT64, the gap between 1.0 and the next value above it: 2^-23 or 2^-52.', group: 'Precision' }, + // ---- whole numbers bit by bit, and arithmetic that wraps + GCD: { kind: 'function', sig: 'GCD(A, B)', doc: 'The greatest common divisor of two whole numbers; GCD(0, 0) is 0.', group: 'Whole numbers' }, + LCM: { kind: 'function', sig: 'LCM(A, B)', doc: 'The least common multiple of two whole numbers; 0 when either is 0.', group: 'Whole numbers' }, + BIT_AND: { kind: 'function', sig: 'BIT_AND(A, B)', doc: 'The bits set in both A and B (two\'s complement for negative values).', group: 'Whole numbers' }, + BIT_OR: { kind: 'function', sig: 'BIT_OR(A, B)', doc: 'The bits set in A or B or both.', group: 'Whole numbers' }, + BIT_XOR: { kind: 'function', sig: 'BIT_XOR(A, B)', doc: 'The bits set in exactly one of A and B.', group: 'Whole numbers' }, + BIT_NOT: { kind: 'function', sig: 'BIT_NOT(A)', doc: 'Every bit of A flipped, in A\'s type: BIT_NOT of a UINT8 0 is 255, of an INT8 or INTEGER 0 is -1.', group: 'Whole numbers' }, + SHIFT_LEFT: { kind: 'function', sig: 'SHIFT_LEFT(A, N)', doc: 'A times 2^N: the bits moved N places up. A result outside A\'s type stops the run, as any overflow does.', group: 'Whole numbers' }, + SHIFT_RIGHT: { kind: 'function', sig: 'SHIFT_RIGHT(A, N)', doc: 'A divided by 2^N, rounded down: the bits moved N places down (the sign kept, for a negative A).', group: 'Whole numbers' }, + WRAP: { kind: 'function', sig: 'WRAP(A, T)', doc: 'The whole number A brought into the sized type T (up to 32 bits) by wrapping around, as hardware does: WRAP(300, UINT8) is 44. Arithmetic that is meant to wrap says so with WRAP.', group: 'Whole numbers' }, + KEYS: { kind: 'function', sig: 'KEYS(H)', doc: 'The keys of a HASHMAP as a LIST, smallest first.', group: 'Collections' }, + SORTED: { kind: 'function', sig: 'SORTED(X)', doc: 'The values of a LIST or SET as a new LIST, smallest first.', group: 'Collections' }, + APPEND: { kind: 'procedure', sig: 'APPEND(L, V)', doc: 'Adds V to the end of the LIST L.', group: 'Collections' }, + INSERT: { kind: 'procedure', sig: 'INSERT(L, I, V)', doc: 'Puts V at position I of the LIST L, moving every later value one place along.', group: 'Collections' }, + REMOVE: { kind: 'procedure', sig: 'REMOVE(L, I)', doc: 'Takes the value at position I out of the LIST L, moving every later value one place back.', group: 'Collections' }, + ADD: { kind: 'procedure', sig: 'ADD(S, V)', doc: 'Makes V a member of the SET S. Adding a member already there changes nothing.', group: 'Collections' }, + DELETE: { kind: 'procedure', sig: 'DELETE(X, K)', doc: 'Removes the key K from a HASHMAP, or the member K from a SET.', group: 'Collections' }, + SORT: { kind: 'procedure', sig: 'SORT(L)', doc: 'Puts the values of the LIST L in order, smallest first. Equal values keep their order.', group: 'Collections' }, +}; + +/** + * A sized type's name followed by brackets converts a number to that type, as Mojo writes Int32(x): + * INT32(X) and the other whole-number types stop the run when X does not fit (and drop a FLOAT's fraction toward + * zero first); FLOAT32(X) rounds to the nearest single; FLOAT64(X) gives a FLOAT. + */ +export const CONVERSIONS: Record = Object.fromEntries(Object.keys(SIZED).map((n) => [n, { + sig: `${n}(X)`, + doc: SIZED[n].k === 'INTEGER' ? `X as ${n}, from ${intRangeText(SIZED[n])}; a FLOAT loses its fraction toward zero first, and a value outside the range stops the run.` : n === 'FLOAT32' ? 'X rounded to the nearest FLOAT32 (single precision, about 7 significant digits).' : 'X as a FLOAT64 (the same type as FLOAT).', + group: 'Converting', +}])); + +/** Built-ins that take a type's name as an argument, and at which position. */ +export const TYPE_ARGS: Record = { MAX_VALUE: 0, MIN_VALUE: 0, EPSILON: 0, WRAP: 1 }; + +/** + * Built-ins added for numerical work after programs were already being written. A program may still name its + * own variable E or define its own FUNCTION GCD. A FUNCTION or PROCEDURE of the same name replaces the built-in + * inside the program. A variable, constant or parameter does not: it is read as E, and E() still calls the + * built-in, since a name followed by brackets is a call and a variable cannot be called (tests/numeric.test.mjs, + * "names"). The older built-ins stay reserved. + */ +export const SOFT_BUILTINS = new Set([ + 'ASIN', 'ACOS', 'ATAN', 'SINH', 'COSH', 'TANH', 'HYPOT', 'POW', 'CBRT', 'EXPM1', 'LOG1P', 'E', 'TRUNC', 'SIGN', + 'COPYSIGN', 'CLAMP', 'FMA', 'IS_NAN', 'IS_INF', 'IS_FINITE', 'INF', 'NAN', 'ULP', 'NEXT_AFTER', 'IS_CLOSE', 'EXACT', + 'MAX_VALUE', 'MIN_VALUE', 'EPSILON', 'GCD', 'LCM', 'BIT_AND', 'BIT_OR', 'BIT_XOR', 'BIT_NOT', 'SHIFT_LEFT', + 'SHIFT_RIGHT', 'WRAP', +]); + +/** Built-ins that change their first argument in place. */ +export const MUTATORS = new Set(['APPEND', 'INSERT', 'REMOVE', 'ADD', 'DELETE', 'SORT']); + +/* ------------------------------------------------------------------ */ +/* Operator precedence */ +/* ------------------------------------------------------------------ */ + +/** + * Loosest first. The table is the notation's whole answer to "which part of + * this expression happens first", including the question the old guide left + * open: DIV and MOD bind exactly as tightly as * and /. + */ +export const PRECEDENCE = [ + { level: 1, ops: ['OR'], assoc: 'left', note: 'TRUE when either side is; the right side is skipped once the left is TRUE' }, + { level: 2, ops: ['AND'], assoc: 'left', note: 'TRUE when both sides are; the right side is skipped once the left is FALSE' }, + { level: 3, ops: ['NOT'], assoc: 'prefix', note: 'NOT A == B means NOT (A == B)' }, + { level: 4, ops: ['==', '!=', '<', '<=', '>', '>=', 'IN'], assoc: 'chain', note: '0 <= I < N means 0 <= I AND I < N' }, + { level: 5, ops: ['&'], assoc: 'left', note: 'joins STRING and CHAR values into a STRING' }, + { level: 6, ops: ['+', '-'], assoc: 'left', note: '' }, + { level: 7, ops: ['*', '/', 'DIV', 'MOD'], assoc: 'left', note: '/ always gives a FLOAT' }, + { level: 8, ops: ['- (negation)'], assoc: 'prefix', note: '' }, + { level: 9, ops: ['^'], assoc: 'right', note: '-2 ^ 2 is -4; 2 ^ 3 ^ 2 is 2 ^ 9' }, + { level: 10, ops: ['X[I]', 'X.F', 'F(X)'], assoc: 'postfix', note: 'indexing, fields, and calls bind tightest' }, +]; diff --git a/src/engines/pseudo/logic.ts b/src/engines/pseudo/logic.ts new file mode 100644 index 0000000..7df2691 --- /dev/null +++ b/src/engines/pseudo/logic.ts @@ -0,0 +1,706 @@ +/** + * pseudo/logic.js + * + * The verifier's own small language of terms and formulas, with the + * rewrite rules that make reasoning about sequences possible. A formula is + * just a term of type 'bool'. + * + * Term kinds + * num { v, t: 'int' | 'float' } a number + * lit { v, t: 'char' | 'string' | 'bool' } + * sym { name, id, t } a value the program cannot see into: a + * parameter, or a variable after a loop has + * changed it an unknown number of times + * bv { name, t } a quantifier's own name, inside its body + * op { op, args, t } + - * / DIV MOD ^ neg & + * and / or { args } not { a } cmp { op, a, b } + * ite { c, a, b, t } if c then a else b + * app { f, args, t } LENGTH, AT (indexing), APPENDED, STORE, + * IN, a user FUNCTION, a built-in ... + * q { q, v, lo, hi, body, t } SIZE, SORTED, FORALL, EXISTS, SUM, SETOF + * over the range [lo, hi) + * qc { q, v, dom, body, t } the same over a collection; opaque + */ + +import { typeName } from './lang.js'; + +/* ------------------------------------------------------------------ */ +/* The term language, as types */ +/* ------------------------------------------------------------------ */ + +/** A term's sort: 'int', 'float', 'char', 'string', 'bool', 'list', 'array', 'map', 'set', 'record' or 'any'. */ +export type Sort = string; +export type CmpOp = '==' | '!=' | '<' | '<=' | '>' | '>='; + +export interface Num { k: 'num'; v: number; t: Sort } +export interface Lit { k: 'lit'; v: string | boolean; t: Sort } +export interface Sym { k: 'sym'; name: string; id: number; t: Sort } +export interface Bv { k: 'bv'; name: string; t: Sort } +export interface Op { k: 'op'; op: string; args: Term[]; t: Sort } +export interface And { k: 'and'; args: Term[]; t: 'bool' } +export interface Or { k: 'or'; args: Term[]; t: 'bool' } +export interface Not { k: 'not'; a: Term; t: 'bool' } +export interface Cmp { k: 'cmp'; op: CmpOp; a: Term; b: Term; t: 'bool' } +export interface Ite { k: 'ite'; c: Term; a: Term; b: Term; t: Sort } +export interface App { k: 'app'; f: string; args: Term[]; t: Sort } +/** A quantifier over the range [lo, hi). */ +export interface Quant { k: 'q'; q: string; v: string; lo: Term; hi: Term; body: Term; t: Sort } +/** A quantifier over a collection; opaque to the solver. */ +export interface QuantC { k: 'qc'; q: string; v: string; dom: Term; body: Term; t: Sort } + +export type Term = Num | Lit | Sym | Bv | Op | And | Or | Not | Cmp | Ite | App | Quant | QuantC; + +/** A type as the parser writes it: `{ k: 'INTEGER' }`, `{ k: 'LIST', of: ... }`. */ +export interface TypeLike { k: string } + +/* ------------------------------------------------------------------ */ +/* Work, counted rather than timed */ +/* ------------------------------------------------------------------ */ + +/** + * What the student reads where the prover stopped for want of work: an obligation it did not finish is open with + * this reason, never proved (docs/limits.md). + */ +export const TOO_LARGE = 'too large to prove here'; + +/** Thrown where the work a budget allows runs out; `metered` turns it into the budget's own answer. */ +export class OutOfWork extends Error { + constructor() { super(TOO_LARGE); this.name = 'OutOfWork'; } +} + +/** Units of work left in the innermost budget (Infinity outside every budget). */ +let workLeft = Infinity; + +/** + * Count `n` units of work. The prover's work is counted, not timed, where it allocates: a term visited or built, + * a case of a case split, a pair of constraints combined, a fact copied, a key by its length. A budget then runs + * out at the same point on every machine and in every test, and memory stays in proportion to it: nested ABS, + * whose case split doubles with every level, once took a 1 GB heap down (tests/prover-limits.test.mjs). + */ +export function work(n = 1): void { + if ((workLeft -= n) < 0) throw new OutOfWork(); +} + +/** + * Run `fn` with at most `n` units of work, and no more than the enclosing budget has left. Past `n`, `out` gives + * the answer instead; when it is the enclosing budget that runs out first, OutOfWork goes on to it. The work done + * here is taken from the enclosing budget either way. + */ +export function metered(n: number, fn: () => T, out: () => T): T { + const outer = workLeft; + const own = n < outer; + const start = own ? n : outer; + workLeft = start; + let result: T; + try { + result = fn(); + } catch (e) { + const used = start - workLeft; + workLeft = outer - used; + if (own && e instanceof OutOfWork) return out(); + throw e; + } + workLeft = outer - (start - workLeft); + return result; +} + +/** Run `fn` outside every budget: for work that is bounded on its own, such as writing a term out with `show`. */ +export function unmetered(fn: () => T): T { + const saved = workLeft; + workLeft = Infinity; + try { return fn(); } finally { workLeft = saved; } +} + +/* ------------------------------------------------------------------ */ +/* Types */ +/* ------------------------------------------------------------------ */ + +export function sortOf(type: TypeLike | null | undefined): Sort { + if (!type) return 'any'; + switch (type.k) { + case 'INTEGER': return 'int'; + case 'FLOAT': return 'float'; + case 'CHAR': return 'char'; + case 'STRING': return 'string'; + case 'BOOL': return 'bool'; + case 'LIST': return 'list'; + case 'ARRAY': return 'array'; + case 'HASHMAP': return 'map'; + case 'SET': return 'set'; + case 'RECORD': return 'record'; + default: return 'any'; + } +} +export const numeric = (t: Sort | undefined): boolean => t === 'int' || t === 'float'; +export const isNum = (x: Term): boolean => numeric(x.t); + +/* ------------------------------------------------------------------ */ +/* Constructors, simplifying as they go */ +/* ------------------------------------------------------------------ */ + +export const TRUE: Lit = { k: 'lit', v: true, t: 'bool' }; +export const FALSE: Lit = { k: 'lit', v: false, t: 'bool' }; +export const num = (v: number, t?: Sort): Num => ({ k: 'num', v, t: t || (Number.isInteger(v) ? 'int' : 'float') }); +export const lit = (v: string | boolean, t: Sort): Lit => ({ k: 'lit', v, t }); +let symIds = 0; +export const sym = (name: string, t: Sort): Sym => ({ k: 'sym', name, id: ++symIds, t }); +export const bv = (name: string, t: Sort): Bv => ({ k: 'bv', name, t }); + +const isConst = (x: Term): x is Num => x.k === 'num'; +const isTrue = (x: Term): boolean => x.k === 'lit' && x.v === true; +const isFalse = (x: Term): boolean => x.k === 'lit' && x.v === false; + +export function numType(a: Term, b?: Term | null): Sort { return a.t === 'float' || b?.t === 'float' ? 'float' : 'int'; } + +/** + * Whether two constants may be folded into `r`: only whole numbers a double holds exactly, with an exact result. + * Anything else stays a term, and linear() adds it up in exact rationals: 9007199254740991 + 2 folded in doubles + * is 9007199254740992 (library/pseudo/prover/constant-doubles.pseudo). + */ +const exact = (a: Num, b: Num, r: number): boolean => Number.isSafeInteger(a.v) && Number.isSafeInteger(b.v) && Number.isSafeInteger(r); + +export function add(a: Term, b: Term): Term { + if (isConst(a) && isConst(b) && exact(a, b, a.v + b.v)) return num(a.v + b.v, numType(a, b)); + if (isConst(a) && a.v === 0) return b; + if (isConst(b) && b.v === 0) return a; + return { k: 'op', op: '+', args: [a, b], t: numType(a, b) }; +} +export function sub(a: Term, b: Term): Term { + if (isConst(a) && isConst(b) && exact(a, b, a.v - b.v)) return num(a.v - b.v, numType(a, b)); + if (isConst(b) && b.v === 0) return a; + if (key(a) === key(b)) return num(0, numType(a, b)); + return { k: 'op', op: '-', args: [a, b], t: numType(a, b) }; +} +export function mul(a: Term, b: Term): Term { + if (isConst(a) && isConst(b) && exact(a, b, a.v * b.v)) return num(a.v * b.v, numType(a, b)); + if ((isConst(a) && a.v === 0) || (isConst(b) && b.v === 0)) return num(0, numType(a, b)); + if (isConst(a) && a.v === 1) return b; + if (isConst(b) && b.v === 1) return a; + return { k: 'op', op: '*', args: [a, b], t: numType(a, b) }; +} +export function neg(a: Term): Term { return isConst(a) ? num(-a.v, a.t) : { k: 'op', op: 'neg', args: [a], t: a.t }; } +export function arith(op: string, a: Term, b: Term, t?: Sort): Term { + switch (op) { + case '+': return add(a, b); + case '-': return sub(a, b); + case '*': return mul(a, b); + case '/': + if (isConst(a) && isConst(b) && b.v !== 0) return num(a.v / b.v, 'float'); + if (isConst(b) && b.v !== 0) return { k: 'op', op: '*', args: [num(1 / b.v, 'float'), a], t: 'float' }; + return { k: 'op', op: '/', args: [a, b], t: 'float' }; + case 'DIV': + if (isConst(a) && isConst(b) && b.v !== 0 && exact(a, b, 0)) return num(Math.floor(a.v / b.v), 'int'); + if (isConst(b) && b.v === 1) return a; + return { k: 'op', op: 'DIV', args: [a, b], t: 'int' }; + case 'MOD': + if (isConst(a) && isConst(b) && b.v !== 0 && exact(a, b, 0)) return num(a.v - b.v * Math.floor(a.v / b.v), 'int'); + return { k: 'op', op: 'MOD', args: [a, b], t: 'int' }; + case '^': + if (isConst(a) && isConst(b) && exact(a, b, a.v ** b.v)) return num(a.v ** b.v, t === 'float' ? 'float' : numType(a, b)); + return { k: 'op', op: '^', args: [a, b], t: t || numType(a, b) }; + case '&': + if (a.k === 'lit' && b.k === 'lit') return lit(String(a.v) + String(b.v), 'string'); + return { k: 'op', op: '&', args: [a, b], t: 'string' }; + default: return { k: 'op', op, args: [a, b], t: t || 'any' }; + } +} + +export function and(...xs: Array): Term { + const out: Term[] = []; + for (const x of xs.flat()) { + if (!x || isTrue(x)) continue; + if (isFalse(x)) return FALSE; + if (x.k === 'and') out.push(...x.args); else out.push(x); + } + if (!out.length) return TRUE; + return out.length === 1 ? out[0] : { k: 'and', args: out, t: 'bool' }; +} +export function or(...xs: Array): Term { + const out: Term[] = []; + for (const x of xs.flat()) { + if (!x || isFalse(x)) continue; + if (isTrue(x)) return TRUE; + if (x.k === 'or') out.push(...x.args); else out.push(x); + } + if (!out.length) return FALSE; + return out.length === 1 ? out[0] : { k: 'or', args: out, t: 'bool' }; +} +const FLIP: Record = { '==': '!=', '!=': '==', '<': '>=', '<=': '>', '>': '<=', '>=': '<' }; +export function not(a: Term): Term { + if (isTrue(a)) return FALSE; + if (isFalse(a)) return TRUE; + if (a.k === 'not') return a.a; + if (a.k === 'cmp' && (numeric(a.a.t) || a.op === '==' || a.op === '!=')) return cmp(FLIP[a.op], a.a, a.b); + return { k: 'not', a, t: 'bool' }; +} +export function implies(a: Term, b: Term): Term { return or(not(a), b); } + +export function cmp(op: CmpOp, a: Term, b: Term): Term { + if (isConst(a) && isConst(b)) return evalCmp(op, a.v, b.v) ? TRUE : FALSE; + if (a.k === 'lit' && b.k === 'lit' && (op === '==' || op === '!=')) return (a.v === b.v) === (op === '==') ? TRUE : FALSE; + if ((op === '==' || op === '!=') && key(a) === key(b)) return op === '==' ? TRUE : FALSE; + if (op === '==' || op === '!=') { + const r = listEquality(a, b); + if (r) return op === '==' ? r : not(r); + if (a.t === 'bool' || b.t === 'bool') { + if (isTrue(b)) return op === '==' ? a : not(a); + if (isFalse(b)) return op === '==' ? not(a) : a; + if (isTrue(a)) return op === '==' ? b : not(b); + if (isFalse(a)) return op === '==' ? not(b) : b; + const same = or(and(a, b), and(not(a), not(b))); + return op === '==' ? same : not(same); + } + } + return { k: 'cmp', op, a, b, t: 'bool' }; +} +function evalCmp(op: CmpOp, x: number, y: number): boolean { + switch (op) { case '==': return x === y; case '!=': return x !== y; case '<': return x < y; case '<=': return x <= y; case '>': return x > y; default: return x >= y; } +} + +/** Two lists are equal when they have the same length and the same values in order. */ +function listEquality(a: Term, b: Term): Term | null { + const asList = (x: Term): Term[] | null => (x.k === 'app' && x.f === 'LIST' ? x.args : null); + const la = asList(a); const lb = asList(b); + if (la && lb) return la.length !== lb.length ? FALSE : and(la.map((x, i) => cmp('==', x, lb[i]))); + const ap = (x: Term): App | null => (x.k === 'app' && x.f === 'APPENDED' ? x : null); + const pa = ap(a); const pb = ap(b); + if (pa && pb) return and(cmp('==', pa.args[0], pb.args[0]), cmp('==', pa.args[1], pb.args[1])); + if ((pa && la && !la.length) || (pb && lb && !lb.length)) return FALSE; + if (pa && lb && lb.length) return and(cmp('==', pa.args[0], app('LIST', lb.slice(0, -1), 'list')), cmp('==', pa.args[1], lb[lb.length - 1])); + if (pb && la && la.length) return and(cmp('==', pb.args[0], app('LIST', la.slice(0, -1), 'list')), cmp('==', pb.args[1], la[la.length - 1])); + return null; +} + +export function ite(c: Term, a: Term, b: Term): Term { + if (isTrue(c)) return a; + if (isFalse(c)) return b; + if (key(a) === key(b)) return a; + if (a.t === 'bool') return or(and(c, a), and(not(c), b)); + return { k: 'ite', c, a, b, t: a.t }; +} + +export function app(f: string, args: Term[], t: Sort): App { return { k: 'app', f, args, t }; } + +/** X[I], looking through APPENDED and STORE to the value actually there. */ +export function at(x: Term, i: Term, t: Sort): Term { + if (x.k === 'app' && x.f === 'APPENDED') return ite(cmp('==', i, length(x.args[0])), x.args[1], at(x.args[0], i, t)); + if (x.k === 'app' && x.f === 'STORE') return ite(cmp('==', i, x.args[1]), x.args[2], at(x.args[0], i, t)); + if (x.k === 'app' && x.f === 'MAPPUT') return ite(cmp('==', i, x.args[1]), x.args[2], at(x.args[0], i, t)); + if (x.k === 'app' && x.f === 'LIST' && isConst(i) && i.v >= 0 && i.v < x.args.length) return x.args[i.v]; + if (x.k === 'lit' && x.t === 'string' && isConst(i)) { const cs = Array.from(String(x.v)); if (i.v >= 0 && i.v < cs.length) return lit(cs[i.v], 'char'); } + if (x.k === 'app' && x.f === 'SLICE') return at(x.args[0], add(x.args[1], i), t); + return app('AT', [x, i], t); +} +export function at2(x: Term, r: Term, c: Term, t: Sort): Term { + if (x.k === 'app' && x.f === 'STORE2') return ite(and(cmp('==', r, x.args[1]), cmp('==', c, x.args[2])), x.args[3], at2(x.args[0], r, c, t)); + return app('AT2', [x, r, c], t); +} + +export function length(x: Term): Term { + if (x.k === 'app') { + switch (x.f) { + case 'APPENDED': return add(length(x.args[0]), num(1)); + case 'STORE': case 'STORE2': case 'SORTLIST': return length(x.args[0]); + case 'LIST': return num(x.args.length); + case 'EMPTY': return num(0); + case 'NEWARRAY': return x.args[1]; + case 'SLICE': return sub(x.args[2], x.args[1]); + case 'SORTEDOF': return length(x.args[0]); + default: break; + } + } + if (x.k === 'lit' && x.t === 'string') return num(Array.from(String(x.v)).length); + if (x.k === 'op' && x.op === '&') return add(length(x.args[0]), length(x.args[1])); + if (x.t === 'char') return num(1); + if (x.k === 'q' && x.q === 'SETOF') return { k: 'q', q: 'SIZE', v: x.v, lo: x.lo, hi: x.hi, body: x.body, t: 'int' }; + if (x.k === 'q' && x.q === 'SORTED') return { k: 'q', q: 'SIZE', v: x.v, lo: x.lo, hi: x.hi, body: x.body, t: 'int' }; + return app('LENGTH', [x], 'int'); +} + +/** X IN S, unfolded wherever S's shape allows. */ +export function member(x: Term, s: Term): Term { + // Only a whole number is in a set written by rule over a range, and a run's SET finds a collection by identity, not + // by its contents: in both cases the answer is left open (library/pseudo/prover/float-in-range.pseudo, set-of-lists.pseudo). + if (s.k === 'q' && (s.q === 'SETOF' || s.q === 'SORTED') && x.t !== 'int') return app('IN', [x, s], 'bool'); + if (s.t === 'set' && ['list', 'array', 'map', 'set', 'record'].includes(x.t)) return app('IN', [x, s], 'bool'); + if (s.k === 'q' && s.q === 'SETOF') return and(cmp('<=', s.lo, x), cmp('<', x, s.hi), substBound(s.body, s.v, x)); + if (s.k === 'app' && (s.f === 'SET' || s.f === 'LIST')) return or(s.args.map((y) => cmp('==', x, y))); + if (s.k === 'app' && (s.f === 'SETADD' || s.f === 'APPENDED')) return or(cmp('==', x, s.args[1]), member(x, s.args[0])); + if (s.k === 'app' && s.f === 'MAPPUT') return or(cmp('==', x, s.args[1]), member(x, s.args[0])); + if (s.k === 'app' && (s.f === 'SETDEL' || s.f === 'MAPDEL')) return and(cmp('!=', x, s.args[1]), member(x, s.args[0])); + if (s.k === 'app' && s.f === 'EMPTY') return FALSE; + if (s.k === 'q' && s.q === 'SORTED') return and(cmp('<=', s.lo, x), cmp('<', x, s.hi), substBound(s.body, s.v, x)); + return app('IN', [x, s], 'bool'); +} + +/* ------------------------------------------------------------------ */ +/* Keys: one string per term, equal for equal terms */ +/* ------------------------------------------------------------------ */ + +/** + * The longest key written out in full. A term shares its parts (ABS(X) is IF X >= 0 THEN X ELSE -X, so each ABS + * around it holds X three times), and a key written out in full is as long as the term unshared: 20 nested ABS + * make 3^20 copies of X. A key longer than this one is replaced by a name of its own (`#big` and a number), equal + * only to itself. Two such terms are then never found equal by their keys, which only keeps the prover from + * concluding something (a key is used to find equal terms, never different ones); no program of the corpus has a + * key a tenth this long (tests/prover-limits.test.mjs). + */ +export const MAX_KEY = 2000; +let bigKeys = 0; + +/** Keys of compound terms: at the top level, and under binders by the binders' names (a `bv`'s key depends on them). */ +const KEY = new WeakMap(); +const BOUND_KEY = new WeakMap>(); +export function key(x: Term | null | undefined): string { + if (!x || typeof x !== 'object') return String(x); + return keyIn(x, []); +} +/** + * The compound parts a key is written from directly, as a tree. Past them it is made from its parts' own keys, + * each made once and kept: a term that shares its parts then costs its number of distinct parts, not its size + * written out. + */ +const FLAT_PARTS = 400; +let flatLeft = 0; +const TOO_MANY_PARTS = { tooMany: true }; +function keyIn(x: Term, bound: string[]): string { + if (!bound.length) { const hit = KEY.get(x); if (hit !== undefined) return hit; } + switch (x.k) { + case 'num': case 'lit': case 'sym': { const k = flat(x, bound); if (!bound.length) KEY.set(x, k); return k; } + case 'bv': return flat(x, bound); + default: break; + } + const ctx = bound.length ? JSON.stringify(bound) : null; + let memo: Map | undefined; + if (ctx !== null) { memo = BOUND_KEY.get(x); const hit = memo?.get(ctx); if (hit !== undefined) return hit; } + let k: string; + flatLeft = FLAT_PARTS; + try { k = flat(x, bound); } catch (e) { + if (e !== TOO_MANY_PARTS) throw e; + work(8); + k = fromParts(x, bound); + } + work(1 + (k.length >> 4)); + if (k.length > MAX_KEY) k = `#big${++bigKeys}`; + if (ctx === null) KEY.set(x, k); + else { if (!memo) { memo = new Map(); BOUND_KEY.set(x, memo); } memo.set(ctx, k); } + return k; +} +/** A key written out from the term as a tree; past FLAT_PARTS compound parts it gives up (TOO_MANY_PARTS). */ +function flat(x: Term, bound: string[]): string { + switch (x.k) { + case 'num': return `${x.v}`; + case 'lit': return x.t === 'bool' ? (x.v ? 'TRUE' : 'FALSE') : `${x.t}:${JSON.stringify(x.v)}`; + case 'sym': return `${x.name}#${x.id}`; + case 'bv': { const i = bound.lastIndexOf(x.name); return i >= 0 ? `$${bound.length - 1 - i}` : `bv:${x.name}`; } + default: break; + } + if (--flatLeft < 0) throw TOO_MANY_PARTS; + switch (x.k) { + case 'op': return `(${x.op} ${x.args.map((a) => flat(a, bound)).join(' ')})`; + case 'and': case 'or': return `(${x.k} ${x.args.map((a) => flat(a, bound)).join(' ')})`; + case 'not': return `(not ${flat(x.a, bound)})`; + case 'cmp': return `(${x.op} ${flat(x.a, bound)} ${flat(x.b, bound)})`; + case 'ite': return `(ite ${flat(x.c, bound)} ${flat(x.a, bound)} ${flat(x.b, bound)})`; + case 'app': return `(${x.f} ${x.args.map((a) => flat(a, bound)).join(' ')})`; + case 'q': return `(${x.q} ${flat(x.lo, bound)} ${flat(x.hi, bound)} ${flat(x.body, [...bound, x.v])})`; + case 'qc': return `(${x.q}c ${flat(x.dom, bound)} ${flat(x.body, [...bound, x.v])})`; + default: return '?'; + } +} +/** The same key, made from the keys of the term's parts (each made once and kept). */ +function fromParts(x: Term, bound: string[]): string { + switch (x.k) { + case 'op': return `(${x.op} ${x.args.map((a) => keyIn(a, bound)).join(' ')})`; + case 'and': case 'or': return `(${x.k} ${x.args.map((a) => keyIn(a, bound)).join(' ')})`; + case 'not': return `(not ${keyIn(x.a, bound)})`; + case 'cmp': return `(${x.op} ${keyIn(x.a, bound)} ${keyIn(x.b, bound)})`; + case 'ite': return `(ite ${keyIn(x.c, bound)} ${keyIn(x.a, bound)} ${keyIn(x.b, bound)})`; + case 'app': return `(${x.f} ${x.args.map((a) => keyIn(a, bound)).join(' ')})`; + case 'q': return `(${x.q} ${keyIn(x.lo, bound)} ${keyIn(x.hi, bound)} ${keyIn(x.body, [...bound, x.v])})`; + case 'qc': return `(${x.q}c ${keyIn(x.dom, bound)} ${keyIn(x.body, [...bound, x.v])})`; + default: return '?'; + } +} +/** The body of a quantifier, keyed so that renaming its variable changes nothing. */ +export function bodyKey(q: Quant | QuantC): string { return keyIn(q.body, [q.v]); } + +/* ------------------------------------------------------------------ */ +/* Substitution */ +/* ------------------------------------------------------------------ */ + +/** Replace a quantifier's variable with a term throughout its body. */ +export function substBound(body: Term, name: string, value: Term): Term { + return rebuild(body, (x, depthNames) => (x.k === 'bv' && x.name === name && !depthNames.includes(name) ? value : null), []); +} + +/** Replace symbols, by key, throughout a term. */ +export function substSyms(t: Term, map: Map): Term { + if (!map.size) return t; + return rebuild(t, (x) => (x.k === 'sym' && map.has(key(x)) ? map.get(key(x)) : null), []); +} + +/** Rebuild a term bottom-up through the smart constructors, so rewrites cascade. */ +/** A rewrite step: a replacement for a term, or null to keep rebuilding below it. */ +export type Rewrite = (x: Term, shadow: string[]) => Term | null | undefined; +export function rebuild(x: Term, fn: Rewrite, shadow: string[] = []): Term { + work(); + const r = fn(x, shadow); + if (r) return r; + switch (x.k) { + case 'num': case 'lit': case 'sym': case 'bv': return x; + case 'op': { + const args = x.args.map((a) => rebuild(a, fn, shadow)); + return x.op === 'neg' ? neg(args[0]) : arith(x.op, args[0], args[1], x.t); + } + case 'and': return and(x.args.map((a) => rebuild(a, fn, shadow))); + case 'or': return or(x.args.map((a) => rebuild(a, fn, shadow))); + case 'not': return not(rebuild(x.a, fn, shadow)); + case 'cmp': return cmp(x.op, rebuild(x.a, fn, shadow), rebuild(x.b, fn, shadow)); + case 'ite': return ite(rebuild(x.c, fn, shadow), rebuild(x.a, fn, shadow), rebuild(x.b, fn, shadow)); + case 'app': { + const args = x.args.map((a) => rebuild(a, fn, shadow)); + switch (x.f) { + case 'AT': return at(args[0], args[1], x.t); + case 'AT2': return at2(args[0], args[1], args[2], x.t); + case 'LENGTH': return length(args[0]); + case 'IN': return member(args[0], args[1]); + default: return app(x.f, args, x.t); + } + } + case 'q': { + const inner = [...shadow, x.v]; + return mkq(x.q, x.v, rebuild(x.lo, fn, shadow), rebuild(x.hi, fn, shadow), rebuild(x.body, fn, inner), x.t); + } + case 'qc': return { ...x, dom: rebuild(x.dom, fn, shadow), body: rebuild(x.body, fn, [...shadow, x.v]) }; + default: return x; + } +} + +/** A quantifier over [lo, hi), folded when the range is empty or a single position. */ +export function mkq(q: string, v: string, lo: Term, hi: Term, body: Term, t: Sort): Term { + const span = constDiff(hi, lo); + // Every quantifier mkq is given has an identity; an unknown one stays unfolded, which is still right. + if (span !== null && span <= 0) return identity(q, t) ?? { k: 'q', q, v, lo, hi, body, t }; + return { k: 'q', q, v, lo, hi, body, t }; +} + +export function identity(q: string, t: Sort): Term | null { + switch (q) { + case 'SIZE': return num(0); + case 'SUM': return num(0, t === 'float' ? 'float' : 'int'); + case 'FORALL': return TRUE; + case 'EXISTS': return FALSE; + case 'SORTED': return app('LIST', [], 'list'); + case 'SETOF': return app('EMPTY', [], 'set'); + default: return null; + } +} + +/** Add one more position, h, to a quantifier's range: the step that makes induction work. */ +export function extend(q: Quant, prev: Term, h: Term): Term | null { + const b = substBound(q.body, q.v, h); + switch (q.q) { + case 'SIZE': return add(prev, ite(b, num(1), num(0))); + // a whole-number SUM is exact; a FLOAT one rounds at every step, so one more position is the rounded FLOAT+ of the + // sum so far and the new value (0 + X is X exactly) (library/pseudo/prover/sum-float-rounding.pseudo) + case 'SUM': return q.t === 'int' ? add(prev, b) : prev.k === 'num' && prev.v === 0 ? b : app('FLOAT+', [prev, b], 'float'); + case 'FORALL': return and(prev, b); + case 'EXISTS': return or(prev, b); + case 'SORTED': return ite(b, app('APPENDED', [prev, h], 'list'), prev); + default: return null; + } +} + +/* ------------------------------------------------------------------ */ +/* Linear forms, for comparing bounds */ +/* ------------------------------------------------------------------ */ + +const babs = (x: bigint): bigint => (x < 0n ? -x : x); +export function bgcd(a: bigint, b: bigint): bigint { a = babs(a); b = babs(b); while (b) [a, b] = [b, a % b]; return a || 1n; } + +/** An exact rational number: constants and coefficients are added and multiplied in it, never in doubles. */ +export class Q { + n: bigint; + d: bigint; + constructor(n: bigint, d = 1n) { + if (d < 0n) { n = -n; d = -d; } + const g = bgcd(n, d); + this.n = n / g; this.d = d / g; + } + static of(x: number | Q): Q { + if (x instanceof Q) return x; + if (Number.isInteger(x)) return new Q(BigInt(x)); + const s = x.toString(); + if (/e/i.test(s)) { const [m, e] = s.split(/e/i); const q = Q.of(Number(m)); const p = 10n ** BigInt(Math.abs(Number(e))); return Number(e) >= 0 ? new Q(q.n * p, q.d) : new Q(q.n, q.d * p); } + const [i, f = ''] = s.split('.'); + const d = 10n ** BigInt(f.length); + return new Q(BigInt(i + f), d); + } + add(o: Q): Q { return new Q(this.n * o.d + o.n * this.d, this.d * o.d); } + sub(o: Q): Q { return new Q(this.n * o.d - o.n * this.d, this.d * o.d); } + mul(o: Q): Q { return new Q(this.n * o.n, this.d * o.d); } + div(o: Q): Q { return new Q(this.n * o.d, this.d * o.n); } + neg(): Q { return new Q(-this.n, this.d); } + get zero(): boolean { return this.n === 0n; } + get sign(): number { return this.n > 0n ? 1 : this.n < 0n ? -1 : 0; } + toString(): string { return this.d === 1n ? `${this.n}` : `${this.n}/${this.d}`; } +} + +/** a - b as a constant, when the difference is a plain number. */ +export function constDiff(a: Term, b: Term): number | null { + const la = linear(a); const lb = linear(b); + if (!la || !lb) return null; + const keys = new Set([...la.terms.keys(), ...lb.terms.keys()]); + for (const k of keys) if (!(la.terms.get(k) || Q_ZERO).sub(lb.terms.get(k) || Q_ZERO).zero) return null; + const d = la.c.sub(lb.c); + if (d.d === 1n) return Number(d.n); // a whole number; past 2^53 it keeps its sign and stays far from every small one + const f = Number(d.n) / Number(d.d); + return Number.isInteger(f) ? null : f; // never a whole number by rounding +} + +const Q_ZERO = new Q(0n); + +/** + * A term as sum(coefficient * atom) + constant. The coefficients and the constant are exact rationals: added as + * doubles, 9007199254740991 + 2 - 9007199254740991 comes to 1 (library/pseudo/prover/linear-doubles.pseudo). + */ +export function linear(x: Term): { terms: Map; c: Q } | null { + const terms = new Map(); + let c = Q_ZERO; + const isNumber = (t: Term): t is Num => t.k === 'num' && Number.isFinite(t.v); + const visit = (t: Term, m: Q): void => { + work(); + if (isNumber(t)) { c = c.add(m.mul(Q.of(t.v))); return; } + if (t.k === 'op' && t.op === '+') { visit(t.args[0], m); visit(t.args[1], m); return; } + if (t.k === 'op' && t.op === '-') { visit(t.args[0], m); visit(t.args[1], m.neg()); return; } + if (t.k === 'op' && t.op === 'neg') { visit(t.args[0], m.neg()); return; } + if (t.k === 'op' && t.op === '*' && isNumber(t.args[0])) { visit(t.args[1], m.mul(Q.of(t.args[0].v))); return; } + if (t.k === 'op' && t.op === '*' && isNumber(t.args[1])) { visit(t.args[0], m.mul(Q.of(t.args[1].v))); return; } + const k = key(t); + terms.set(k, (terms.get(k) || Q_ZERO).add(m)); + }; + visit(x, new Q(1n)); + for (const [k, v] of terms) if (v.zero) terms.delete(k); + return { terms, c }; +} + +/* ------------------------------------------------------------------ */ +/* Printing, as pseudocode */ +/* ------------------------------------------------------------------ */ + +/** + * Print a term the way a student would write it. `names` maps a symbol's + * key to its display name, so that two versions of I can read as I and I'. + */ +export function show(x: Term, names: Map = new Map(), prec = 0): string { + if (!showing) showLeft = SHOW_PARTS; + if (--showLeft < 0) return '…'; + showing++; + try { return showTerm(x, names, prec); } finally { showing--; } +} +/** + * The most parts of a term one `show` writes; past them it writes "…". A term that shares its parts is far longer + * written out: 20 nested ABS hold their argument 3^20 times. + */ +const SHOW_PARTS = 1000; +let showLeft = 0; +let showing = 0; +function showTerm(x: Term, names: Map, prec: number): string { + const p = (inner: string, level: number): string => (level < prec ? `(${inner})` : inner); + switch (x.k) { + case 'num': return x.t === 'float' && Number.isInteger(x.v) ? `${x.v}.0` : String(x.v); + case 'lit': + if (x.t === 'bool') return x.v ? 'TRUE' : 'FALSE'; + if (x.t === 'char') return `'${x.v}'`; + return JSON.stringify(x.v); + case 'sym': return names.get(key(x)) || x.name; + case 'bv': return x.name; + case 'op': { + if (x.op === 'neg') return p(`-${show(x.args[0], names, 8)}`, 8); + const lv = { '+': 6, '-': 6, '*': 7, '/': 7, DIV: 7, MOD: 7, '^': 9, '&': 5 }[x.op] || 6; + if (x.op === '+' && x.args[1].k === 'num' && x.args[1].v < 0) return p(`${show(x.args[0], names, lv)} - ${-x.args[1].v}`, lv); + if (x.op === '*' && x.args[0].k === 'num' && x.args[0].t === 'float' && x.args[0].v !== 0 && Number.isInteger(1 / x.args[0].v)) return p(`${show(x.args[1], names, 7)} / ${1 / x.args[0].v}`, 7); + return p(`${show(x.args[0], names, lv)} ${x.op} ${show(x.args[1], names, lv + (x.op === '^' ? 0 : 1))}`, lv); + } + case 'and': return p(x.args.map((a) => show(a, names, 3)).join(' AND '), 2); + case 'or': return p(x.args.map((a) => show(a, names, 2)).join(' OR '), 1); + case 'not': { + if (x.a.k === 'app' && x.a.f === 'IN') return p(`NOT (${show(x.a, names, 0)})`, 3); + return p(`NOT ${show(x.a, names, 4)}`, 3); + } + case 'cmp': return p(`${show(x.a, names, 5)} ${x.op} ${show(x.b, names, 5)}`, 4); + case 'ite': return `(IF ${show(x.c, names)} THEN ${show(x.a, names)} ELSE ${show(x.b, names)})`; + case 'app': return showApp(x, names, p); + case 'q': { + const dom = `[${show(x.lo, names)}, ${show(x.hi, names)})`; + switch (x.q) { + case 'SIZE': return `LENGTH({ ${x.v} IN ${dom} : ${show(x.body, names)} })`; + case 'SORTED': return `SORTED({ ${x.v} IN ${dom} : ${show(x.body, names)} })`; + case 'SETOF': return `{ ${x.v} IN ${dom} : ${show(x.body, names)} }`; + case 'SUM': return p(`SUM ${x.v} IN ${dom} : ${show(x.body, names)}`, 0); + default: return p(`${x.q} ${x.v} IN ${dom} : ${show(x.body, names)}`, 0); + } + } + case 'qc': return p(`${x.q} ${x.v} IN ${show(x.dom, names)} : ${show(x.body, names)}`, 0); + default: return '?'; + } +} + +function showApp(x: App, names: Map, p: (inner: string, level: number) => string): string { + if (x.f === 'TOFLOAT32') return `FLOAT32(${show(x.args[0], names)})`; + if (/^(FLOAT|F32)[-+*/^]$/.test(x.f)) { + const op = x.f.slice(-1); + const lv = ({ '+': 6, '-': 6, '*': 7, '/': 7, '^': 9 } as Record)[op] ?? 7; + return p(`${show(x.args[0], names, lv)} ${op} ${show(x.args[1], names, lv + (op === '^' ? 0 : 1))}`, lv); + } + const a = x.args.map((y) => show(y, names)); + switch (x.f) { + case 'AT': return `${show(x.args[0], names, 10)}[${a[1]}]`; + case 'AT2': return `${show(x.args[0], names, 10)}[${a[1]}, ${a[2]}]`; + case 'LENGTH': return `LENGTH(${a[0]})`; + case 'IN': return p(`${show(x.args[0], names, 5)} IN ${show(x.args[1], names, 5)}`, 4); + case 'LIST': return `[${a.join(', ')}]`; + case 'SET': return `{${a.join(', ')}}`; + case 'EMPTY': return '{}'; + case 'APPENDED': return `${show(x.args[0], names, 10)} with ${a[1]} appended`; + case 'STORE': return `${show(x.args[0], names, 10)} with [${a[1]}] set to ${a[2]}`; + case 'STORE2': return `${show(x.args[0], names, 10)} with [${a[1]}, ${a[2]}] set to ${a[3]}`; + case 'MAPPUT': return `${show(x.args[0], names, 10)} with [${a[1]}] set to ${a[2]}`; + case 'SETADD': return `${show(x.args[0], names, 10)} with ${a[1]} added`; + case 'SETDEL': case 'MAPDEL': return `${show(x.args[0], names, 10)} without ${a[1]}`; + default: break; + } + if (x.f.startsWith('FIELD.')) return `${show(x.args[0], names, 10)}.${x.f.slice(6)}`; + if (x.f.startsWith('FSTORE.')) return `${show(x.args[0], names, 10)} with .${x.f.slice(7)} set to ${a[1]}`; + switch (x.f) { + case 'NEWARRAY': return `a new ARRAY of length ${a[1]}`; + case 'SORTLIST': return `${show(x.args[0], names, 10)} sorted`; + default: return `${x.f.replace(/^FN:/, '')}(${a.join(', ')})`; + } +} + +/** How deep a term is (a leaf is 1). Kept for each part, so a term built on others costs only its new parts. */ +const DEPTH = new WeakMap(); +export function depthOf(x: Term): number { + const hit = DEPTH.get(x); + if (hit !== undefined) return hit; + let d = 0; + const r = x as unknown as Record; + for (const k of ['a', 'b', 'c', 'lo', 'hi', 'body', 'dom']) if (r[k] && typeof r[k] === 'object') d = Math.max(d, depthOf(r[k] as Term)); + if ('args' in x) for (const y of x.args) d = Math.max(d, depthOf(y)); + DEPTH.set(x, d + 1); + return d + 1; +} +/** Every symbol a term mentions. Each shared part is looked at once, however often the term holds it. */ +export function symsOf(x: Term | null | undefined, out: Map = new Map(), seen: Set = new Set()): Map { + if (!x || typeof x !== 'object' || seen.has(x)) return out; + seen.add(x); + if (x.k === 'sym') { out.set(key(x), x); return out; } + const r = x as unknown as Record; + for (const k of ['a', 'b', 'c', 'lo', 'hi', 'body', 'dom']) if (r[k] && typeof r[k] === 'object') symsOf(r[k] as Term, out, seen); + if ('args' in x) for (const y of x.args) symsOf(y, out, seen); + return out; +} + +export function describeSort(t: Sort): string { return typeName({ k: { int: 'INTEGER', float: 'FLOAT', char: 'CHAR', string: 'STRING', bool: 'BOOL' }[t] || 'ANY' }); } diff --git a/src/engines/pseudo/numeric.ts b/src/engines/pseudo/numeric.ts new file mode 100644 index 0000000..89c0cbc --- /dev/null +++ b/src/engines/pseudo/numeric.ts @@ -0,0 +1,198 @@ +/** + * pseudo/numeric.ts + * + * The exact arithmetic behind the numerical built-ins: what a FLOAT holds digit for digit, the next value + * after it, a fused multiply-add rounded once, and whole numbers bit by bit. Every function here is exact + * (or correctly rounded) for the values a run can hold, so a lesson about rounding is never itself rounded. + * + * Nothing here depends on the DOM. + */ + +const view = new DataView(new ArrayBuffer(8)); + +/** A finite double as M * 2^E with M a BigInt (signed). 0 is { m: 0n, e: 0 }. */ +export function decompose(x: number): { m: bigint; e: number } { + if (x === 0) return { m: 0n, e: 0 }; + view.setFloat64(0, x); + const hi = view.getUint32(0); + const lo = view.getUint32(4); + const exp = (hi >>> 20) & 0x7ff; + let mant = (BigInt(hi & 0xfffff) << 32n) | BigInt(lo); + let e: number; + if (exp === 0) e = -1074; + else { mant |= 1n << 52n; e = exp - 1075; } + return { m: x < 0 ? -mant : mant, e }; +} + +const bitLength = (a: bigint): number => (a === 0n ? 0 : a.toString(2).length); + +/** x times 2^e, for an x and e whose product is representable: never rounds twice. */ +function scale(x: number, e: number): number { + while (e > 1000) { x *= 2 ** 1000; e -= 1000; } + while (e < -1000) { x *= 2 ** -1000; e += 1000; } + return x * 2 ** e; +} + +/** + * The value M * 2^E rounded to the nearest binary floating-point number with `p` bits of precision and smallest + * exponent `emin` (ties to even): p = 53, emin = -1074 for FLOAT64; p = 24, emin = -149 for FLOAT32. + */ +export function roundDyadic(m: bigint, e: number, p = 53, emin = -1074): number { + if (m === 0n) return 0; + const neg = m < 0n; + let a = neg ? -m : m; + const lead = bitLength(a) - 1 + e; + const q = Math.max(lead - (p - 1), emin); + const shift = q - e; + if (shift > 0) { + const s = BigInt(shift); + let quot = a >> s; + const rem = a - (quot << s); + const half = 1n << (s - 1n); + if (rem > half || (rem === half && (quot & 1n) === 1n)) quot += 1n; + a = quot; + e = q; + } + const r = scale(Number(a), e); + return neg ? -r : r; +} + +/** A * B + C with one rounding at the end, in FLOAT64 or (f32) FLOAT32 precision. */ +export function fma(a: number, b: number, c: number, f32 = false): number { + // an infinite C with finite A and B: the exact product is finite however large, so the answer is C. A * B + C in + // FLOAT arithmetic would overflow the product first and give infinity minus infinity (FMA(1e308, 10.0, -INF())). + if (Number.isFinite(a) && Number.isFinite(b) && (c === Infinity || c === -Infinity)) return c; + if (![a, b, c].every(Number.isFinite)) { const r = a * b + c; return f32 ? Math.fround(r) : r; } + const x = decompose(a); const y = decompose(b); const z = decompose(c); + const pm = x.m * y.m; const pe = x.e + y.e; + if (pm === 0n && z.m === 0n) return f32 ? Math.fround(a * b + c) : a * b + c; // keeps the sign of zero + let m: bigint; let e: number; + if (pm === 0n) { m = z.m; e = z.e; } else if (z.m === 0n) { m = pm; e = pe; } else { + e = Math.min(pe, z.e); + m = (pm << BigInt(pe - e)) + (z.m << BigInt(z.e - e)); + } + if (m === 0n) return 0; + const r = f32 ? roundDyadic(m, e, 24, -149) : roundDyadic(m, e, 53, -1074); + return f32 ? Math.fround(r) : r; +} + +/** + * The FLOAT32 nearest a decimal literal, rounded once from the text: "1.0000000596046448" lies just above the + * middle of two singles, and going through the nearest FLOAT64 first lands exactly on the middle and rounds the + * wrong way. `text` is digits, an optional point and an optional exponent, as the lexer reads a FLOAT. + */ +export function float32OfDecimal(text: string): number { + const m = /^(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(text); + if (!m) return Math.fround(Number(text)); + const frac = m[2] || ''; + const exp = Number(m[3] || 0) - frac.length; + let num = BigInt(m[1] + frac); + if (num === 0n) return 0; + // far outside what a single holds: no need to build the power of ten + const mag = (m[1] + frac).replace(/^0+/, '').length + exp; + if (mag > 60) return Infinity; + if (mag < -60) return 0; + let den = 1n; + if (exp >= 0) num *= 10n ** BigInt(exp); else den = 10n ** BigInt(-exp); + // the quotient to 24 bits or the smallest single's exponent, ties to even, with the remainder as a sticky bit + const lead = bitLength(num) - bitLength(den); + const q = Math.max(lead - 25, -151); // two bits below the smallest single's exponent, so the half is still seen + if (q >= 0) den <<= BigInt(q); else num <<= BigInt(-q); + let quot = num / den; + if (quot * den !== num) quot = (quot << 1n) | 1n; else quot <<= 1n; + return Math.fround(roundDyadic(quot, q - 1, 24, -149)); +} + +/** The exact decimal value of a finite FLOAT: 0.1 is 0.1000000000000000055511151231257827021181583404541015625. */ +export function exactDecimal(x: number): string { + if (Number.isNaN(x)) return 'not a number'; + if (!Number.isFinite(x)) return x > 0 ? 'infinity' : '-infinity'; + if (x === 0) return Object.is(x, -0) ? '-0.0' : '0.0'; + const { m, e } = decompose(x); + const neg = m < 0n; + const a = neg ? -m : m; + let s: string; + if (e >= 0) s = `${(a << BigInt(e)).toString()}.0`; + else { + const digits = (a * 5n ** BigInt(-e)).toString().padStart(-e + 1, '0'); + const cut = digits.length + e; + s = `${digits.slice(0, cut)}.${digits.slice(cut)}`.replace(/0+$/, ''); + if (s.endsWith('.')) s += '0'; + } + return neg ? `-${s}` : s; +} + +const f32buf = new Float32Array(1); +const u32buf = new Uint32Array(f32buf.buffer); + +/** The next representable value after x toward y, in FLOAT64 or (f32) FLOAT32. */ +export function nextAfter(x: number, y: number, f32 = false): number { + if (Number.isNaN(x) || Number.isNaN(y)) return NaN; + if (x === y) return y; + if (f32) { + if (x === 0) { u32buf[0] = 1; return y > 0 ? f32buf[0] : -f32buf[0]; } + f32buf[0] = x; + if ((y > x) === (x > 0)) u32buf[0] += 1; else u32buf[0] -= 1; + return f32buf[0]; + } + if (x === 0) return y > 0 ? Number.MIN_VALUE : -Number.MIN_VALUE; + view.setFloat64(0, x); + let bits = view.getBigUint64(0); + if ((y > x) === (x > 0)) bits += 1n; else bits -= 1n; + view.setBigUint64(0, bits); + return view.getFloat64(0); +} + +/** One unit in the last place of x: the gap from |x| to the next value away from zero (or below it, at the top). */ +export function ulp(x: number, f32 = false): number { + // an infinity's gap is infinite (Python's math.ulp and Mojo's ulp say the same); NaN's is NaN + if (Number.isNaN(x)) return NaN; + if (!Number.isFinite(x)) return Infinity; + const a = Math.abs(x); + const up = nextAfter(a, Infinity, f32); + if (Number.isFinite(up)) return up - a; + return a - nextAfter(a, 0, f32); +} + +/** The sign of y on the size of x, as IEEE 754 copySign (the sign of zero counts). */ +export function copySign(x: number, y: number): number { + const neg = y < 0 || Object.is(y, -0); + return neg ? -Math.abs(x) : Math.abs(x); +} + +export function gcd(a: number, b: number): number { + a = Math.abs(a); b = Math.abs(b); + while (b) [a, b] = [b, a % b]; + return a; +} + +/** Bitwise operations on whole numbers held exactly, two's complement for negative values. */ +export function bitOp(op: 'AND' | 'OR' | 'XOR' | 'SHL' | 'SHR', a: number, b: number): number { + const x = BigInt(a); const y = BigInt(b); + switch (op) { + case 'AND': return Number(x & y); + case 'OR': return Number(x | y); + case 'XOR': return Number(x ^ y); + case 'SHL': return Number(x << y); + case 'SHR': return Number(x >> y); + default: throw new Error(op); + } +} + +/** A whole number brought into an n-bit type by wrapping around (n up to 32). */ +export function wrap(a: number, bits: number, signed: boolean | undefined): number { + const m = 2 ** bits; + let r = ((a % m) + m) % m; + if (signed && r >= m / 2) r -= m; + return r; +} + +/** + * X to the power Y as IEEE 754's pow says, where JavaScript's ** differs: 1 to any power, NaN included, is 1, and + * so is -1 to an infinite power (** gives NaN for both). Everything else is **. + */ +export function ieeePow(x: number, y: number): number { + if (x === 1 || y === 0) return 1; + if (x === -1 && (y === Infinity || y === -Infinity)) return 1; + return x ** y; +} diff --git a/src/engines/pseudo/parse.ts b/src/engines/pseudo/parse.ts new file mode 100644 index 0000000..679b505 --- /dev/null +++ b/src/engines/pseudo/parse.ts @@ -0,0 +1,1518 @@ +/** + * pseudo/parse.ts + * + * Text to syntax tree. The parser keeps going after a mistake, so one pass + * reports every line that needs attention rather than stopping at the + * first. Every message names the rule that was broken, in the guide's own + * words where the guide has words for it. + * + * parse(source) -> { ast, diags, tokens, lines, comments } + * + * Lines are numbered from 1, as a student counts them. Every node carries + * `line` and `col`, and every block carries `endLine`, the line of its + * closing keyword. + */ + +import { KEYWORDS, FOREIGN_WORDS, CLOSERS, BUILTINS, SIZED_NAMES } from './lang.js'; +import type { + Diag, DiagExtra, Severity, Loc, Token, TextToken, TokenKind, LineInfo, Comment, Lexed, Parsed, TypeExpr, ScalarName, + Expr, NumExpr, Stmt, StmtKind, ContractStmt, IfStmt, IfBranch, CaseStmt, CaseArm, CaseLabel, ForStmt, ForEachStmt, + WhileStmt, RepeatStmt, SubStmt, ParamDecl, ParamMode, DeclName, FieldDecl, ProgramNode, TypeRefExpr, QuantExpr, LooseType, +} from './ast.js'; + +/** The statement node of a kind (or of each of several kinds). */ +type NodeOf = S extends { kind: infer SK } ? ([K] extends [SK] ? S : never) : never; +type StmtOf = K extends StmtKind ? NodeOf : never; +/** A name, or a template's blank standing where a name goes, as `nameTok` reads it. */ +interface NameTok { v: string; line: number; col: number; end: number; blank?: boolean } +/** What `block` read: the statements, and the token that stopped it (null at the end of the text or an outer closer). */ +interface BlockRead { body: Stmt[]; stop: Token | null } + +/** Keywords that name a type, so they can stand as an argument: MAX_VALUE(INT8), WRAP(X, UINT8). */ +const TYPE_WORDS = new Set(['INTEGER', 'FLOAT', ...SIZED_NAMES]); + +/* ------------------------------------------------------------------ */ +/* Diagnostics */ +/* ------------------------------------------------------------------ */ + +/** + * One finding about a program. + * severity 'error' (the program cannot run), 'warning' (it runs, and is + * probably wrong), or 'note' (a habit the guide recommends) + * code a stable identifier, such as 'missing-closer' + * guide an anchor in the pseudocode guide that explains the rule + * fix { line, col, endCol, text }: a mechanical repair, when there is one + */ +export function diag(severity: Severity, code: string, line: number, message: string, extra: DiagExtra = {}): Diag { + return { severity, code, line, col: extra.col ?? 1, message, ...extra }; +} + +/* ------------------------------------------------------------------ */ +/* Lexer */ +/* ------------------------------------------------------------------ */ + +const OPS = [ + '...', '<-', '==', '!=', '<=', '>=', '<>', '&&', '||', ':=', '+=', '-=', '++', '->', '=>', + '<', '>', '+', '-', '*', '/', '^', '&', '(', ')', '[', ']', '{', '}', ',', ':', '.', '=', '!', '%', +]; +const UNICODE_OPS: Record = { '←': '<-', '≤': '<=', '≥': '>=', '≠': '!=' }; + +/** Operators other notations use, and what this one writes instead. */ +const WRONG_OPS: Record = { + ':=': ['<-', 'Assignment is written <-, with the arrow pointing at the name that receives the value.'], + '<>': ['!=', 'Not-equal is written != in this notation.'], + '&&': ['AND', 'Write AND to join two conditions.'], + '||': ['OR', 'Write OR to join two conditions.'], + '!': ['NOT', 'Write NOT in front of a condition to negate it.'], + '%': ['MOD', 'The remainder is written MOD, as in I MOD 3.'], + '+=': [null, 'This notation has no +=. Write the whole assignment: Total <- Total + 1.'], + '-=': [null, 'This notation has no -=. Write the whole assignment: Total <- Total - 1.'], + '++': [null, 'This notation has no ++. Write the whole assignment: I <- I + 1.'], + '=>': [null, 'This notation has no =>. Compare with >=, or write the step out in words in a comment.'], +}; + +const OPERAND_END = new Set(['name', 'int', 'float', 'str', 'char', 'blank']); + +/** Names to suggest when a keyword was wanted as a variable. */ +const RENAME_IDEAS: Record = { + NEXT: 'Neighbor, Following, or NextNode', STEP: 'Stride, Generation, or StepSize', SET: 'Chosen, Members, or Collection', TO: 'Target or Upper', OF: 'Owner', IN: 'Inside or Within', + CALL: 'Visit', TYPE: 'Kind', FOR: 'Purpose', IF: 'Condition', CASE: 'Instance', INPUT: 'Given or Entry', OUTPUT: 'Shown or Report', RETURN: 'Answer or Yield', RETURNS: 'Answer', + AND: 'Both', OR: 'Either', NOT: 'Negated', DIV: 'Quotient', MOD: 'Remainder', WHILE: 'Waiting', REPEAT: 'Repeats', UNTIL: 'Deadline', BREAK: 'Pause', CONTINUE: 'Carry', + TRUE: 'Flag', FALSE: 'Flag', CONSTANT: 'Fixed', DECLARE: 'Declared', LIST: 'Items', ARRAY: 'Grid', STRING: 'Text', CHAR: 'Letter', INTEGER: 'Whole', FLOAT: 'Measure', BOOL: 'Flag', BOOLEAN: 'Flag', + HASHMAP: 'Table', FUNCTION: 'Operation', PROCEDURE: 'Routine', THEN: 'After', ELSE: 'Other', BYREF: 'Shared', BYVAL: 'Copied', ASSERT: 'Claim', REQUIRE: 'Needed', ENSURE: 'Promised', INVARIANT: 'Holds', DECREASES: 'Shrinking', FORALL: 'Every', EXISTS: 'Some', +}; + +/** + * Whether a lower- or mixed-case keyword is being used as a name: it sits where only a name can + * go (after DECLARE, FOR, EACH, NEXT, BYREF, CONSTANT, a dot, or in a parameter list), or it is + * followed by what follows a name (<-, a type colon, a field dot, an index bracket), or the same + * spelling was already used as a name earlier in the program. + */ +function usedAsName(w: string, up: string, prev: Token | null, rest: string, lineText: string, seen: Set): boolean { + if (seen.has(w)) return true; + if (prev && (prev.t === 'kw' || prev.t === 'name') && ['DECLARE', 'FOR', 'EACH', 'NEXT', 'BYREF', 'BYVAL', 'CONSTANT'].includes(String(prev.v).toUpperCase())) return true; + if (prev && prev.t === 'op' && prev.v === '.') return true; + const head = /^\s*(DECLARE|FUNCTION|PROCEDURE)\b/i.test(lineText); + if (prev && prev.t === 'op' && (prev.v === ',' || prev.v === '(') && head) return true; + if (/^\s*(<-|←|:(?!=)|\.[A-Za-z]|\[)/.test(rest)) return true; + return false; +} + +export function lex(source: unknown): Lexed { + const src = String(source ?? '').replace(/\r\n?/g, '\n'); + const rawLines = src.split('\n'); + const tokens: Token[] = []; + const diags: Diag[] = []; + const comments: Comment[] = []; + const lines: LineInfo[] = rawLines.map((text, i) => { + const m = /^[ \t]*/.exec(text)![0]; + const indent = m.replace(/\t/g, ' ').length; + return { n: i + 1, text, indent, blank: !text.trim().length, onlyComment: /^\s*\/\//.test(text) }; + }); + + let prev = null as Token | null; // last significant token on the current line (set by `push`, which TypeScript does not follow) + const reservedSeen = new Set(); // keywords this program uses as names, reported once each + for (let li = 0; li < rawLines.length; li++) { + const text = rawLines[li]; + const line = li + 1; + let i = 0; + prev = null as Token | null; + const push = (t: TokenKind, v: string | number, col: number, end: number, extra: { text?: string; wrote?: string } = {}): Token => { + const tok = { t, v, line, col: col + 1, end: end + 1, raw: text.slice(col, end), ...extra } as Token; + tokens.push(tok); + prev = tok; + return tok; + }; + while (i < text.length) { + const c = text[i]; + if (c === ' ' || c === '\t') { i++; continue; } + + // comment + if (c === '/' && text[i + 1] === '/') { + const body = text.slice(i + 2); + comments.push({ line, col: i + 1, text: body.replace(/^ /, ''), standalone: !prev }); + if (prev && OPERAND_END.has(prev.t) && /^\s*\d/.test(body)) { + diags.push(diag('warning', 'comment-as-div', line, '// starts a comment, so everything after it on this line is ignored. Integer division is written DIV.', { col: i + 1, guide: 'arithmetic' })); + } + break; + } + + // blank in a template, such as + if (c === '<' && !(prev && (OPERAND_END.has(prev.t) || (prev.t === 'op' && (prev.v === ')' || prev.v === ']')) || (prev.t === 'kw' && (prev.v === 'TRUE' || prev.v === 'FALSE'))))) { + const m = /^<([A-Za-z][A-Za-z0-9 _'.-]*)>/.exec(text.slice(i)); + if (m) { push('blank', m[1].trim(), i, i + m[0].length); i += m[0].length; continue; } + } + + // numbers: 255, 2.5, 6.022e23, 1_000_000, 0xFF, 0b1010 + if (/[0-9]/.test(c)) { + const based = /^0([xX][0-9A-Fa-f_]+|[bB][01_]+)/.exec(text.slice(i)); + if (based) { + const s = based[0]; + if (/_$|__|^0[xXbB]_/.test(s)) diags.push(diag('error', 'bad-number', line, `An underscore goes between two digits, as in 0xFF_FF; ${s} has one elsewhere.`, { col: i + 1 })); + if (/[A-Za-z0-9_]/.test(text[i + s.length] || '')) diags.push(diag('error', 'bad-number', line, `${/^[0-9A-Za-z_]+/.exec(text.slice(i))![0]} is not a number: ${/[xX]/.test(s[1]) ? 'hexadecimal digits are 0 to 9 and A to F' : 'binary digits are 0 and 1'}.`, { col: i + 1 })); + push('int', Number(s.replace(/_/g, '')), i, i + s.length); + i += s.length; + continue; + } + const m = /^[0-9][0-9_]*(\.[0-9][0-9_]*)?([eE][+-]?[0-9]+)?/.exec(text.slice(i))!; + let s = m[0]; + if (/_$|__|_\.|\._/.test(s)) diags.push(diag('error', 'bad-number', line, `An underscore goes between two digits, as in 1_000_000; ${s} has one elsewhere.`, { col: i + 1 })); + const isFloat = Boolean(m[1] || m[2]); + if (text[i + s.length] === '.' && /[0-9]/.test(text[i + s.length + 1] || '') === false && !m[1]) { + diags.push(diag('error', 'bad-number', line, `Write a FLOAT with a digit on each side of the point, as in ${s}.0.`, { col: i + 1 })); + } + if (/[A-Za-z_]/.test(text[i + s.length] || '')) { + diags.push(diag('error', 'bad-name', line, `A name must start with a letter, and "${/^[0-9A-Za-z_]+/.exec(text.slice(i))![0]}" starts with a digit.`, { col: i + 1, guide: 'names' })); + } + push(isFloat ? 'float' : 'int', Number(s.replace(/_/g, '')), i, i + s.length, isFloat ? { text: s.replace(/_/g, '') } : {}); + i += s.length; + continue; + } + if (c === '.' && /[0-9]/.test(text[i + 1] || '')) { + const m = /^\.[0-9]+/.exec(text.slice(i))!; + diags.push(diag('error', 'bad-number', line, `Write a FLOAT with a digit before the point, as in 0${m[0]}.`, { col: i + 1, fix: { line, col: i + 1, endCol: i + 1, text: '0' } })); + push('float', Number(`0${m[0]}`), i, i + m[0].length); + i += m[0].length; + continue; + } + + // strings and characters + if (c === '"' || c === "'") { + let j = i + 1; + let value = ''; + let closed = false; + while (j < text.length) { + const d = text[j]; + if (d === '\\' && j + 1 < text.length) { + const e = text[j + 1]; + value += e === 'n' ? '\n' : e === 't' ? '\t' : e; + j += 2; + continue; + } + if (d === c) { closed = true; j++; break; } + value += d; + j++; + } + if (!closed) { + diags.push(diag('error', 'unclosed-text', line, c === '"' + ? 'This STRING has no closing double quote. Text in double quotes has to open and close on the same line.' + : 'This CHAR has no closing single quote. A CHAR is one character between single quotes, such as \'A\'.', { col: i + 1 })); + } + if (c === "'") { + const n = Array.from(value).length; + if (closed && n !== 1) { + diags.push(diag('error', 'char-length', line, n === 0 + ? 'A CHAR holds exactly one character, and \'\' holds none. An empty piece of text is the STRING "".' + : `A CHAR holds exactly one character, and '${value}' holds ${n}. Text of any length goes in double quotes: "${value}".`, + { col: i + 1, guide: 'the-types', fix: { line, col: i + 1, endCol: j + 1, text: `"${value}"` } })); + push('str', value, i, j); + } else push('char', value, i, j); + } else push('str', value, i, j); + i = j; + continue; + } + + // words + // the same letters as the pattern below: an arrow such as → (U+2190 to U+21FF) is not one, and is reported as a + // character with no meaning instead of reaching a pattern that cannot match it + if (/[A-Za-z_À-↏∀-￿]/.test(c) && !UNICODE_OPS[c]) { + const m = /^[A-Za-z_À-↏∀-￿][A-Za-z0-9_À-↏∀-￿]*/.exec(text.slice(i))!; + const w = m[0]; + const up = w.toUpperCase(); + if (w.startsWith('_')) diags.push(diag('error', 'bad-name', line, `Names start with a letter, and ${w} starts with an underscore.`, { col: i + 1, guide: 'names' })); + if (KEYWORDS.has(up) && w !== up && usedAsName(w, up, prev, text.slice(i + w.length), text, reservedSeen)) { + // A keyword written as a name (Next, Step, Set): one message that says so, then + // read it as a name so the rest of the line parses and nothing else cascades. + if (!reservedSeen.has(w)) { + reservedSeen.add(w); + diags.push(diag('error', 'reserved-name', line, `${w} cannot be a name: ${up} is a keyword, and names ignore case. Try ${RENAME_IDEAS[up] || `${w}Value`}.`, { col: i + 1, guide: 'names' })); + } + push('name', w, i, i + w.length); + } else if (KEYWORDS.has(up)) { + if (w !== up) { + diags.push(diag('error', 'keyword-case', line, `Keywords are uppercase, always: write ${up}, not ${w}.`, { col: i + 1, guide: 'how-to-read-this-guide', fix: { line, col: i + 1, endCol: i + 1 + w.length, text: up } })); + } + push('kw', up === 'BOOLEAN' ? 'BOOL' : up, i, i + w.length); + } else { + push('name', w, i, i + w.length); + } + i += w.length; + continue; + } + + // operators + let op = UNICODE_OPS[c] ? c : OPS.find((o) => text.startsWith(o, i)); + if (op) { + const v = UNICODE_OPS[op] || op; + if (WRONG_OPS[v] && !(v === '->')) { + let [instead, why] = WRONG_OPS[v]; + // name the variable the line is about, when there is one: I += 1 + // should suggest I <- I + 1, not somebody else's variable + const lhs = /([A-Za-z_][A-Za-z0-9_]*(?:\[[^\]]*\])?)\s*$/.exec(text.slice(0, i)); + const rest = text.slice(i + op.length).replace(/\/\/.*$/, '').trim(); + if (lhs && (v === '+=' || v === '-=')) why = `This notation has no ${v}. Write the whole assignment: ${lhs[1]} <- ${lhs[1]} ${v[0]} ${rest || '1'}.`; + if (lhs && v === '++') why = `This notation has no ++. Write the whole assignment: ${lhs[1]} <- ${lhs[1]} + 1.`; + diags.push(diag('error', 'wrong-operator', line, why, { + col: i + 1, guide: v === ':=' ? 'assignment' : 'operators', + fix: instead ? { line, col: i + 1, endCol: i + 1 + op.length, text: instead } : undefined, + })); + } + push('op', instead(v), i, i + op.length, { wrote: v }); + i += op.length; + continue; + } + + diags.push(diag('error', 'bad-character', line, `The character ${JSON.stringify(c)} has no meaning in this notation.`, { col: i + 1 })); + i++; + } + tokens.push({ t: 'nl', v: '\n', line, col: text.length + 1, end: text.length + 1 }); + } + tokens.push({ t: 'eof', v: '', line: rawLines.length, col: 1, end: 1 }); + return { tokens, diags, comments, lines }; +} + +/** The operator the parser should act on when a student wrote another notation's. */ +function instead(v: string): string { + const map: Record = { ':=': '<-', '<>': '!=', '&&': 'AND', '||': 'OR', '!': 'NOT', '%': 'MOD' }; + return map[v] || v; +} + +/* ------------------------------------------------------------------ */ +/* Parser */ +/* ------------------------------------------------------------------ */ + +const COMPARE = new Set(['==', '!=', '<', '<=', '>', '>=']); +const BLOCK_ENDS = new Set(['ENDIF', 'ELSE', 'ELSEIF', 'ENDCASE', 'OTHERWISE', 'NEXT', 'ENDWHILE', 'UNTIL', 'ENDFUNCTION', 'ENDPROCEDURE', 'ENDTYPE']); +const OPENER_WORD: Record = { If: 'IF', Case: 'CASE', For: 'FOR', ForEach: 'FOR EACH', While: 'WHILE', Repeat: 'REPEAT', Function: 'FUNCTION', Procedure: 'PROCEDURE', TypeDef: 'TYPE' }; + +/** How deep blocks and brackets may nest, and how many values one chain of an operator may join. */ +const MAX_NESTING = 200; +const MAX_CHAIN = 400; +/** + * How tall one expression's tree may be. The two limits above are counted apart, and a chain leans left one level + * per value, so brackets that each start a long chain multiply them: three levels of 399 values were a tree about + * 1,200 levels tall, inside both limits, and the checker ended in the engine's own stack error. The tree is held + * to what one chain alone may make it, so no reader after this one walks a taller tree than it already had to. + */ +const MAX_TREE = MAX_CHAIN; + +class Parser { + declare toks: Token[]; + declare lines: LineInfo[]; + declare diags: Diag[]; + declare p: number; + declare open: Array<{ word: string; line: number; stops: string[] }>; + declare depth: number; + declare tooDeep: boolean; + /** Set once a line nested too deeply has been reported. */ + declare deepSaid?: boolean; + /** The height of every expression node measured so far (heightOf). */ + declare heights: WeakMap; + constructor(lexed: Lexed) { + this.toks = lexed.tokens; + this.lines = lexed.lines; + this.diags = lexed.diags; + this.p = 0; + this.open = []; // stack of { word, line, stops } + this.depth = 0; // blocks and brackets open around the token being read + this.tooDeep = false; // set once the limit is passed, until the statement ends + this.heights = new WeakMap(); + } + + /* ---------- token helpers ---------- */ + get tok(): Token { return this.toks[this.p]; } + peek(k = 1): Token { return this.toks[Math.min(this.p + k, this.toks.length - 1)]; } + next(): Token { const t = this.toks[this.p]; if (t.t !== 'eof') this.p++; return t; } + isKw(v: string, t: Token = this.tok): boolean { return t.t === 'kw' && t.v === v; } + isOp(v: string, t: Token = this.tok): boolean { return t.t === 'op' && t.v === v; } + isWord(v: string, t: Token = this.tok): boolean { return t.t === 'name' && t.v.toUpperCase() === v; } + atLineEnd(): boolean { return this.tok.t === 'nl' || this.tok.t === 'eof'; } + + error(code: string, tok: Loc, message: string, extra: DiagExtra = {}): void { + if (this.tooDeep) return; // one message for a line nested too deeply, not one per bracket left open + this.diags.push(diag('error', code, tok.line, message, { col: tok.col, ...extra })); + } + warn(code: string, tok: Loc, message: string, extra: DiagExtra = {}): void { + this.diags.push(diag('warning', code, tok.line, message, { col: tok.col, ...extra })); + } + + expectKw(v: string, why?: string): Token | null { + if (this.isKw(v)) return this.next(); + this.error('expected', this.tok, why || `Expected ${v} here${this.describeFound()}.`); + return null; + } + expectOp(v: string, why?: string): Token | null { + if (this.isOp(v)) return this.next(); + this.error('expected', this.tok, why || `Expected ${v} here${this.describeFound()}.`); + return null; + } + describeFound(): string { + const t = this.tok; + if (t.t === 'nl' || t.t === 'eof') return ', and the line ended'; + return `, and found ${t.t === 'str' ? `"${t.v}"` : t.t === 'char' ? `'${t.v}'` : t.raw || t.v}`; + } + + /** Skip whatever is left of a line that could not be parsed. */ + skipLine(): void { while (!this.atLineEnd()) this.next(); } + endLine(what: string): void { + if (!this.atLineEnd()) { + const t = this.tok; + if (this.isOp('=', t) || this.isOp('<-', t)) this.error('extra', t, `Unexpected ${t.v} after ${what}. One line holds one statement.`); + else this.error('extra', t, `Unexpected ${t.raw || t.v} after ${what}. One line holds one statement, and each block keyword sits on a line of its own.`); + this.skipLine(); + } + if (this.tok.t === 'nl') this.next(); + } + skipBlankLines(): void { while (this.tok.t === 'nl') this.next(); } + + loc(t: Loc): Loc { return { line: t.line, col: t.col }; } + + /** A name, or a template's blank standing where a name goes. */ + nameTok(): NameTok | null { + const t = this.tok; + if (t.t === 'name') { this.next(); return { v: t.v, line: t.line, col: t.col, end: t.end }; } + if (t.t === 'blank') { this.next(); return { v: `<${t.v}>`, line: t.line, col: t.col, end: t.end, blank: true }; } + return null; + } + node(kind: K, t: Loc, props: Omit, 'kind' | 'line' | 'col'>): StmtOf { return { kind, line: t.line, col: t.col, ...props } as StmtOf; } + + /* ---------- program ---------- */ + program(): ProgramNode { + const body: Stmt[] = []; + this.skipBlankLines(); + while (this.tok.t !== 'eof') { + const t = this.tok; + if (t.t === 'kw' && BLOCK_ENDS.has(t.v)) { + this.error('stray-closer', t, `${t.v} closes a block, and no block is open here.`, { hint: this.strayHint(t.v) }); + this.skipLine(); + this.skipBlankLines(); + continue; + } + const s = this.statement(); + if (s) body.push(s); + this.skipBlankLines(); + } + return { kind: 'Program', line: 1, col: 1, body }; + } + + strayHint(word: string): string | undefined { + const opener = Object.entries(CLOSERS).find(([, c]) => c === word)?.[0]; + return opener ? `Either the ${opener} this ${word} belongs to is missing, or an earlier ${word} already closed it.` : undefined; + } + + /** + * Statements until one of `stops` begins a line. Returns the body and the + * token that stopped it: a stop word, or null at the end of the text or at + * a closer that belongs to an enclosing block. + */ + block(stops: string[]): BlockRead { + const body: Stmt[] = []; + this.skipBlankLines(); + for (; ;) { + const t = this.tok; + if (t.t === 'eof') return { body, stop: null }; + if (t.t === 'kw' && stops.includes(t.v)) return { body, stop: t }; + if (t.t === 'kw' && BLOCK_ENDS.has(t.v)) { + if (this.open.some((o) => o.stops.includes(t.v))) return { body, stop: null }; + this.error('stray-closer', t, `${t.v} closes a block, and no block that ${t.v} can close is open here.`, { hint: this.strayHint(t.v) }); + this.skipLine(); + this.skipBlankLines(); + continue; + } + const s = this.statement(); + if (s) body.push(s); + this.skipBlankLines(); + } + } + + /** Where the indentation suggests a block's closer belongs. */ + closerHint(openLine: number, closer: string): string | undefined { + const opener = this.lines[openLine - 1]; + let last = openLine; + for (let i = openLine; i < this.lines.length; i++) { + const l = this.lines[i]; + if (l.blank || l.onlyComment) continue; + if (l.indent <= opener.indent && !(l.indent === opener.indent && /^\s*(ELSEIF|ELSE|OTHERWISE)\b/.test(l.text))) break; + last = l.n; + } + return last > openLine ? `Going by the indentation, its body ends on line ${last}, so ${closer} belongs on the line after that.` : undefined; + } + + missingCloser(word: string, openTok: Loc, closer: string, extra = ''): void { + this.diags.push(diag('error', 'missing-closer', openTok.line, `The ${word} on line ${openTok.line} is never closed: it needs ${closer}.${extra}`, { + col: openTok.col, guide: 'how-to-read-this-guide', hint: extra ? undefined : this.closerHint(openTok.line, closer), + })); + } + + withOpen(word: string, tok: Loc, stops: string[], fn: () => R): R { + this.open.push({ word, line: tok.line, stops }); + try { return fn(); } finally { this.open.pop(); } + } + + /* ---------- statements ---------- */ + statement(): Stmt | null { + this.tooDeep = false; + if (this.depth >= MAX_NESTING) { + // said once for the program: every line further in would say the same + if (!this.deepSaid) { this.deepSaid = true; this.error('too-deep', this.tok, `This line is inside more than ${MAX_NESTING} blocks, which is more than this notation reads. Move the inner part into a FUNCTION or PROCEDURE of its own.`); } + this.skipLine(); + return null; + } + this.depth++; + try { return this.statementInner(); } finally { this.depth--; } + } + + statementInner(): Stmt | null { + const t = this.tok; + if (t.t === 'kw') { + switch (t.v) { + case 'DECLARE': return this.declare(); + case 'CONSTANT': return this.constant(); + case 'DEFINE': return this.define(); + case 'TYPE': return this.typeDef(); + case 'IF': return this.ifStmt(); + case 'CASE': return this.caseStmt(); + case 'FOR': return this.forStmt(); + case 'PARALLEL': { + this.next(); + if (!this.isKw('FOR')) { this.error('expected', this.tok, 'PARALLEL is followed by FOR: PARALLEL FOR I <- 0 TO N marks a loop whose passes are independent.', { guide: 'parallel-for' }); this.skipLine(); return null; } + const s = this.forStmt(); + if (s && s.kind === 'For') s.parallel = true; + else if (s) this.error('parallel-each', s, 'PARALLEL FOR takes a counted loop, FOR I <- A TO B, so that each pass has a position of its own to write.', { guide: 'parallel-for' }); + return s; + } + case 'WHILE': return this.whileStmt(); + case 'REPEAT': return this.repeatStmt(); + case 'FUNCTION': case 'PROCEDURE': return this.subroutine(); + case 'BREAK': case 'CONTINUE': { this.next(); const s = this.node(t.v === 'BREAK' ? 'Break' : 'Continue', t, {}); this.endLine(t.v); return s; } + case 'RETURN': { + this.next(); + const value = this.atLineEnd() ? null : this.expr(); + const s = this.node('Return', t, { value }); + this.endLine('RETURN'); + return s; + } + case 'INPUT': { + this.next(); + const target = this.target(); + const s = this.node('Input', t, { target }); + this.endLine('INPUT'); + return s; + } + case 'OUTPUT': { + this.next(); + const values = []; + if (this.atLineEnd()) this.error('expected', this.tok, 'OUTPUT needs at least one value to show.'); + else { + values.push(this.expr()); + while (this.isOp(',')) { this.next(); values.push(this.expr()); } + } + const s = this.node('Output', t, { values }); + this.endLine('OUTPUT'); + return s; + } + case 'CALL': return this.callStmt(); + case 'USE': return this.useStmt(); + case 'REQUIRE': case 'ENSURE': case 'INVARIANT': case 'DECREASES': case 'ASSERT': { + this.next(); + const kind = (t.v[0] + t.v.slice(1).toLowerCase()) as ContractStmt['kind']; + if (this.atLineEnd()) { this.error('expected', this.tok, `${t.v} needs a condition after it.`); this.endLine(t.v); return null; } + const s = this.node(kind, t, { expr: this.expr() }); + this.endLine(t.v); + return s; + } + case 'THEN': this.error('misplaced', t, 'THEN belongs at the end of an IF line, after the condition.'); this.skipLine(); return null; + default: + this.error('bad-start', t, `A line cannot start with ${t.v}.${t.v === 'ELSE' || t.v === 'ENDIF' ? '' : ''}`); + this.skipLine(); + return null; + } + } + if (t.t === 'blank') { + // "" on a line of its own: a template's blank to fill + this.next(); + const s = this.node('BlankStmt', t, { text: t.v }); + this.endLine(`<${t.v}>`); + return s; + } + if (t.t === 'name') { + const up = t.v.toUpperCase(); + if (FOREIGN_WORDS[up] && (this.peek().t === 'nl' || this.peek().t === 'eof' || up === 'ENDFOR' || up === 'PRINT' || up === 'ELIF' || up === 'END')) { + this.error('foreign-word', t, FOREIGN_WORDS[up]); + this.skipLine(); + return null; + } + if (up === 'PRINT' || up === 'ELIF' || up === 'SWITCH') { this.error('foreign-word', t, FOREIGN_WORDS[up]); this.skipLine(); return null; } + return this.simpleStatement(); + } + if (t.t === 'op' && t.v === '(') { this.error('bad-start', t, 'A line cannot start with a bracket. A value on a line of its own does nothing.'); this.skipLine(); return null; } + this.error('bad-start', t, `A line cannot start with ${t.raw || t.v}.`); + this.skipLine(); + return null; + } + + /** Assignment, a built-in procedure, or a call written without CALL. */ + simpleStatement(): Stmt | null { + const t = this.tok as TextToken; // called at a name + const up = t.v.toUpperCase(); + if (this.isOp('(', this.peek())) { + if (BUILTINS[up]?.kind === 'procedure') { + this.next(); this.next(); + const args = this.args(')'); + const s = this.node('BuiltinCall', t, { name: up, args }); + this.endLine(up); + return s; + } + const call = this.postfix(this.primary()); + if (this.isOp('<-') || this.isOp('=')) return this.assignment(t, call); + const s = this.node('ExprStmt', t, { expr: call }); + this.endLine('the call'); + return s; + } + const target = this.postfix(this.primary()); + if (this.isOp('<-') || this.isOp('=')) return this.assignment(t, target); + if (this.isOp('==')) { + this.error('compare-as-statement', this.tok, 'A comparison on a line of its own computes TRUE or FALSE and throws it away. To store a value, write <-.', { guide: 'assignment' }); + this.skipLine(); + return null; + } + if (this.atLineEnd()) { + this.error('bare-value', t, `${t.v} on a line of its own does nothing. Assign to it with <-, or CALL it if it is a procedure.`); + } else if (!(this.tok.wrote && WRONG_OPS[this.tok.wrote])) { + // an operator from another language has its own message already + this.error('expected', this.tok, `Expected <- after ${t.v}${this.describeFound()}.`, { guide: 'assignment' }); + } + this.skipLine(); + return null; + } + + assignment(t: Loc, target: Expr): Stmt | null { + const opTok = this.next(); + if (opTok.v === '=') { + this.error('equals-assign', opTok, `= is not assignment in this notation. Write ${sourceOf(target)} <- ..., with the arrow pointing at the name that receives the value.`, { + guide: 'assignment', fix: { line: opTok.line, col: opTok.col, endCol: opTok.end, text: '<-' }, + }); + } + if (target.kind !== 'Name' && target.kind !== 'Index' && target.kind !== 'Field') { + this.error('bad-target', t, 'Only a name, a position such as T[I], or a field such as R.Name can receive a value.'); + } + if (this.atLineEnd()) { this.error('expected', this.tok, 'The assignment has nothing on the right of its arrow.'); this.endLine('<-'); return null; } + const value = this.expr(); + const s = this.node('Assign', t, { target, value }); + this.endLine('the assignment'); + return s; + } + + target(): Expr | null { + if (this.tok.t !== 'name') { this.error('expected', this.tok, `Expected a name here${this.describeFound()}.`); this.skipLine(); return null; } + return this.postfix(this.primary(), { noCall: true }); + } + + callStmt(): Stmt | null { + const t = this.next(); + if (this.tok.t !== 'name') { this.error('expected', this.tok, 'CALL needs the name of a procedure.'); this.skipLine(); return null; } + const nameTok = this.next() as TextToken; + let args: Expr[] = []; + if (this.isOp('(')) { this.next(); args = this.args(')'); } + const s = this.node('Call', t, { name: nameTok.v, nameCol: nameTok.col, args }); + this.endLine(`CALL ${nameTok.v}`); + return s; + } + + /** USE stats, USE stats.Mean, or USE IEEE: routines from the library, or IEEE 754 arithmetic. */ + useStmt(): Stmt | null { + const t = this.next(); + if (this.tok.t !== 'name') { + this.error('expected', this.tok, 'USE names a module of the library, as in USE stats, or one routine of it, as in USE stats.Mean. USE IEEE asks for IEEE 754 arithmetic.', { guide: 'libraries' }); + this.skipLine(); this.endLine('USE'); return null; + } + const m = this.next() as TextToken; + let routine: string | null = null; + let routineCol: number | null = null; + if (this.isOp('.')) { + this.next(); + if (this.tok.t !== 'name') { this.error('expected', this.tok, `After USE ${m.v}. comes the name of one routine, as in USE ${m.v}.Mean.`, { guide: 'libraries' }); this.skipLine(); this.endLine('USE'); return null; } + const r = this.next() as TextToken; + routine = r.v; routineCol = r.col; + } + const s = this.node('Use', t, { module: m.v, moduleCol: m.col, routine, routineCol, ieee: m.v.toUpperCase() === 'IEEE' && !routine }); + this.endLine('USE'); + return s; + } + + declare(): Stmt | null { + const t = this.next(); + const names: DeclName[] = []; + do { + if (names.length) this.next(); + if (this.tok.t === 'blank') { names.push({ name: `<${this.tok.v}>`, blank: true, line: this.tok.line, col: this.tok.col }); this.next(); continue; } + if (this.tok.t !== 'name') { + this.error('expected', this.tok, this.tok.t === 'kw' ? `${this.tok.v} is a keyword, so it cannot be a name.` : `DECLARE needs a name${this.describeFound()}.`, { guide: 'names' }); + this.skipLine(); this.endLine('DECLARE'); return null; + } + const n = this.next() as TextToken; + names.push({ name: n.v, line: n.line, col: n.col }); + } while (this.isOp(',')); + if (this.isOp('<-') || this.isOp('=')) { + this.error('declare-assign', this.tok, 'A declaration fixes a name and a type, and gives no value. Assign on the next line: DECLARE Total : INTEGER, then Total <- 0.', { guide: 'declaring' }); + this.skipLine(); this.endLine('DECLARE'); return null; + } + if (!this.expectOp(':', `DECLARE needs a colon and a type after the name, as in DECLARE ${names[0]?.name || 'Counter'} : INTEGER.`)) { this.skipLine(); this.endLine('DECLARE'); return null; } + const type = this.typeExpr(); + const s = this.node('Declare', t, { names, type }); + this.endLine('the declaration'); + return s; + } + + typeExpr(): TypeExpr { + if (this.depth >= MAX_NESTING) { const t = this.tok; this.giveUp(`This type is nested more than ${MAX_NESTING} levels deep, which is more than this notation reads.`); return { tk: 'error', line: t.line, col: t.col }; } + this.depth++; + try { return this.typeInner(); } finally { this.depth--; } + } + + typeInner(): TypeExpr { + const t = this.tok; + if (t.t === 'blank') { this.next(); return { tk: 'blank', text: t.v, line: t.line, col: t.col }; } + if (t.t === 'name') { this.next(); return { tk: 'named', name: t.v, line: t.line, col: t.col }; } + if (t.t !== 'kw') { this.error('expected', t, `Expected a type here${this.describeFound()}, such as INTEGER, STRING, or LIST OF INTEGER.`, { guide: 'the-types' }); return { tk: 'error', line: t.line, col: t.col }; } + switch (t.v) { + case 'INTEGER': case 'FLOAT': case 'CHAR': case 'STRING': case 'BOOL': + case 'INT8': case 'INT16': case 'INT32': case 'INT64': case 'UINT8': case 'UINT16': case 'UINT32': case 'UINT64': case 'FLOAT32': case 'FLOAT64': + this.next(); return { tk: 'scalar', name: t.v as ScalarName, line: t.line, col: t.col }; + case 'LIST': case 'SET': { + this.next(); + if (!this.expectKw('OF', `${t.v} needs OF and the type of what it holds, as in ${t.v} OF INTEGER.`)) return { tk: 'error', line: t.line, col: t.col }; + return { tk: t.v === 'LIST' ? 'list' : 'set', of: this.typeExpr(), line: t.line, col: t.col }; + } + case 'ARRAY': { + this.next(); + const dims: Expr[] = []; + if (this.expectOp('[', 'ARRAY needs its length in square brackets, as in ARRAY[4] OF INTEGER.')) { + dims.push(this.expr()); + while (this.isOp(',')) { this.next(); dims.push(this.expr()); } + if (this.isOp(':')) { + this.error('array-bounds', this.tok, 'Arrays here are declared by length, not by first and last index: ARRAY[30] holds positions 0 to 29.', { guide: 'array-when-you-know-how-many' }); + this.next(); this.expr(); + } + this.expectOp(']'); + } + if (dims.length > 2) this.error('array-dims', t, 'An ARRAY has one or two dimensions in this notation.'); + if (!this.expectKw('OF', 'ARRAY needs OF and the type of what it holds, as in ARRAY[4] OF INTEGER.')) return { tk: 'error', line: t.line, col: t.col }; + return { tk: 'array', dims, of: this.typeExpr(), line: t.line, col: t.col }; + } + case 'HASHMAP': { + this.next(); + if (!this.expectKw('OF', 'HASHMAP needs OF, the key type, a colon, and the value type: HASHMAP OF STRING : CHAR.')) return { tk: 'error', line: t.line, col: t.col }; + const key = this.typeExpr(); + if (!this.expectOp(':', 'HASHMAP needs a colon between the key type and the value type: HASHMAP OF STRING : CHAR.')) return { tk: 'error', line: t.line, col: t.col }; + return { tk: 'map', key, val: this.typeExpr(), line: t.line, col: t.col }; + } + default: + this.error('expected', t, `${t.v} is not a type. The types are INTEGER, FLOAT, CHAR, STRING, BOOL, the sized numbers (INT8 to INT64, UINT8 to UINT64, FLOAT32, FLOAT64), ARRAY, LIST, HASHMAP, SET, and records you define with TYPE.`, { guide: 'the-types' }); + this.next(); + return { tk: 'error', line: t.line, col: t.col }; + } + } + + constant(): Stmt | null { + const t = this.next(); + if (this.tok.t !== 'name') { this.error('expected', this.tok, 'CONSTANT needs a name.'); this.skipLine(); this.endLine('CONSTANT'); return null; } + const n = this.next() as TextToken; + if (this.isOp('<-')) { + this.error('constant-arrow', this.tok, 'CONSTANT uses =, because nothing is being assigned: the line fixes a name and a value together, once.', { guide: 'constants', fix: { line: this.tok.line, col: this.tok.col, endCol: this.tok.end, text: '=' } }); + this.next(); + } else if (!this.expectOp('=', `CONSTANT needs = and a value, as in CONSTANT ${n.v} = 3.`)) { this.skipLine(); this.endLine('CONSTANT'); return null; } + const value = this.expr(); + const s = this.node('Constant', t, { name: n.v, value }); + this.endLine('the constant'); + return s; + } + + define(): Stmt | null { + const t = this.next(); + const n = this.nameTok(); + if (!n) { this.error('expected', this.tok, 'DEFINE needs the name of the set it creates.'); this.skipLine(); this.endLine('DEFINE'); return null; } + let values: Expr[] = []; + if (this.expectOp('(', `DEFINE lists the members in brackets: DEFINE ${n.v} ('A', 'G') : BaseSet.`)) values = this.args(')'); + let typeName: string | null = null; + if (this.expectOp(':', 'DEFINE ends with a colon and the name of a set type made with TYPE ... = SET OF ....')) { + if (this.tok.t === 'name' || this.tok.t === 'blank') typeName = this.nameTok()!.v; + else this.error('expected', this.tok, 'Expected the name of a set type after the colon.'); + } + const s = this.node('Define', t, { name: n.v, values, typeName }); + this.endLine('DEFINE'); + return s; + } + + typeDef(): Stmt | null { + const t = this.next(); + const n = this.nameTok(); + if (!n) { this.error('expected', this.tok, 'TYPE needs a name for the new type.'); this.skipLine(); return null; } + if (this.isOp('=')) { + this.next(); + const of = this.typeExpr(); + if (of.tk !== 'set') this.error('type-alias', t, 'A TYPE with = names a set type: TYPE BaseSet = SET OF CHAR. Records use TYPE Name ... ENDTYPE.', { guide: 'set-for-membership' }); + const s = this.node('SetTypeDef', t, { name: n.v, of: (of as { of?: TypeExpr }).of || of }); + this.endLine('the TYPE line'); + return s; + } + this.endLine(`TYPE ${n.v}`); + const fields: FieldDecl[] = []; + const r = this.withOpen('TYPE', t, ['ENDTYPE'], () => this.block(['ENDTYPE'])); + for (const s of r.body) { + if (s.kind === 'Declare') for (const nm of s.names) fields.push({ name: nm.name, type: s.type, line: s.line, col: nm.col }); + else this.diags.push(diag('error', 'record-body', s.line, 'A TYPE ... ENDTYPE block holds only DECLARE lines, one per field.', { col: s.col, guide: 'record-for-grouping-related-fields' })); + } + const s = this.node('TypeDef', t, { name: n.v, fields, endLine: r.stop ? r.stop.line : null }); + if (r.stop) { this.next(); this.endLine('ENDTYPE'); } else this.missingCloser('TYPE', t, 'ENDTYPE'); + return s; + } + + ifStmt(): IfStmt { + const t = this.next(); + const branches: IfBranch[] = []; + let elseBody: Stmt[] | null = null; + let elseLine: number | null = null; + let cond = this.condition('IF'); + if (!this.isKw('THEN')) { + if (this.isOp('{') || this.isOp(':')) this.error('expected', this.tok, 'IF ends its line with THEN, not with a bracket or colon.'); + else this.error('missing-then', this.tok, 'An IF line ends with THEN, after the condition.', { guide: 'if', fix: { line: t.line, col: this.lines[t.line - 1].text.length + 1, endCol: this.lines[t.line - 1].text.length + 1, text: ' THEN' } }); + this.skipLine(); + } else this.next(); + this.endLine('THEN'); + let viaElseIf = false; + const stops = ['ELSEIF', 'ELSE', 'ENDIF']; + let branchTok = t; + for (; ;) { + const r = this.withOpen('IF', t, stops, () => this.block(stops)); + branches.push({ cond, body: r.body, line: branchTok.line }); + if (!r.stop) { this.missingCloser('IF', t, 'ENDIF', t.elseIf ? ' An ELSE IF written on one line opens a second IF with its own ENDIF; write ELSEIF, as one word, to continue the first IF.' : ''); return this.node('If', t, { branches, elseBody, elseLine, endLine: null }); } + if (r.stop.v === 'ELSEIF') { + branchTok = this.next(); + cond = this.condition('ELSEIF'); + if (!this.expectKw('THEN', 'An ELSEIF line ends with THEN, after the condition.')) this.skipLine(); + this.endLine('THEN'); + viaElseIf = true; + continue; + } + if (r.stop.v === 'ELSE') { + const e = this.next(); + elseLine = e.line; + if (this.isKw('IF')) { + // ELSE IF on one line: a second IF, nested in the ELSE, with its own ENDIF. + this.tok.elseIf = true; + const inner = this.ifStmt(); + const rest = this.withOpen('IF', t, ['ENDIF'], () => this.block(['ENDIF'])); + elseBody = [inner, ...rest.body]; + if (!rest.stop) { this.missingCloser('IF', t, 'ENDIF', ' The ELSE IF on line ' + e.line + ' opened a second IF, which took the first ENDIF. Write ELSEIF, as one word, to continue this IF instead.'); return this.node('If', t, { branches, elseBody, elseLine, endLine: null, viaElseIf }); } + const end = this.next(); + this.endLine('ENDIF'); + return this.node('If', t, { branches, elseBody, elseLine, endLine: end.line, viaElseIf }); + } + this.endLine('ELSE'); + const r2 = this.withOpen('IF', t, ['ENDIF'], () => this.block(['ENDIF', 'ELSE', 'ELSEIF'])); + elseBody = r2.body; + if (r2.stop && r2.stop.v !== 'ENDIF') { + this.error('else-order', r2.stop, `${r2.stop.v} cannot follow ELSE: ELSE is the last branch of an IF.`, { guide: 'if' }); + this.skipLine(); + const r3 = this.withOpen('IF', t, ['ENDIF'], () => this.block(['ENDIF'])); + elseBody.push(...r3.body); + if (!r3.stop) { this.missingCloser('IF', t, 'ENDIF'); return this.node('If', t, { branches, elseBody, elseLine, endLine: null }); } + const end = this.next(); this.endLine('ENDIF'); + return this.node('If', t, { branches, elseBody, elseLine, endLine: end.line, viaElseIf }); + } + if (!r2.stop) { this.missingCloser('IF', t, 'ENDIF'); return this.node('If', t, { branches, elseBody, elseLine, endLine: null, viaElseIf }); } + const end = this.next(); + this.endLine('ENDIF'); + return this.node('If', t, { branches, elseBody, elseLine, endLine: end.line, viaElseIf }); + } + const end = this.next(); + this.endLine('ENDIF'); + return this.node('If', t, { branches, elseBody, elseLine, endLine: end.line, viaElseIf }); + } + } + + condition(word: string): Expr { + if (this.atLineEnd() || this.isKw('THEN')) { + this.error('expected', this.tok, `${word} needs a condition.`); + return { kind: 'Bool', value: true, line: this.tok.line, col: this.tok.col, missing: true }; + } + return this.expr(); + } + + caseStmt(): CaseStmt { + const t = this.next(); + if (!this.isKw('OF')) { + this.error('expected', this.tok, 'CASE is followed by OF and the value being tested: CASE OF Base.', { guide: 'case' }); + } else this.next(); + const subject = this.atLineEnd() ? null : this.expr(); + this.endLine('CASE OF'); + const arms: CaseArm[] = []; + let otherwise: Stmt[] | null = null; + let otherwiseLine: number | null = null; + this.open.push({ word: 'CASE', line: t.line, stops: ['ENDCASE', 'OTHERWISE'] }); + try { + this.skipBlankLines(); + for (; ;) { + const k = this.tok; + if (k.t === 'eof') { this.missingCloser('CASE', t, 'ENDCASE'); return this.node('Case', t, { subject, arms, otherwise, otherwiseLine, endLine: null }); } + if (this.isKw('ENDCASE')) { const end = this.next(); this.endLine('ENDCASE'); return this.node('Case', t, { subject, arms, otherwise, otherwiseLine, endLine: end.line }); } + if (k.t === 'kw' && BLOCK_ENDS.has(k.v) && k.v !== 'OTHERWISE' && this.open.slice(0, -1).some((o) => o.stops.includes(k.v))) { + this.missingCloser('CASE', t, 'ENDCASE'); + return this.node('Case', t, { subject, arms, otherwise, otherwiseLine, endLine: null }); + } + if (this.isKw('OTHERWISE')) { + const o = this.next(); + otherwiseLine = o.line; + if (otherwise) this.error('case-otherwise', o, 'A CASE has at most one OTHERWISE.', { guide: 'case' }); + this.expectOp(':', 'OTHERWISE is followed by a colon: OTHERWISE : .'); + otherwise = this.armBody(); + continue; + } + if (otherwise) this.error('case-otherwise', k, 'OTHERWISE must come last in a CASE, and this branch comes after it.', { guide: 'case' }); + // a label line: values and ranges, a colon, then a statement or an indented block + const labels: CaseLabel[] = []; + const armTok = k; + for (; ;) { + const low = this.expr(); + if (this.isKw('TO')) { this.next(); labels.push({ low, high: this.expr() }); } else labels.push({ value: low }); + if (this.isOp(',')) { this.next(); continue; } + break; + } + if (!this.expectOp(':', 'Each CASE branch is a value, a colon, and what to do: \'A\' : Complement <- \'T\'.')) { this.skipLine(); this.skipBlankLines(); continue; } + arms.push({ labels, body: this.armBody(), line: armTok.line }); + } + } finally { this.open.pop(); } + } + + /** What follows a CASE label's colon: the rest of the line, or an indented block. */ + armBody(): Stmt[] { + if (!this.atLineEnd()) { + const s = this.statement(); + this.skipBlankLines(); + return s ? [s] : []; + } + this.next(); + const body: Stmt[] = []; + this.skipBlankLines(); + for (; ;) { + const k = this.tok; + if (k.t === 'eof' || this.isKw('ENDCASE') || this.isKw('OTHERWISE')) break; + if (k.t === 'kw' && BLOCK_ENDS.has(k.v)) break; + if (this.looksLikeLabel()) break; + const s = this.statement(); + if (s) body.push(s); + this.skipBlankLines(); + } + return body; + } + + /** A line with a colon at the top level that no statement keyword begins. */ + looksLikeLabel(): boolean { + const k = this.tok; + if (k.t === 'kw' && !['TRUE', 'FALSE', 'NOT'].includes(k.v)) return false; + let depth = 0; + for (let j = this.p; j < this.toks.length; j++) { + const x = this.toks[j]; + if (x.t === 'nl' || x.t === 'eof') return false; + if (x.t === 'op' && (x.v === '(' || x.v === '[' || x.v === '{')) depth++; + if (x.t === 'op' && (x.v === ')' || x.v === ']' || x.v === '}')) depth--; + if (x.t === 'op' && x.v === '<-') return false; + if (depth === 0 && x.t === 'op' && x.v === ':') return true; + } + return false; + } + + forStmt(): ForStmt | ForEachStmt | null { + const t = this.next(); + if (this.isWord('EACH')) { + this.next(); + if (this.tok.t !== 'name') { this.error('expected', this.tok, 'FOR EACH needs a name for the value it visits.'); this.skipLine(); } + const v = this.tok.t === 'name' ? this.next() as TextToken : { v: '?', line: t.line, col: t.col }; + if (!this.expectKw('IN', `FOR EACH ${v.v} is followed by IN and the collection to visit.`)) this.skipLine(); + if (this.atLineEnd()) this.error('expected', this.tok, `FOR EACH ${v.v} IN needs the collection to visit after IN.`); + const iterable = this.atLineEnd() ? null : this.domain(); + this.endLine('FOR EACH'); + const r = this.withOpen('FOR', t, ['NEXT'], () => this.block(['NEXT'])); + const s = this.node('ForEach', t, { var: v.v, varCol: v.col, iterable, body: r.body, endLine: null }); + this.finishNext(s, t, v.v, r); + return s; + } + if (this.tok.t !== 'name' && this.tok.t !== 'blank') { + this.error('expected', this.tok, this.tok.t === 'kw' ? `${this.tok.v} is a keyword, so it cannot be a loop counter.` : 'FOR needs a counter name: FOR I <- 0 TO N.', { guide: 'for-when-you-know-how-many-times' }); + this.skipLine(); + const r = this.withOpen('FOR', t, ['NEXT'], () => this.block(['NEXT'])); + if (r.stop) { this.next(); this.skipLine(); } + return null; + } + const v = this.nameTok()!; // the token is a name or a blank + if (this.isKw('IN')) { + this.error('for-in', this.tok, `To visit every value of a collection, write FOR EACH ${v.v} IN .... A counting loop is FOR ${v.v} <- 0 TO N.`, { guide: 'for-each-when-you-want-every-value' }); + this.next(); + const iterable = this.domain(); + this.endLine('FOR'); + const r = this.withOpen('FOR', t, ['NEXT'], () => this.block(['NEXT'])); + const s = this.node('ForEach', t, { var: v.v, varCol: v.col, iterable, body: r.body, endLine: null }); + this.finishNext(s, t, v.v, r); + return s; + } + if (this.isOp('=')) { + this.error('equals-assign', this.tok, `A FOR loop gives its counter a starting value with <-: FOR ${v.v} <- 0 TO N.`, { guide: 'for-when-you-know-how-many-times', fix: { line: this.tok.line, col: this.tok.col, endCol: this.tok.end, text: '<-' } }); + this.next(); + } else if (!this.expectOp('<-', `A FOR loop gives its counter a starting value with <-: FOR ${v.v} <- 0 TO N.`)) { this.skipLine(); } + const start = this.atLineEnd() ? null : this.expr(); + let stop: Expr | null = null; + let step: Expr | null = null; + if (this.expectKw('TO', 'A FOR loop needs TO and a stopping value: FOR I <- 0 TO N.')) { + stop = this.expr(); + if (this.isKw('STEP')) { this.next(); step = this.expr(); } + } + this.endLine('the FOR line'); + const r = this.withOpen('FOR', t, ['NEXT'], () => this.block(['NEXT'])); + const s = this.node('For', t, { var: v.v, varCol: v.col, start, stop, step, body: r.body, endLine: null }); + this.finishNext(s, t, v.v, r); + return s; + } + + finishNext(s: ForStmt | ForEachStmt, t: Loc, name: string, r: BlockRead): void { + if (!r.stop) { this.missingCloser(`FOR ${name}`, t, `NEXT ${name}`); return; } + const n = this.next(); + s.endLine = n.line; + if (this.tok.t === 'name' || this.tok.t === 'blank') { + const w = this.nameTok()!; + s.nextName = w.v; + if (!w.blank && !name.startsWith('<') && w.v.toUpperCase() !== name.toUpperCase()) { + this.diags.push(diag('error', 'next-mismatch', n.line, `NEXT ${w.v} closes a loop over ${w.v}, and the innermost open loop is FOR ${name} on line ${t.line}.`, { + col: w.col, guide: 'nested-loops', hint: 'Inner loops close before outer ones: the last FOR opened is the first one NEXT closes.', + fix: { line: w.line, col: w.col, endCol: w.end, text: name }, + })); + } + } else { + s.nextName = null; + } + this.endLine(`NEXT ${name}`); + } + + whileStmt(): WhileStmt { + const t = this.next(); + const cond = this.condition('WHILE'); + if (this.isWord('DO')) { + this.error('while-do', this.tok, FOREIGN_WORDS.DO, { guide: 'while-when-the-condition-is-checked-first', fix: { line: this.tok.line, col: this.tok.col - 1, endCol: this.tok.end, text: '' } }); + this.next(); + } + this.endLine('the WHILE condition'); + const r = this.withOpen('WHILE', t, ['ENDWHILE'], () => this.block(['ENDWHILE'])); + const s = this.node('While', t, { cond, body: r.body, endLine: null }); + if (!r.stop) this.missingCloser('WHILE', t, 'ENDWHILE'); + else { s.endLine = this.next().line; this.endLine('ENDWHILE'); } + return s; + } + + repeatStmt(): RepeatStmt { + const t = this.next(); + this.endLine('REPEAT'); + const r = this.withOpen('REPEAT', t, ['UNTIL'], () => this.block(['UNTIL'])); + const s = this.node('Repeat', t, { body: r.body, cond: null, endLine: null }); + if (!r.stop) { this.missingCloser('REPEAT', t, 'UNTIL and a condition'); return s; } + const u = this.next(); + s.endLine = u.line; + s.cond = this.condition('UNTIL'); + this.endLine('the UNTIL condition'); + return s; + } + + subroutine(): SubStmt { + const t = this.next() as TextToken; // FUNCTION or PROCEDURE + const isFn = t.v === 'FUNCTION'; + const word = t.v; + if (this.open.length) this.error('nested-subroutine', t, `A ${word} is defined at the top level, never inside another block.`); + const n = this.nameTok() || { v: '?', col: t.col }; + if (n.v === '?') { this.error('expected', this.tok, `${word} needs a name.`); this.skipLine(); } + const params: ParamDecl[] = []; + if (this.isOp('(')) { + this.next(); + if (!this.isOp(')')) { + for (; ;) { + let mode: ParamMode = 'BYVAL'; + let explicit = false; + if (this.isKw('BYREF') || this.isKw('BYVAL')) { mode = this.next().v as ParamMode; explicit = true; } + if (this.isOp('...')) { this.next(); break; } + if (this.tok.t === 'blank') { + const b = this.next(); + let type: TypeExpr = { tk: 'blank', text: b.v as string }; + if (this.isOp(':')) { this.next(); type = this.typeExpr(); } + params.push({ name: `<${b.v}>`, blank: true, mode, type, line: b.line, col: b.col }); + } else if (this.tok.t !== 'name') { this.error('expected', this.tok, `Each parameter is a name, a colon, and a type: ${isFn ? 'FUNCTION' : 'PROCEDURE'} ${n.v}(B : CHAR, T : STRING).`); break; } + else { + const pn = this.next() as TextToken; + let type: TypeExpr = { tk: 'error' }; + if (this.expectOp(':', `Parameter ${pn.v} needs a colon and a type, as in ${pn.v} : INTEGER.`)) type = this.typeExpr(); + params.push({ name: pn.v, mode, explicit, type, line: pn.line, col: pn.col }); + } + if (this.isOp(',')) { this.next(); continue; } + break; + } + } + this.expectOp(')', 'The parameter list ends with a closing bracket.'); + } else if (isFn) { + this.error('expected', this.tok, `A FUNCTION's name is followed by its parameters in brackets, even when there are none: FUNCTION ${n.v}().`); + } + let returns: TypeExpr | null = null; + if (isFn) { + if (this.isKw('RETURNS')) { this.next(); returns = this.typeExpr(); } + else this.error('missing-returns', this.tok, `A FUNCTION states the type of its answer: FUNCTION ${n.v}(...) RETURNS INTEGER. A subroutine with no answer is a PROCEDURE.`, { guide: 'function' }); + } else if (this.isKw('RETURNS')) { + this.error('procedure-returns', this.tok, 'A PROCEDURE hands nothing back, so it has no RETURNS. A subroutine with an answer is a FUNCTION.', { guide: 'procedure' }); + this.next(); this.typeExpr(); + } + this.endLine(`the ${word} line`); + const closer = isFn ? 'ENDFUNCTION' : 'ENDPROCEDURE'; + const r = this.withOpen(word, t, [closer], () => this.block([closer, isFn ? 'ENDPROCEDURE' : 'ENDFUNCTION'])); + const s = this.node(isFn ? 'Function' : 'Procedure', t, { name: n.v, nameCol: n.col, params, returns, body: r.body, endLine: null }); + if (!r.stop) this.missingCloser(`${word} ${n.v}`, t, closer); + else { + const end = this.next(); + s.endLine = end.line; + if (end.v !== closer) this.error('wrong-closer', end, `${word} ${n.v} closes with ${closer}, not ${end.v}.`, { fix: { line: end.line, col: end.col, endCol: end.end, text: closer } }); + this.endLine(closer); + } + return s; + } + + /* ---------- expressions ---------- */ + args(close: string): Expr[] { + const list: Expr[] = []; + if (this.isOp(close)) { this.next(); return list; } + for (; ;) { + if (this.atLineEnd()) { this.error('expected', this.tok, `Expected ${close} to close the bracket.`); return list; } + list.push(this.expr()); + if (this.isOp(',')) { this.next(); continue; } + if (this.isOp(close)) { this.next(); return list; } + this.error('expected', this.tok, `Expected a comma or ${close}${this.describeFound()}.`); + while (!this.atLineEnd() && !this.isOp(close)) this.next(); + if (this.isOp(close)) this.next(); + return list; + } + } + + expr(): Expr { return this.nested(() => this.orExpr()); } + + /** + * Read one more level of nesting (a bracketed expression, a block, a prefix operator). Every reader after this + * one (the checker, the analysis, the translators) walks the tree by calling itself, so a tree thousands of + * levels deep would end in the engine's own stack error instead of a diagnostic. Past MAX_NESTING the line is + * given up with one error, and what stands there reads as 0 (tests/pseudo-robustness.test.mjs). + */ + nested(read: () => R): R | NumExpr { + if (this.depth >= MAX_NESTING) return this.giveUp(`This is nested more than ${MAX_NESTING} levels deep, which is more than this notation reads. Give the inner parts names of their own.`); + this.depth++; + try { return this.capped(read()); } finally { this.depth--; } + } + + /** + * An expression that is complete, or a 0 in its place (one `too-deep` error) when its tree is taller than MAX_TREE. + * Every expression of a statement comes through here: `expr()` at each level of brackets, and `postfix()` for an + * assignment's target. + */ + capped(e: R): R | NumExpr { + if (this.heightOf(e) <= MAX_TREE) return e; + return this.giveUp(`This builds a value more than ${MAX_TREE} levels deep (brackets that each hold a long chain), which is more than this notation reads. Give the inner parts names of their own.`); + } + + /** + * How tall an expression's tree is: 1 for a single value, one more for each operator, call or index over it. + * Measured with a stack of its own (a tree this is meant to refuse would overflow a walk that calls itself), and + * kept for every node, so measuring again at each level of brackets costs one pass over the line in all. + */ + heightOf(root: Expr): number { + const heights = this.heights; + const known = heights.get(root); + if (known !== undefined) return known; + const parts = (node: object): object[] => { + const out: object[] = []; + const take = (v: unknown): void => { + if (Array.isArray(v)) for (const x of v) take(x); + else if (v !== null && typeof v === 'object') out.push(v); + }; + for (const v of Object.values(node)) take(v); + return out; + }; + const stack: object[] = [root]; + const open = new Set([root]); + while (stack.length) { + const node = stack[stack.length - 1]!; + let tallest = 0; + let waiting = false; + for (const part of parts(node)) { + const h = heights.get(part); + if (h !== undefined) { if (h > tallest) tallest = h; } + else if (!open.has(part)) { open.add(part); stack.push(part); waiting = true; } + } + if (!waiting) { heights.set(node, tallest + 1); stack.pop(); } + } + return heights.get(root)!; + } + + /** One `too-deep` error for the statement, the rest of its line skipped, and a 0 where a value was expected. */ + giveUp(message: string): NumExpr { + const t = this.tok; + this.error('too-deep', t, message); + this.tooDeep = true; + this.skipLine(); + return { kind: 'Num', value: 0, float: false, line: t.line, col: t.col }; + } + + /** The `n`th operator of a chain such as A + B + C, which brings it to n + 1 operands: the tree leans left one level per operand. */ + chained(n: number): boolean { + if (n + 1 <= MAX_CHAIN) return true; + this.giveUp(`This joins more than ${MAX_CHAIN} values in one chain, which is more than this notation reads. Build the result in steps, or in a loop.`); + return false; + } + + orExpr(): Expr { + let left = this.andExpr(); + for (let n = 1; (this.isKw('OR')) && this.chained(n); n++) { + const op = this.next(); + left = { kind: 'Binary', op: 'OR', left, right: this.andExpr(), line: op.line, col: op.col }; + } + return left; + } + + andExpr(): Expr { + let left = this.notExpr(); + for (let n = 1; (this.isKw('AND')) && this.chained(n); n++) { + const op = this.next(); + left = { kind: 'Binary', op: 'AND', left, right: this.notExpr(), line: op.line, col: op.col }; + } + return left; + } + + notExpr(): Expr { + if (this.isKw('NOT')) { + const op = this.next(); + const arg = this.nested(() => this.notExpr()); + return { kind: 'Unary', op: 'NOT', arg, line: op.line, col: op.col }; + } + return this.compareExpr(); + } + + compareExpr(): Expr { + const first = this.concatExpr(); + const ops: Array<{ op: string; line: number; col: number; negated?: boolean }> = []; + const operands = [first]; + for (; ;) { + const t = this.tok; + if (t.t === 'op' && COMPARE.has(t.v)) { this.next(); ops.push({ op: t.v, line: t.line, col: t.col }); operands.push(this.concatExpr()); continue; } + if (t.t === 'op' && t.v === '=') { + this.error('equals-compare', t, '= does not compare in this notation. Write == to ask whether two values are equal.', { guide: 'comparison', fix: { line: t.line, col: t.col, endCol: t.end, text: '==' } }); + this.next(); ops.push({ op: '==', line: t.line, col: t.col }); operands.push(this.concatExpr()); continue; + } + if (this.isKw('IN')) { this.next(); ops.push({ op: 'IN', line: t.line, col: t.col }); operands.push(this.domain()); continue; } + if (this.isKw('NOT') && this.isKw('IN', this.peek())) { + this.error('not-in', t, 'Write NOT (X IN S) to ask whether X is missing from S.'); + this.next(); this.next(); ops.push({ op: 'IN', negated: true, line: t.line, col: t.col }); operands.push(this.domain()); continue; + } + break; + } + if (!ops.length) return first; + if (ops.length === 1) { + const e: Expr = { kind: 'Binary', op: ops[0].op, left: operands[0], right: operands[1], line: ops[0].line, col: ops[0].col }; + return ops[0].negated ? { kind: 'Unary', op: 'NOT', arg: e, line: e.line, col: e.col } : e; + } + return { kind: 'Chain', ops: ops.map((o) => o.op), operands, line: first.line, col: first.col }; + } + + /** The right side of IN, or a quantifier's domain: a range [A, B) or any value. */ + domain(): Expr { + if (this.isOp('[')) { + const save = this.p; + const t = this.next(); + if (this.isOp(']')) { this.next(); return { kind: 'ListLit', items: [], line: t.line, col: t.col }; } + const a = this.expr(); + if (this.isOp(',')) { + this.next(); + const b = this.expr(); + if (this.isOp(')')) { this.next(); return { kind: 'Range', lo: a, hi: b, line: t.line, col: t.col }; } + if (this.isOp(']') && !this.isOp(',', this.peek())) { + // [A, B] is a two-value list; say so if it looks like a range was meant + this.p = save; + const lst = this.concatExpr(); + if (lst.kind === 'ListLit' && lst.items.length === 2) lst.maybeRange = true; + return lst; + } + } + this.p = save; + } + return this.concatExpr(); + } + + concatExpr(): Expr { + let left = this.addExpr(); + for (let n = 1; (this.isOp('&')) && this.chained(n); n++) { + const op = this.next(); + left = { kind: 'Binary', op: '&', left, right: this.addExpr(), line: op.line, col: op.col }; + } + return left; + } + + addExpr(): Expr { + let left = this.mulExpr(); + for (let n = 1; (this.isOp('+') || this.isOp('-')) && this.chained(n); n++) { + const op = this.next(); + left = { kind: 'Binary', op: op.v as string, left, right: this.mulExpr(), line: op.line, col: op.col }; + } + return left; + } + + mulExpr(): Expr { + let left = this.unaryExpr(); + for (let n = 1; (this.isOp('*') || this.isOp('/') || this.isKw('DIV') || this.isKw('MOD')) && this.chained(n); n++) { + const op = this.next(); + left = { kind: 'Binary', op: op.v as string, left, right: this.unaryExpr(), line: op.line, col: op.col }; + } + return left; + } + + unaryExpr(): Expr { + if (this.isOp('-')) { + const op = this.next(); + const arg = this.nested(() => this.unaryExpr()); + if ((arg.kind === 'Num') && !arg.negated) return { ...arg, value: -arg.value, line: op.line, col: op.col, negated: true }; + return { kind: 'Unary', op: '-', arg, line: op.line, col: op.col }; + } + if (this.isOp('+')) { this.next(); return this.nested(() => this.unaryExpr()); } + if (this.isKw('NOT')) return this.notExpr(); + return this.powExpr(); + } + + powExpr(): Expr { + const base = this.postfix(this.primary()); + if (this.isOp('^')) { + const op = this.next(); + return { kind: 'Binary', op: '^', left: base, right: this.nested(() => this.unaryExpr()), line: op.line, col: op.col }; + } + return base; + } + + /** What follows a value: indexes, fields and calls, held to MAX_TREE as a whole. */ + postfix(e: Expr, opts: { noCall?: boolean } = {}): Expr { return this.capped(this.postfixChain(e, opts)); } + + postfixChain(e: Expr, opts: { noCall?: boolean } = {}): Expr { + for (let n = 1; ; n++) { + if (!this.chained(n)) return e; // L[0][0][0]... leans left like a chain of an operator + if (this.isOp('[')) { + const t = this.next(); + const indices: Expr[] = []; + if (this.isOp(']')) this.error('expected', this.tok, 'Square brackets after a name need a position inside them.'); + else { + indices.push(this.expr()); + while (this.isOp(',')) { this.next(); indices.push(this.expr()); } + if (this.isOp(':')) { + this.error('slice-syntax', this.tok, 'Take a run of positions with SLICE(X, A, B), which gives positions [A, B).', { guide: 'text-and-positions' }); + this.next(); this.expr(); + } + } + this.expectOp(']', 'A position in square brackets needs a closing ].'); + e = { kind: 'Index', target: e, indices, line: t.line, col: t.col }; + continue; + } + if (this.isOp('.')) { + const t = this.next(); + if (this.tok.t !== 'name') { this.error('expected', this.tok, 'A dot is followed by the name of a field, as in R.Name.'); return e; } + const f = this.next() as TextToken; + e = { kind: 'Field', target: e, field: f.v, line: t.line, col: f.col }; + continue; + } + if (this.isOp('(') && !opts.noCall && e.kind === 'Name' && !e.paren) { + const t = this.next(); + const args = this.args(')'); + e = { kind: 'CallExpr', name: e.name, args, line: e.line, col: e.col, openCol: t.col }; + continue; + } + return e; + } + } + + primary(): Expr { + const t = this.tok; + switch (t.t) { + case 'int': this.next(); return { kind: 'Num', value: t.v, float: false, line: t.line, col: t.col }; + case 'float': this.next(); return { kind: 'Num', value: t.v, float: true, text: t.text, line: t.line, col: t.col }; + case 'str': this.next(); return { kind: 'Str', value: t.v, line: t.line, col: t.col }; + case 'char': this.next(); return { kind: 'Char', value: t.v, line: t.line, col: t.col }; + case 'blank': this.next(); return { kind: 'Blank', text: t.v, line: t.line, col: t.col }; + case 'name': { + const up = t.v.toUpperCase(); + if (up === 'SUM' && this.peek().t === 'name' && this.isKw('IN', this.peek(2))) return this.quantifier('SUM'); + this.next(); + return { kind: 'Name', name: t.v, line: t.line, col: t.col }; + } + case 'kw': + if (t.v === 'TRUE' || t.v === 'FALSE') { this.next(); return { kind: 'Bool', value: t.v === 'TRUE', line: t.line, col: t.col }; } + if (SIZED_NAMES.has(t.v) && this.isOp('(', this.peek())) { + // INT32(X), FLOAT32(X): a conversion, written the way Mojo writes Int32(x) + this.next(); + const open = this.next(); + return { kind: 'CallExpr', name: t.v, args: this.args(')'), convert: true, line: t.line, col: t.col, openCol: open.col }; + } + if (TYPE_WORDS.has(t.v)) { this.next(); return { kind: 'TypeRef', name: t.v as TypeRefExpr['name'], line: t.line, col: t.col }; } + if (t.v === 'FORALL' || t.v === 'EXISTS') return this.quantifier(t.v); + if (t.v === 'NOT') return this.notExpr(); + this.error('expected', t, BUILTINS[t.v] + ? `${t.v} needs brackets around what it works on.` + : `${t.v} is a keyword, and a value was expected here.`); + this.next(); + return { kind: 'Error', line: t.line, col: t.col }; + case 'op': + if (t.v === '(') { + this.next(); + const e = this.expr(); + this.expectOp(')', 'This bracket is never closed.'); + return { ...e, paren: true }; + } + if (t.v === '[') { + this.next(); + const items = this.args(']'); + return { kind: 'ListLit', items, line: t.line, col: t.col }; + } + if (t.v === '{') return this.braces(); + if (t.v === '...') { this.next(); return { kind: 'Blank', text: '...', line: t.line, col: t.col }; } + this.error('expected', t, `Expected a value here${this.describeFound()}.`); + if (!this.atLineEnd()) this.next(); + return { kind: 'Error', line: t.line, col: t.col }; + default: + this.error('expected', t, t.t === 'nl' || t.t === 'eof' ? 'The line ended where a value was expected.' : `Expected a value here${this.describeFound()}.`); + return { kind: 'Error', line: t.line, col: t.col }; + } + } + + /** { 'A', 'G' } is a SET; { I IN [0, N) : T[I] == B } is the set of positions that pass a test. */ + braces(): Expr { + const t = this.next(); + if (this.tok.t === 'name' && this.isKw('IN', this.peek())) { + const v = this.next() as TextToken; + this.next(); + const dom = this.domain(); + this.expectOp(':', `A set written by rule needs a colon between the candidates and the test: { ${v.v} IN [0, N) : }.`); + const body = this.expr(); + this.expectOp('}', 'The set is never closed with }.'); + return { kind: 'SetBuilder', var: v.v, domain: dom, body, line: t.line, col: t.col }; + } + if (this.isOp('}')) { this.next(); return { kind: 'SetLit', items: [], line: t.line, col: t.col }; } + const first = this.expr(); + if (this.isOp(':')) { + // { key : value, ... } is a HASHMAP + const entries: Array<{ key: Expr; value: Expr }> = []; + let k = first; + for (; ;) { + this.expectOp(':', 'Each entry of a HASHMAP is a key, a colon, and a value.'); + entries.push({ key: k, value: this.expr() }); + if (this.isOp(',')) { this.next(); k = this.expr(); continue; } + this.expectOp('}', 'The HASHMAP is never closed with }.'); + return { kind: 'MapLit', entries, line: t.line, col: t.col }; + } + } + const items = [first]; + while (this.isOp(',')) { this.next(); items.push(this.expr()); } + this.expectOp('}', 'The set is never closed with }.'); + return { kind: 'SetLit', items, line: t.line, col: t.col }; + } + + quantifier(q: QuantExpr['q']): Expr { + const t = this.next(); + if (this.tok.t !== 'name') { this.error('expected', this.tok, `${q} needs a name for what it ranges over: ${q} I IN [0, N) : ....`); return { kind: 'Error', line: t.line, col: t.col }; } + const v = this.next() as TextToken; + if (!this.expectKw('IN', `${q} ${v.v} is followed by IN and what ${v.v} ranges over.`)) return { kind: 'Error', line: t.line, col: t.col }; + const dom = this.domain(); + this.expectOp(':', `${q} needs a colon before the part that must hold: ${q} ${v.v} IN [0, N) : ....`); + const body = this.expr(); + return { kind: 'Quant', q, var: v.v, domain: dom, body, line: t.line, col: t.col }; + } +} + +/** What sourceOf renders: an expression, or RESULT and OLD(X) as other readers of a contract build them. */ +type SourceNode = Expr | { kind: 'Result' } | { kind: 'Old'; arg: Expr }; + +/** A short source-like rendering of an expression, for messages. */ +// typed-port: the prover (vcgen.ts) and the JavaScript readers pass nodes of their own (LooseType-like, or kinds the +// parser never makes, such as Result and Old), read here as an Expr. +export function sourceOf(e0: Expr | (LooseType & { kind?: string }) | { kind: string } | null | undefined): string { + const e = e0 as SourceNode | null | undefined; + if (!e) return '?'; + const wrap = (x: Expr): string => (x && (x.kind === 'Binary' || x.kind === 'Chain' || x.kind === 'Quant') ? `(${sourceOf(x)})` : sourceOf(x)); + switch (e.kind) { + case 'Num': return e.float ? formatNum(e.value) : String(e.value); + case 'Str': return JSON.stringify(e.value); + case 'Char': return `'${e.value === "'" ? "\\'" : e.value}'`; + case 'Bool': return e.value ? 'TRUE' : 'FALSE'; + case 'Name': return e.name; + case 'Result': return 'RESULT'; + case 'Old': return `OLD(${sourceOf(e.arg)})`; + case 'Blank': return `<${e.text}>`; + case 'TypeRef': return e.name; + case 'Unary': return e.op === 'NOT' ? `NOT ${wrap(e.arg)}` : `-${wrap(e.arg)}`; + case 'Binary': return `${sideOf(e.left, e.op, 'l')} ${e.op} ${sideOf(e.right, e.op, 'r')}`; + case 'Chain': return e.operands.map((o, i) => (i ? `${e.ops[i - 1]} ${sourceOf(o)}` : sourceOf(o))).join(' '); + case 'Index': return `${postfix(e.target)}[${e.indices.map(sourceOf).join(', ')}]`; + case 'Field': return `${postfix(e.target)}.${e.field}`; + case 'CallExpr': return `${e.name}(${e.args.map(sourceOf).join(', ')})`; + case 'ListLit': return `[${e.items.map(sourceOf).join(', ')}]`; + case 'SetLit': return `{${e.items.map(sourceOf).join(', ')}}`; + case 'MapLit': return `{${e.entries.map((x) => `${sourceOf(x.key)}: ${sourceOf(x.value)}`).join(', ')}}`; + case 'Range': return `[${sourceOf(e.lo)}, ${sourceOf(e.hi)})`; + case 'SetBuilder': return `{ ${e.var} IN ${sourceOf(e.domain)} : ${sourceOf(e.body)} }`; + case 'Quant': return `${e.q} ${e.var} IN ${sourceOf(e.domain)} : ${sourceOf(e.body)}`; + default: return '?'; + } +} + +/** A target of [ ] or . keeps its parentheses unless it is a name, a call, or another postfix. */ +function postfix(x: Expr): string { + const s = sourceOf(x); + return x && ['Binary', 'Chain', 'Unary', 'Quant', 'SetBuilder'].includes(x.kind) ? `(${s})` : s; +} + +const BIND: Record = { OR: 1, AND: 2, '==': 4, '!=': 4, '<': 4, '<=': 4, '>': 4, '>=': 4, IN: 4, '&': 5, '+': 6, '-': 6, '*': 7, '/': 7, DIV: 7, MOD: 7, '^': 9 }; +function sideOf(x: Expr, op: string, side: 'l' | 'r'): string { + const s = sourceOf(x); + if (!x) return s; + if (x.kind === 'Quant' || x.kind === 'Chain') return `(${s})`; + if (x.kind === 'Unary' && x.op === 'NOT' && BIND[op] > 3) return `(${s})`; + // -2 ^ 2 is -(2 ^ 2): a negative number on the left of ^ was written in brackets, and keeps them + if ((x.kind === 'Unary' && x.op === '-' || x.kind === 'Num' && (x.value < 0 || Object.is(x.value, -0))) && op === '^' && side === 'l') return `(${s})`; + if (x.kind !== 'Binary') return s; + const a = BIND[x.op]; + const b = BIND[op]; + if (a < b) return `(${s})`; + if (a === b && side === 'r' && op !== '^' && !(op === x.op && (op === '+' || op === '*' || op === 'AND' || op === 'OR' || op === '&'))) return `(${s})`; + if (a === b && side === 'l' && op === '^') return `(${s})`; + if (a === 4 && b === 4) return `(${s})`; + return s; +} + +function formatNum(v: number): string { const s = String(v); return /[.e]/.test(s) ? s : `${s}.0`; } + +/* ------------------------------------------------------------------ */ +/* Entry point */ +/* ------------------------------------------------------------------ */ + +export function parse(source: unknown): Parsed { + const lexed = lex(source); + const p = new Parser(lexed); + const ast = p.program(); + const diags = lexed.diags.sort((a, b) => a.line - b.line || a.col - b.col); + return { ast, diags, tokens: lexed.tokens, lines: lexed.lines, comments: lexed.comments, source: String(source ?? '') }; +} + +/** Parse one expression on its own, for test cases and the lab's call box. */ +export function parseExpression(source: unknown): { expr: Expr; diags: Diag[] } { + const lexed = lex(source); + const p = new Parser(lexed); + p.skipBlankLines(); + const e = p.expr(); + p.skipBlankLines(); + if (p.tok.t !== 'eof') p.error('extra', p.tok, `Unexpected ${p.tok.raw || p.tok.v} after the expression.`); + return { expr: e, diags: lexed.diags }; +} + +export { OPENER_WORD }; diff --git a/src/engines/pseudo/plan.js b/src/engines/pseudo/plan.js new file mode 100644 index 0000000..b139bc1 --- /dev/null +++ b/src/engines/pseudo/plan.js @@ -0,0 +1,501 @@ +/** + * pseudo/plan.js + * + * From an idea to pseudocode. Almost every algorithm in this course is a + * scan: set up, visit, test, update, then report. A plan is three choices, + * each a question a student can answer in words: + * + * What do you want back? answer: count, total, all positions, first, ... + * What do you walk over? walk: every position, every window, every start, ... + * What do you ask each time? test: the letter is B, the window is P, ... + * + * buildPlan({ answer, walk, test, names }) -> { + * source, pseudocode, with the four jobs marked in comments + * notes, [{ line, job, text }]: why each line is what it is + * spec, the specification in the course's notation + * tests, test lines to predict, with the answers left open + * warnings, combinations that need care + * } + * + * jobsReport(unit, diags) -> [{ job, status, lines, message }] + * the four jobs in a student's own program: which are there, and + * which are missing or in the wrong place. + */ + +import { PATTERN_NAMES } from './analyze.js'; + +/* ------------------------------------------------------------------ */ +/* The catalog */ +/* ------------------------------------------------------------------ */ + +export const ANSWERS = { + count: { + label: 'How many positions pass a test', example: 'how many A\'s are in a sequence', returns: 'INTEGER', job: 'Count', + words: 'one number, counting the positions that pass', + }, + total: { + label: 'A total of numbers found along the way', example: 'the sum of the quality scores', returns: 'INTEGER', job: 'Total', + words: 'one number, adding up a value at each position', + }, + all: { + label: 'Every position that passes a test', example: 'every place a primer binds', returns: 'LIST OF INTEGER', job: 'Find them all', + words: 'a LIST of positions, in order', + }, + first: { + label: 'The first position that passes a test', example: 'where the first N is', returns: 'INTEGER', job: 'Find the first', + words: 'one position; the length of the sequence means there is none', + }, + any: { + label: 'Whether any position passes', example: 'does the read contain an N at all', returns: 'BOOL', job: 'Check', + words: 'TRUE or FALSE: is there at least one', + }, + every: { + label: 'Whether every position passes', example: 'is the sequence all A, C, G, and T', returns: 'BOOL', job: 'Check', + words: 'TRUE or FALSE: does every position pass', + }, + best: { + label: 'The position of the largest value', example: 'the position of the best quality score', returns: 'INTEGER', job: 'Find the best', + words: 'one position, where the value is largest', + }, + filter: { + label: 'A new list of the values that pass', example: 'the reads that are long enough', returns: 'LIST OF', job: 'Filter', + words: 'a new LIST, keeping only the values that pass; the original is untouched', + }, + tally: { + label: 'How many times each value appears', example: 'how often each base occurs', returns: 'HASHMAP', job: 'Tally', + words: 'a HASHMAP from each value seen to how often it was seen', + }, + build: { + label: 'A new sequence, one letter for each letter', example: 'the complement of a strand', returns: 'STRING', job: 'Build', + words: 'a new STRING, built one letter at a time', + }, +}; + +export const WALKS = { + positions: { label: 'Every position of one sequence', example: 'each base of T', needs: ['T'] }, + windows: { label: 'Every window of K letters', example: 'each 3-mer of T', needs: ['T', 'K'] }, + starts: { label: 'Every start where a pattern could fit', example: 'each place P could begin in T', needs: ['T', 'P'] }, + codons: { label: 'Every codon, three letters at a time', example: 'each codon of a reading frame', needs: ['T'] }, + pairs: { label: 'Every pair of positions', example: 'each pair I < J', needs: ['T'] }, + paired: { label: 'Two sequences side by side', example: 'the same position of A and B', needs: ['A', 'B'] }, + values: { label: 'Every value in a list', example: 'each score in a LIST', needs: ['L'] }, +}; + +/** The tests that make sense for each walk. `expr(names)` gives the pseudocode, `words` the sentence. */ +export const TESTS = { + positions: [ + { id: 'is-base', label: 'the letter is a given base B', needs: ['B'], expr: (n, i) => `${n.T}[${i}] == ${n.B}`, words: (n) => `the letter is ${n.B}` }, + { id: 'is-gc', label: 'the letter is G or C', expr: (n, i) => `${n.T}[${i}] == 'G' OR ${n.T}[${i}] == 'C'`, words: () => 'the letter is G or C' }, + { id: 'is-unknown', label: 'the letter is N, an unknown base', expr: (n, i) => `${n.T}[${i}] == 'N'`, words: () => 'the letter is N' }, + { id: 'custom', label: 'something else (leave a blank)', expr: () => '', words: () => 'the test you fill in' }, + ], + windows: [ + { id: 'window-is', label: 'the window is a given word W', needs: ['W'], expr: (n, i) => `SLICE(${n.T}, ${i}, ${i} + ${n.K}) == ${n.W}`, words: (n) => `the window spells ${n.W}` }, + { id: 'window-gc', label: 'the window is mostly G and C', expr: (n, i) => `GC_COUNT(SLICE(${n.T}, ${i}, ${i} + ${n.K})) * 2 > ${n.K}`, words: () => 'more than half the window is G or C', helper: 'gc' }, + { id: 'custom', label: 'something else (leave a blank)', expr: () => '', words: () => 'the test you fill in' }, + ], + starts: [ + { id: 'pattern-here', label: 'the pattern matches exactly here', expr: null, words: (n) => `${n.P} occurs exactly at this start`, inner: true }, + { id: 'custom', label: 'something else (leave a blank)', expr: () => '', words: () => 'the test you fill in' }, + ], + codons: [ + { id: 'is-stop', label: 'the codon is a stop codon', expr: (n, i) => `SLICE(${n.T}, ${i}, ${i} + 3) IN Stops`, words: () => 'the codon is TAA, TAG, or TGA', define: 'stops' }, + { id: 'is-start', label: 'the codon is ATG', expr: (n, i) => `SLICE(${n.T}, ${i}, ${i} + 3) == "ATG"`, words: () => 'the codon is ATG' }, + { id: 'custom', label: 'something else (leave a blank)', expr: () => '', words: () => 'the test you fill in' }, + ], + pairs: [ + { id: 'same-letter', label: 'the two letters are the same', expr: (n) => `${n.T}[I] == ${n.T}[J]`, words: () => 'the letters at I and J are the same' }, + { id: 'custom', label: 'something else (leave a blank)', expr: () => '', words: () => 'the test you fill in' }, + ], + paired: [ + { id: 'differ', label: 'the two letters differ', expr: (n, i) => `${n.A}[${i}] != ${n.B}[${i}]`, words: () => 'the letters differ' }, + { id: 'agree', label: 'the two letters agree', expr: (n, i) => `${n.A}[${i}] == ${n.B}[${i}]`, words: () => 'the letters agree' }, + { id: 'custom', label: 'something else (leave a blank)', expr: () => '', words: () => 'the test you fill in' }, + ], + values: [ + { id: 'at-least', label: 'the value is at least a threshold', needs: ['Threshold'], expr: (n, i, x) => `${x} >= ${n.Threshold}`, words: (n) => `the value is at least ${n.Threshold}` }, + { id: 'custom', label: 'something else (leave a blank)', expr: () => '', words: () => 'the test you fill in' }, + ], +}; + +/** Which answers each walk can give, since not every combination makes sense. */ +export const COMPATIBLE = { + positions: ['count', 'total', 'all', 'first', 'any', 'every', 'best', 'filter', 'tally', 'build'], + windows: ['count', 'all', 'first', 'any', 'every', 'tally'], + starts: ['count', 'all', 'first', 'any'], + codons: ['count', 'all', 'first', 'any', 'every', 'tally'], + pairs: ['count', 'any'], + paired: ['count', 'all', 'first', 'any', 'every'], + values: ['count', 'total', 'first', 'any', 'every', 'best', 'filter', 'tally'], +}; + +const DEFAULT_NAMES = { T: 'T', B: 'B', K: 'K', P: 'P', A: 'A', W: 'W', L: 'Scores', Threshold: 'Threshold', fn: null }; + +/* ------------------------------------------------------------------ */ +/* Building a plan */ +/* ------------------------------------------------------------------ */ + +export function buildPlan(choice) { + const answer = ANSWERS[choice.answer]; + const walkId = choice.walk; + const walk = WALKS[walkId]; + const tests = TESTS[walkId] || []; + const test = tests.find((t) => t.id === choice.test) || tests[0]; + const n = { ...DEFAULT_NAMES, ...(choice.names || {}) }; + const withSpec = choice.contracts !== false; + const withInvariant = Boolean(choice.invariant); + const warnings = []; + if (!answer || !walk) return { source: '', notes: [], spec: '', tests: [], warnings: ['Choose an answer and a walk.'] }; + if (!COMPATIBLE[walkId].includes(choice.answer)) warnings.push(`${walk.label} and "${answer.label.toLowerCase()}" do not fit together well. The plan below does its best; consider another walk.`); + + const lines = []; + const notes = []; + const add = (depth, text, job, why) => { + lines.push(`${' '.repeat(depth)}${text}`); + if (why) notes.push({ line: lines.length, job, text: why }); + }; + const blank = () => lines.push(''); + + /* ---------- parameters and the loop's shape ---------- */ + const params = []; + const seqParam = (name) => params.push(`${name} : STRING`); + let dom; let visit; let I = 'I'; let current; let lengthName = 'N'; let elemType = 'CHAR'; + const extraSetup = []; + const pre = []; + switch (walkId) { + case 'positions': + seqParam(n.T); + dom = `[0, LENGTH(${n.T}))`; + visit = { head: `FOR I <- 0 TO N`, tail: 'NEXT I', why: `I takes every position of ${n.T}, from 0 up to but not including N, its length: the half-open range [0, N). The bound is the length itself, because FOR stops one short of it.` }; + current = `${n.T}[I]`; + extraSetup.push(['N <- LENGTH(' + n.T + ')', `Measure ${n.T} once, into a name, since its length does not change during the scan.`]); + break; + case 'windows': + seqParam(n.T); params.push(`${n.K} : INTEGER`); + pre.push([`REQUIRE ${n.K} >= 1`, `A window has at least one letter. A REQUIRE is the caller's promise, and the checker holds every call to it.`]); + dom = `[0, LENGTH(${n.T}) - ${n.K} + 1)`; + visit = { head: `FOR I <- 0 TO N - ${n.K} + 1`, tail: 'NEXT I', why: `A window starting at I covers [I, I + ${n.K}), so its last letter is at I + ${n.K} - 1, which has to be a real position: I <= N - ${n.K}. The legal starts are [0, N - ${n.K} + 1), and the bound is N - ${n.K} + 1 because FOR stops one short.` }; + current = `SLICE(${n.T}, I, I + ${n.K})`; + elemType = 'STRING'; + extraSetup.push([`N <- LENGTH(${n.T})`, `Measure ${n.T} once.`]); + break; + case 'starts': + seqParam(n.P); seqParam(n.T); + pre.push([`REQUIRE LENGTH(${n.P}) >= 1`, 'An empty pattern would match everywhere, which is rarely what anyone means, so rule it out.']); + dom = `[0, LENGTH(${n.T}) - LENGTH(${n.P}) + 1)`; + visit = { head: 'FOR I <- 0 TO N - M + 1', tail: 'NEXT I', why: 'A pattern starting at I occupies I through I + M - 1, and that last position has to exist, so I <= N - M. The legal starts are [0, N - M + 1).' }; + current = null; + elemType = 'STRING'; + extraSetup.push([`N <- LENGTH(${n.T})`, `Measure ${n.T} once.`], [`M <- LENGTH(${n.P})`, `Measure ${n.P} once.`]); + break; + case 'codons': + seqParam(n.T); + dom = `[0, LENGTH(${n.T}) DIV 3 * 3)`; + visit = { head: 'FOR I <- 0 TO N DIV 3 * 3 STEP 3', tail: 'NEXT I', why: `STEP 3 visits codon starts: 0, 3, 6, and so on. The bound N DIV 3 * 3 is the length of the longest prefix made of whole codons, so I + 2 always exists. N MOD 3 letters at the end are left over.` }; + current = `SLICE(${n.T}, I, I + 3)`; + elemType = 'STRING'; + extraSetup.push([`N <- LENGTH(${n.T})`, `Measure ${n.T} once.`]); + break; + case 'pairs': + seqParam(n.T); + dom = null; + visit = { head: 'FOR I <- 0 TO N', tail: 'NEXT I', inner: { head: 'FOR J <- I + 1 TO N', tail: 'NEXT J' }, why: 'Two loops: I takes every position, and J every position after it, so each pair I < J is visited exactly once. The work is about N * N / 2 tests.' }; + current = null; + extraSetup.push([`N <- LENGTH(${n.T})`, `Measure ${n.T} once.`]); + break; + case 'paired': + seqParam(n.A); seqParam(n.B); + pre.push([`REQUIRE LENGTH(${n.A}) == LENGTH(${n.B})`, 'Walking two sequences in step only makes sense when they are the same length. The REQUIRE says so, and the checker holds every caller to it.']); + dom = `[0, LENGTH(${n.A}))`; + visit = { head: 'FOR I <- 0 TO N', tail: 'NEXT I', why: `One position I, used for both sequences: ${n.A}[I] and ${n.B}[I] sit side by side.` }; + current = null; + extraSetup.push([`N <- LENGTH(${n.A})`, `Measure once; both sequences share the length.`]); + break; + case 'values': + params.push(`${n.L} : LIST OF INTEGER`); + dom = `[0, LENGTH(${n.L}))`; + visit = { head: `FOR I <- 0 TO N`, tail: 'NEXT I', why: `I takes every position of ${n.L}.` }; + current = `${n.L}[I]`; + elemType = 'INTEGER'; + extraSetup.push([`N <- LENGTH(${n.L})`, `Measure ${n.L} once.`]); + break; + default: break; + } + if (test?.needs) for (const x of test.needs) if (x === 'B') params.push(`${n.B} : CHAR`); else if (x === 'W') params.push(`${n.W} : STRING`); else if (x === 'Threshold') params.push(`${n.Threshold} : INTEGER`); + + // the test itself + let testExpr = test && test.expr ? test.expr(n, 'I', current) : null; + const bvName = walkId === 'windows' ? 'S' : 'K'; + let specTest = testExpr ? test.expr(n, bvName, current ? current.replace(/\bI\b/g, bvName) : null) : null; + const innerMatch = walkId === 'starts' && test?.inner; + if (innerMatch) specTest = `FORALL J IN [0, LENGTH(${n.P})) : ${n.T}[K + J] == ${n.P}[J]`; + const codonTest = specTest; + if (specTest && walkId === 'codons') specTest = choice.answer === 'every' ? `${bvName} MOD 3 != 0 OR ${codonTest}` : `${bvName} MOD 3 == 0 AND ${codonTest}`; + if (!testExpr && !innerMatch) { testExpr = ''; specTest = null; } + const usesBlank = testExpr === ''; + + /* ---------- the answer ---------- */ + const fnName = choice.names?.fn || defaultName(choice.answer, walkId, test); + const kind = choice.answer; + let returns = answer.returns; + if (kind === 'filter') returns = `LIST OF ${elemType}`; + if (kind === 'tally') returns = `HASHMAP OF ${elemType} : INTEGER`; + + const acc = { count: 'Total', total: 'Total', all: 'Answer', first: 'I', any: 'Found', every: 'AllPass', best: 'Best', filter: 'Kept', tally: 'Tally', build: 'Answer' }[kind]; + const accType = { count: 'INTEGER', total: 'INTEGER', all: 'LIST OF INTEGER', any: 'BOOL', every: 'BOOL', best: 'INTEGER', filter: `LIST OF ${elemType}`, tally: `HASHMAP OF ${elemType} : INTEGER`, build: 'STRING' }[kind]; + + // specification + const ensures = []; + let specWords = ''; + if (dom && specTest) { + const K = bvName; + const S = `{ ${K} IN ${dom} : ${specTest} }`; + switch (kind) { + case 'count': ensures.push(`RESULT == LENGTH(${S})`); specWords = `the number of positions K in ${dom} where ${test.words(n)}`; break; + case 'all': ensures.push(`RESULT == SORTED(${S})`); specWords = `every position K in ${dom} where ${test.words(n)}, smallest first`; break; + case 'any': ensures.push(`RESULT == (EXISTS ${K} IN ${dom} : ${specTest})`); specWords = `TRUE exactly when some position in ${dom} passes`; break; + case 'every': ensures.push(`RESULT == (FORALL ${K} IN ${dom} : ${specTest})`); specWords = `TRUE exactly when every position in ${dom} passes`; break; + case 'first': { + const hi = dom.replace(/^\[0, /, '').replace(/\)$/, ''); + if (walkId === 'codons') ensures.push(`RESULT MOD 3 == 0`); + const top = walkId === 'windows' || walkId === 'starts' ? `MAX(0, ${hi})` : hi; + ensures.push(`0 <= RESULT AND RESULT <= ${top}`, `FORALL ${K} IN [0, RESULT) : NOT (${specTest})`); + specWords = `the first position that passes, or ${hi} when none does`; + break; + } + default: break; + } + } + if (kind === 'build' && walkId === 'positions') { ensures.push(`LENGTH(RESULT) == LENGTH(${n.T})`); specWords = `a new sequence as long as ${n.T}`; } + if (kind === 'best') { pre.push([`REQUIRE LENGTH(${walkId === 'values' ? n.L : n.T}) >= 1`, 'An empty input has no best position, so the caller has to promise at least one value.']); } + + /* ---------- write it ---------- */ + add(0, `// ${sentence(fnName, answer, walk, test, n)}`, null, null); + add(0, `FUNCTION ${fnName}(${params.join(', ')}) RETURNS ${returns}`, null, `The name says what the answer is; the parameters are everything the procedure needs; RETURNS ${returns} promises ${answer.words}.`); + for (const [p, why] of pre) add(1, p, 'spec', why); + if (withSpec) for (const e of ensures) add(1, `ENSURE ${e}`, 'spec', `The specification: the answer is ${specWords}. The checker tries to prove it, and the lab tests it on every run.`); + if (withSpec && usesBlank && !ensures.length) add(1, '// ENSURE ', 'spec', 'Once the test is filled in, state what the answer is as an ENSURE.'); + blank(); + add(1, '// Set up', 'setup', null); + const decls = []; + if (extraSetup.some(([l]) => l.startsWith('N <-'))) decls.push('N : INTEGER'); + if (extraSetup.some(([l]) => l.startsWith('M <-'))) decls.push('M : INTEGER'); + if (kind !== 'first') decls.push(`${acc} : ${accType}`); + if (kind === 'first') decls.push('I : INTEGER'); + if (innerMatch) decls.push('Matched : BOOL'); + if (kind === 'tally') decls.push(`Key : ${elemType}`); + for (const d of decls) add(1, `DECLARE ${d}`, 'setup', null); + for (const [l, why] of extraSetup) add(1, l, 'setup', why); + const init = { + count: [`${acc} <- 0`, `${acc} is an accumulator for the whole scan, so it starts at 0 here, before the loop, and never inside it. 0 is also the right answer for an input with no matches.`], + total: [`${acc} <- 0`, 'The running total starts at 0, before the loop.'], + all: [null, `A LIST starts out empty, which is already the right answer when nothing matches.`], + any: [`${acc} <- FALSE`, 'Nothing has been found yet, so the flag starts FALSE.'], + every: [`${acc} <- TRUE`, 'Assume every position passes until one is found that does not.'], + best: [`${acc} <- 0`, 'Take the first position as the best so far, rather than inventing a starting value that might beat every real one.'], + filter: [null, 'The kept values start as an empty LIST; the original is left alone.'], + tally: [null, 'A HASHMAP starts out empty; each value gets its entry the first time it is seen.'], + build: [`${acc} <- ""`, 'The new sequence starts empty and grows by one letter per position.'], + first: ['I <- 0', 'A WHILE loop has no header to start its counter, so the line before it does.'], + }[kind]; + if (init[0]) add(1, init[0], 'setup', init[1]); else notes.push({ line: lines.length, job: 'setup', text: init[1] }); + if (test?.define === 'stops') { /* defined at the top level */ } + blank(); + + /* ---------- the loop ---------- */ + const visitWords = walk.label.toLowerCase(); + if (kind === 'first') { + const hi = walkId === 'starts' ? 'N - M + 1' : walkId === 'windows' ? `N - ${n.K} + 1` : walkId === 'codons' ? 'N DIV 3 * 3' : 'N'; + const stride = walkId === 'codons' ? 3 : 1; + add(1, `// Visit ${visitWords}, stopping at the first that passes`, 'visit', null); + const cond = innerMatch ? 'NOT MATCHES_AT(I)' : `NOT (${testExpr})`; + add(1, `WHILE I < ${hi} AND ${innerMatch ? `` : cond}`, 'visit', `The guard I < ${hi} comes first, because AND tests its left side first and skips the right once the answer is settled: the letter at I is read only while I is a real position.`); + if (withInvariant && specTest) add(2, `INVARIANT FORALL ${bvName} IN [0, I) : NOT (${specTest})`, 'visit', 'Every position before I has already failed the test. That is what makes I the first one when the loop stops.'); + add(2, `I <- I + ${stride}`, 'update', `Move on. This line is the only reason the loop ever ends: I grows by ${stride === 1 ? 'one' : 'three, one codon,'} each pass, and the guard fails after at most ${hi} passes.`); + add(1, 'ENDWHILE', 'visit', null); + blank(); + add(1, '// Report', 'report', null); + add(1, 'RETURN I', 'report', `I holds the first passing position, or ${hi} when there was none. The caller tells the two apart by comparing with ${hi}.`); + add(0, 'ENDFUNCTION', null, null); + } else { + add(1, `// Visit ${visitWords}`, 'visit', null); + add(1, visit.head, 'visit', visit.why); + let d = 2; + if (withInvariant && specTest && ['count', 'all', 'any', 'every'].includes(kind) && walkId !== 'pairs') { + const inv = { + count: `${acc} == LENGTH({ ${bvName} IN [0, I) : ${specTest} })`, + all: `${acc} == SORTED({ ${bvName} IN [0, I) : ${specTest} })`, + any: `${acc} == (EXISTS ${bvName} IN [0, I) : ${specTest})`, + every: `${acc} == (FORALL ${bvName} IN [0, I) : ${specTest})`, + }[kind]; + add(2, `INVARIANT ${inv}`, 'visit', `What is true at the top of every pass: ${acc} already holds the answer for the positions before I. When the loop ends with I at the bound, this becomes the ENSURE.`); + } + if (visit.inner) { add(2, visit.inner.head, 'visit', null); d = 3; } + if (innerMatch) { + add(2, '// Test: does the pattern match at I?', 'test', null); + add(2, 'Matched <- TRUE', 'test', 'Matched belongs to one pass of the outer loop, so it is reset here, inside the loop, before each candidate start. Setting it once before the loop is the classic mistake.'); + add(2, 'FOR J <- 0 TO M', 'test', 'J walks the positions of the pattern, [0, M).'); + if (withInvariant) add(3, `INVARIANT Matched == (FORALL K IN [0, J) : ${n.T}[I + K] == ${n.P}[K])`, 'test', 'So far, Matched says whether every letter checked so far agreed.'); + add(3, `IF ${n.T}[I + J] != ${n.P}[J] THEN`, 'test', 'One disagreeing letter settles it.'); + add(4, 'Matched <- FALSE', 'test', null); + add(4, 'BREAK', 'test', 'The answer for this start is settled, so stop comparing. BREAK changes how much work is done, never the answer.'); + add(3, 'ENDIF', 'test', null); + add(2, 'NEXT J', 'test', null); + testExpr = 'Matched'; + } else { + add(d, `// Test: ${test ? test.words(n) : 'the test'}`, 'test', null); + } + const upd = { + count: [`${acc} <- ${acc} + 1`, 'Update: one more position passed.'], + total: [`${acc} <- ${acc} + ${current && elemType === 'INTEGER' ? current : ''}`, 'Update: add this position\'s value.'], + all: [`APPEND(${acc}, I)`, 'Update: record where it happened. The list grows by one.'], + any: [`${acc} <- TRUE`, 'Update: one is enough.'], + every: [`${acc} <- FALSE`, 'Update: one failure is enough to settle the answer.'], + best: [`${acc} <- I`, 'Update: this position beats the best so far.'], + filter: [`APPEND(${acc}, ${current})`, 'Update: keep this value.'], + build: [`${acc} <- ${acc} & `, `Update: add one letter, worked out from ${current}, to the end of the new sequence.`], + tally: null, + }[kind]; + if (kind === 'tally') { + add(d, `Key <- ${current}`, 'test', 'Name the value being counted.'); + add(d, `IF Key IN ${acc} THEN`, 'test', `A key that is not in the HASHMAP yet has no old count to add to, so test with IN first.`); + add(d + 1, `${acc}[Key] <- ${acc}[Key] + 1`, 'update', 'Seen before: one more.'); + add(d, 'ELSE', 'test', null); + add(d + 1, `${acc}[Key] <- 1`, 'update', 'The first sighting of this value.'); + add(d, 'ENDIF', 'test', null); + } else if (kind === 'build') { + add(d, upd[0], 'update', upd[1]); + } else if (kind === 'best') { + add(d, `IF ${current || ''} > ${walkId === 'values' ? `${n.L}[${acc}]` : `${n.T}[${acc}]`} THEN`, 'test', 'Test: does this position beat the best so far? Strictly greater keeps the first of several equal bests.'); + add(d + 1, upd[0], 'update', upd[1]); + add(d, 'ENDIF', 'test', null); + } else if (kind === 'every') { + add(d, `IF NOT (${testExpr}) THEN`, 'test', 'Test: look for evidence against the claim.'); + add(d + 1, upd[0], 'update', upd[1]); + add(d + 1, 'BREAK', 'update', 'The answer cannot change now, so stop.'); + add(d, 'ENDIF', 'test', null); + } else { + add(d, `IF ${testExpr} THEN`, innerMatch ? 'update' : 'test', innerMatch ? 'Update: act on the answer the inner loop settled.' : `Test: ${test ? test.words(n) : 'the condition you choose'}.`); + add(d + 1, upd[0], 'update', upd[1]); + if (kind === 'any') add(d + 1, 'BREAK', 'update', 'The answer cannot change now, so stop.'); + add(d, 'ENDIF', innerMatch ? 'update' : 'test', null); + } + if (visit.inner) add(2, visit.inner.tail, 'visit', null); + add(1, visit.tail, 'visit', 'NEXT moves the counter on and goes back to the top.'); + blank(); + add(1, '// Report', 'report', null); + add(1, `RETURN ${acc}`, 'report', `After the loop, ${acc} holds the answer for every position.`); + add(0, 'ENDFUNCTION', null, null); + } + + // helpers the test uses + const helpers = []; + if (test?.define === 'stops') helpers.unshift('TYPE CodonSet = SET OF STRING', 'DEFINE Stops ("TAA", "TAG", "TGA") : CodonSet', ''); + if (test?.helper === 'gc') helpers.push('', '// How many letters of S are G or C.', 'FUNCTION GC_COUNT(S : STRING) RETURNS INTEGER', ' DECLARE Total : INTEGER', ' Total <- 0', ' FOR EACH X IN S', " IF X == 'G' OR X == 'C' THEN", ' Total <- Total + 1', ' ENDIF', ' NEXT X', ' RETURN Total', 'ENDFUNCTION'); + let source = lines.join('\n'); + let shift = 0; + if (helpers.length && test?.define === 'stops') { source = `${helpers.join('\n')}\n${source}`; shift = helpers.length; } + else if (helpers.length) source = `${source}\n${helpers.join('\n')}`; + const shiftedNotes = notes.map((x) => ({ ...x, line: x.line + shift })); + + if (kind === 'first' && innerMatch) warnings.push('Finding the first match of a pattern needs the match test inside the WHILE condition; the plan leaves it as a blank. Writing it as a FUNCTION MATCHES_AT(P, T, I) RETURNS BOOL keeps the WHILE line readable.'); + if (kind === 'build') warnings.push('What each letter becomes is yours to decide: a CASE ... ENDCASE on the letter is the usual way, as in the complement table in the guide.'); + if (usesBlank) warnings.push('The test is left as : fill it in, and the lab will run and check the rest.'); + + const exampleArgs = exampleCall(fnName, walkId, test, n); + return { + source, + notes: shiftedNotes, + spec: specWords ? `${fnName} returns ${specWords}.` : '', + ensures, + tests: exampleArgs.map((a) => `${a} -> ?`), + warnings, + name: fnName, + pattern: answer.job, + }; +} + +function defaultName(answer, walk, test) { + const what = { + count: 'COUNT', total: 'TOTAL', all: 'FIND_ALL', first: 'FIND_FIRST', any: 'ANY', every: 'ALL', best: 'BEST_POSITION', filter: 'KEEP', tally: 'TALLY', build: 'BUILD', + }[answer]; + const of = { custom: { tally: 'VALUES', total: 'VALUES' }[answer] || 'PASSING', 'is-base': 'BASE', 'is-gc': 'GC', 'is-unknown': 'N', 'window-is': 'WORD', 'window-gc': 'GC_WINDOWS', 'pattern-here': 'MATCHES', 'is-stop': 'STOPS', 'is-start': 'STARTS', 'same-letter': 'PAIRS', differ: 'MISMATCHES', agree: 'AGREEMENTS', 'at-least': 'ABOVE' }[test?.id]; + return of ? `${what}_${of}` : what; +} + +function sentence(fn, answer, walk, test, n) { + const t = test ? test.words(n) : 'the test passes'; + return `${answer.label} (${t}), visiting ${walk.label.toLowerCase()}.`; +} + +function exampleCall(fn, walk, test, n) { + const T = '"GATTACA"'; + const b = test?.needs?.includes('B') ? ", 'A'" : ''; + const w = test?.needs?.includes('W') ? ', "TA"' : ''; + const th = test?.needs?.includes('Threshold') ? ', 20' : ''; + switch (walk) { + case 'positions': return [`${fn}(${T}${b})`, `${fn}(""${b})`]; + case 'windows': return [`${fn}(${T}, 2${w})`, `${fn}("GA", 3${w})`]; + case 'starts': return [`${fn}("A", ${T})`, `${fn}("ATT", ${T})`, `${fn}("GATTACAT", ${T})`]; + case 'codons': return [`${fn}("ATGTAAGG")`, `${fn}("AT")`]; + case 'pairs': return [`${fn}(${T})`]; + case 'paired': return [`${fn}(${T}, "GACTACA")`, `${fn}("", "")`]; + case 'values': return [`${fn}([12, 30, 25, 8]${th})`, `${fn}([]${th})`]; + default: return []; + } +} + +/* ------------------------------------------------------------------ */ +/* The four jobs in a student's program */ +/* ------------------------------------------------------------------ */ + +const JOB_WORDS = { + setup: 'Set up: measure the inputs and create whatever holds the answer.', + visit: 'Visit: move a position through the legal positions.', + test: 'Test: decide something about the current position.', + update: 'Update: change the answer when the test says so.', + report: 'Report: hand the answer back.', +}; + +/** + * Which of the four jobs, plus reporting, a subroutine does, where, and + * what the coach found wrong with each. The status of a job is 'ok', + * 'missing', or 'problem'. + */ +export function jobsReport(unit, diags = []) { + if (!unit) return []; + const byJob = { setup: [], visit: [], test: [], update: [], report: [] }; + for (const [line, r] of unit.roles || new Map()) if (byJob[r.role]) byJob[r.role].push(line); + for (const k of Object.keys(byJob)) byJob[k].sort((a, b) => a - b); + const inUnit = (d) => (unit.node ? d.line >= unit.node.line && d.line <= (unit.node.endLine ?? Infinity) : true); + const probs = diags.filter((d) => d.severity !== 'note' && inUnit(d)); + const jobOf = (d) => { + if (['unset-read', 'reset-outside', 'reset-inside', 'maybe-unset'].includes(d.code)) return 'setup'; + if (['stops-early', 'starts-at-one', 'step-overrun', 'no-progress', 'growing-bound'].includes(d.code)) return 'visit'; + if (['guard-order', 'char-string', 'float-equal', 'missing-key', 'case-gap'].includes(d.code)) return 'test'; + if (['missing-return', 'return-value'].includes(d.code)) return 'report'; + const r = unit.roles?.get(d.line); + return r?.role || null; + }; + const hasLoop = (unit.loops || []).length > 0; + const accs = (unit.accumulators || []).filter((a) => a.how !== 'flag'); + const out = []; + for (const job of ['setup', 'visit', 'test', 'update', 'report']) { + const lines = byJob[job]; + const issues = probs.filter((d) => jobOf(d) === job); + let status = issues.length ? 'problem' : lines.length ? 'ok' : 'missing'; + let message = JOB_WORDS[job]; + if (status === 'missing') { + if (job === 'visit' && !hasLoop) message = 'Nothing visits the positions: there is no loop. A scan needs a FOR loop over the positions, or a WHILE that moves a position forward.'; + else if (job === 'setup') message = accs.length ? `Nothing gives ${accs[0].name} a starting value before the loop.` : 'Nothing measures the inputs or creates a place for the answer.'; + else if (job === 'update') message = hasLoop ? 'Nothing inside the loop changes the answer, so every pass is wasted.' : JOB_WORDS.update; + else if (job === 'test') message = hasLoop ? 'Nothing inside the loop decides anything about the current position. That is fine for a total over every position, and a sign something is missing for anything else.' : JOB_WORDS.test; + else if (job === 'report') message = unit.kind === 'Function' ? 'The function never returns its answer.' : 'Nothing shows or returns the answer.'; + if (job === 'test' && ['sum', 'build'].includes(unit.pattern)) status = 'ok'; + } + out.push({ job, status, lines, message, issues }); + } + return out; +} + +export { PATTERN_NAMES, JOB_WORDS }; diff --git a/src/engines/pseudo/planner.js b/src/engines/pseudo/planner.js new file mode 100644 index 0000000..93ed369 --- /dev/null +++ b/src/engines/pseudo/planner.js @@ -0,0 +1,143 @@ +/** + * pseudo/planner.js + * + * From an idea to pseudocode, on a page. Three questions, each answered by + * choosing a card; the pseudocode appears underneath, every line with a + * note saying which job it does and why it is written the way it is. + * + * planner({ answer, walk, test, labUrl }) -> node + */ + +import { el, add, clear, button } from './dom.js'; +import { lineHtml } from './highlight.js'; +import { ANSWERS, WALKS, TESTS, COMPATIBLE, buildPlan } from './plan.js'; +import { study } from './study.js'; +import { verify } from './vcgen.js'; + +const NAME_FIELDS = { + T: 'the sequence', B: 'the base', K: 'the window length', P: 'the pattern', A: 'the first sequence', W: 'the word', L: 'the list', Threshold: 'the threshold', +}; + +function encode(s) { + const bytes = new TextEncoder().encode(JSON.stringify(s)); + let bin = ''; for (const b of bytes) bin += String.fromCharCode(b); + return btoa(bin).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} + +export function planner(options = {}) { + const opts = { labUrl: '../lab/', ...options }; + const root = el('div', { class: 'pl pl-planner' }); + const choice = { answer: opts.answer || 'count', walk: opts.walk || 'positions', test: opts.test || 'is-base', contracts: true, invariant: false, names: {} }; + + const q1 = el('div', { class: 'pl-choices', role: 'group', 'aria-label': 'What do you want back?' }); + const q2 = el('div', { class: 'pl-choices', role: 'group', 'aria-label': 'What do you walk over?' }); + const q3 = el('div', { class: 'pl-choices', role: 'group', 'aria-label': 'What do you ask at each step?' }); + const names = el('div', { class: 'pl-plan-names' }); + const specBox = el('input', { type: 'checkbox' }); specBox.checked = true; + const invBox = el('input', { type: 'checkbox' }); + const outEl = el('div', { class: 'pl-plan-out' }); + + const card = (label, sub, pressed, disabled, onclick) => { + const b = el('button', { type: 'button', class: 'pl-choice', 'aria-pressed': pressed ? 'true' : 'false', 'aria-disabled': disabled ? 'true' : 'false' }, [el('strong', { text: label }), el('span', { text: sub })]); + b.addEventListener('click', () => { if (!disabled) onclick(); }); + return b; + }; + + function drawQuestions() { + clear(q1); clear(q2); clear(q3); clear(names); + for (const [id, a] of Object.entries(ANSWERS)) { + add(q1, card(a.label, `for example, ${a.example}`, choice.answer === id, false, () => { + choice.answer = id; + if (!COMPATIBLE[choice.walk].includes(id)) choice.walk = Object.keys(COMPATIBLE).find((w) => COMPATIBLE[w].includes(id)); + fixTest(); draw(); + })); + } + for (const [id, w] of Object.entries(WALKS)) { + const ok = COMPATIBLE[id].includes(choice.answer); + add(q2, card(w.label, ok ? `for example, ${w.example}` : 'does not fit this answer', choice.walk === id, !ok, () => { choice.walk = id; fixTest(); draw(); })); + } + for (const t of TESTS[choice.walk]) { + add(q3, card(t.label.replace(/^./, (c) => c.toUpperCase()), t.id === 'custom' ? 'the plan leaves for you to write' : 'the plan writes the comparison', choice.test === t.id, false, () => { choice.test = t.id; draw(); })); + } + const needed = new Set([...(WALKS[choice.walk].needs || []), ...(TESTS[choice.walk].find((t) => t.id === choice.test)?.needs || [])]); + for (const n of needed) { + const input = el('input', { class: 'pl-input', type: 'text', value: choice.names[n] || n, 'aria-label': NAME_FIELDS[n] || n, spellcheck: 'false' }); + input.addEventListener('change', () => { const v = input.value.trim(); if (/^[A-Za-z][A-Za-z0-9_]*$/.test(v)) { choice.names[n] = v; drawOutput(); } else input.value = choice.names[n] || n; }); + add(names, el('label', {}, [`Name for ${NAME_FIELDS[n] || n}:`, input])); + } + const fn = el('input', { class: 'pl-input', type: 'text', value: choice.names.fn || '', placeholder: 'chosen for you', 'aria-label': 'function name', spellcheck: 'false' }); + fn.addEventListener('change', () => { const v = fn.value.trim().toUpperCase(); if (!v || /^[A-Z][A-Z0-9_]*$/.test(v)) { choice.names.fn = v || null; fn.value = v; drawOutput(); } }); + add(names, el('label', {}, ['Function name:', fn])); + } + function fixTest() { if (!TESTS[choice.walk].some((t) => t.id === choice.test)) choice.test = TESTS[choice.walk][0].id; } + + function drawOutput() { + clear(outEl); + choice.contracts = specBox.checked; + choice.invariant = invBox.checked; + const plan = buildPlan(choice); + const lines = plan.source.split('\n'); + const noteAt = new Map(plan.notes.map((n) => [n.line, n])); + const jobOf = []; + let job = null; + lines.forEach((l, i) => { + const n = noteAt.get(i + 1); + const marker = /^\s*\/\/\s*(Set up|Visit|Test|Update|Report)/.exec(l); + if (marker) job = { 'Set up': 'setup', Visit: 'visit', Test: 'test', Update: 'update', Report: 'report' }[marker[1]]; + if (/^\s*(REQUIRE|ENSURE)/.test(l)) jobOf[i] = 'spec'; + else if (n?.job) jobOf[i] = n.job; + else jobOf[i] = /^\s*$|^FUNCTION|^ENDFUNCTION|^\/\//.test(l) ? null : job; + }); + const JOB = { setup: 'Set up', visit: 'Visit', test: 'Test', update: 'Update', report: 'Report', spec: 'Specification' }; + const annot = el('div', { class: 'pl-plan-annot', role: 'table', 'aria-label': 'the plan, line by line' }); + lines.forEach((l, i) => { + const n = noteAt.get(i + 1); + const j = jobOf[i]; + // A line too long to share its row with a note gets the note underneath. + const long = !n || l.length > 36; + add(annot, el('div', { class: `pl-plan-row${j ? ` role-${j}` : ''}${long ? ' is-long' : ''}`, role: 'row' }, [ + el('span', { class: 'pl-code-num', text: String(i + 1) }), + el('code', { html: lineHtml(l) || ' ' }), + el('span', { class: 'pl-plan-note' }, n ? [j ? el('b', { text: `${JOB[j]}. ` }) : null, j ? n.text.replace(/^(Test|Update|Set up|Visit|Report|The specification): ?/, '').replace(/^./, (c) => c.toUpperCase()) : n.text] : []), + ])); + }); + // what the checker makes of it + const s = study(plan.source, { coach: false }); + let verdict = ''; + if (s.ok) { + const v = verify(s.program); + verdict = v.counts.total ? `The checker proves ${v.counts.proved} of ${v.counts.total} claims about this plan${v.counts.unknown ? (invBox.checked || !specBox.checked ? '; open the Prove tab in the lab to see the rest' : '. Tick "Write the loop invariant too" and it has what it needs to prove the ENSURE') : ', including its ENSURE'}.` : ''; + } else if (s.blanks.length) verdict = `${s.blanks.length === 1 ? 'One blank is' : `${s.blanks.length} blanks are`} left for you: ${s.blanks.map((b) => `<${b.text}> on line ${b.line}`).join(', ')}.`; + const call = plan.tests[0]?.split('->')[0].trim(); + const labState = { code: plan.source, call: s.blanks.length ? '' : call, tests: plan.tests.join('\n'), viewSet: 'step through', title: plan.name }; + const toLab = el('a', { class: 'pl-btn pl-btn-primary', href: `${opts.labUrl}#lab=${encode(labState)}`, text: 'Open this plan in the lab' }); + const copy = button('Copy the pseudocode', async () => { try { await navigator.clipboard.writeText(plan.source); copy.textContent = 'Copied'; } catch { copy.textContent = 'Select the code and copy it'; } }); + add(outEl, [ + plan.spec ? el('div', { class: 'pl-plan-spec' }, [el('strong', { text: 'The specification: ' }), plan.spec, ...(plan.ensures.length && specBox.checked ? [el('br'), el('span', { class: 'pl-muted', text: 'In notation: ' }), ...plan.ensures.map((e) => el('code', { html: lineHtml(`ENSURE ${e}`) }))] : [])]) : null, + ...plan.warnings.map((w) => el('p', { class: 'pl-warn-line', text: w })), + annot, + verdict ? el('p', { class: 'pl-muted', text: verdict }) : null, + el('div', {}, [ + el('p', { class: 'pl-plan-q', text: 'Predict before you run' }), + el('p', { class: 'pl-muted', text: 'Work out what each call should give, by hand, before the lab tells you. The ? marks the answer you are predicting.' }), + el('pre', { class: 'pl-console', text: plan.tests.join('\n') }), + ]), + el('div', { class: 'pl-row' }, [toLab, copy]), + ]); + } + + function draw() { drawQuestions(); drawOutput(); } + specBox.addEventListener('change', drawOutput); + invBox.addEventListener('change', drawOutput); + + add(root, [ + el('p', { class: 'pl-plan-q' }, ['1. What do you want back? ', el('span', { class: 'pl-muted', text: 'The answer decides what the scan collects.' })]), q1, + el('p', { class: 'pl-plan-q' }, ['2. What do you walk over? ', el('span', { class: 'pl-muted', text: 'The walk decides the loop and its bound.' })]), q2, + el('p', { class: 'pl-plan-q' }, ['3. What do you ask at each step? ', el('span', { class: 'pl-muted', text: 'The test decides which positions count.' })]), q3, + el('p', { class: 'pl-plan-q', text: '4. Names and extras' }), names, + el('div', { class: 'pl-plan-names' }, [el('label', {}, [specBox, 'Write the specification as ENSURE']), el('label', {}, [invBox, 'Write the loop invariant too'])]), + outEl, + ]); + draw(); + return root; +} diff --git a/src/engines/pseudo/sharpen.js b/src/engines/pseudo/sharpen.js new file mode 100644 index 0000000..93b1e0b --- /dev/null +++ b/src/engines/pseudo/sharpen.js @@ -0,0 +1,793 @@ +/** + * pseudo/sharpen.js + * + * From an idea to precise steps, in five rungs, each one less vague than + * the one before: + * + * 1. Say it the problem as you would say it to a colleague; + * every word that hides a choice gets a question + * 2. Sharpen it the same sentence again, saying every choice made + * 3. Name the pieces the inputs, the answer, and examples worked by hand + * 4. Write the steps plain English that runs: any phrase the tool does not + * know is a phrase to define in sharper words + * 5. See it as code the same steps as the course's pseudocode, and Mojo + * + * sharpen(spec) -> node, for an {ojs} cell + * + * spec = { + * title, blurb, + * question, sharpened, the sentence, and a sharper one + * inputs: [{ name, kind, meaning }], answer: { kind, meaning }, + * examples: [{ values: { P: '"ATT"', T: '"GATTACA"' }, expect: '[1]' }], + * steps: 'To find every start of P in T:\n ...', + * reference: { code, name }, a model answer in pseudocode, to compare + * rungs: ['say', 'sharpen', 'name', 'steps', 'code'], which to show + * start: 'say', the rung to open at + * id: 'sharpen-match', where work is kept in this browser + * labUrl: '../lab/', + * } + */ + +import { el, add, clear, panel, button } from './dom.js'; +import { detect, checkRewrite, segments, QUESTIONS } from './vague.js'; +import { readSteps, narrate, englishValue, englishError, englishLineHtml, kindFromWords, kindText, parseValue, callFor, SENTENCE_FORMS } from './english.js'; +import { run } from './interp.js'; +import { sameResult } from './testing.js'; +import { valuesEqual, formatValue } from './lang.js'; +import { variablesView, frameAt } from './views.js'; +import { editor } from './editor.js'; +import { lineHtml } from './highlight.js'; +import { translate } from './translate.js'; +import { study } from './study.js'; + +const RUNGS = [ + { id: 'say', title: 'Say it', short: 'Say it' }, + { id: 'sharpen', title: 'Sharpen it', short: 'Sharpen it' }, + { id: 'name', title: 'Name the pieces', short: 'Name the pieces' }, + { id: 'steps', title: 'Write the steps', short: 'Write the steps' }, + { id: 'code', title: 'See it as code', short: 'See it as code' }, +]; + +export const KINDS = ['a text', 'a letter', 'a whole number', 'a decimal number', 'yes or no', 'a list of whole numbers', 'a list of texts', 'a tally of texts']; + +const STEPS_TEMPLATE = (inputs) => { + const names = inputs.map((x) => x.name).filter(Boolean); + const list = names.length > 1 ? `${names.slice(0, -1).join(', ')} and ${names[names.length - 1]}` : names[0] || 'T'; + return `To : + // Set up: what the answer starts as + // Visit: which positions or values, in what order + // Test: what to check at each one + // Update: what changes when the test passes + The answer is . +`; +}; + +export function sharpen(spec = {}) { + const root = el('div', { class: 'pl pl-sharpen' }); + const rungs = RUNGS.filter((r) => !spec.rungs || spec.rungs.includes(r.id)); + const key = spec.id ? `pseudo-sharpen:${spec.id}` : null; + let state = { + rung: rungs.findIndex((r) => r.id === (spec.start || rungs[0].id)), + question: spec.question || '', + decided: { ...(spec.decided || {}) }, + sharpened: spec.sharpened || '', + inputs: (spec.inputs || [{ name: 'T', kind: 'a text', meaning: '' }]).map((x) => ({ ...x })), + answer: { kind: '', meaning: '', ...(spec.answer || {}) }, + examples: (spec.examples || []).map((x) => ({ values: { ...x.values }, expect: x.expect ?? '' })), + steps: spec.steps ?? '', + }; + if (state.rung < 0) state.rung = 0; + if (key) { try { const saved = localStorage.getItem(key); if (saved) state = { ...state, ...JSON.parse(saved) }; } catch { /* storage may be off */ } } + if (!state.steps) state.steps = STEPS_TEMPLATE(state.inputs); + const save = () => { if (key) { try { localStorage.setItem(key, JSON.stringify(state)); } catch { /* storage may be off */ } } }; + + if (spec.title) add(root, el('h3', { class: 'pl-title', text: spec.title })); + if (spec.blurb) add(root, el('p', { class: 'pl-blurb', text: spec.blurb })); + + // the ladder of rungs + const nav = el('ol', { class: 'pl-rungs', role: 'tablist' }); + const body = el('div', { class: 'pl-rung-body' }); + add(root, [nav, body]); + const doneOf = { + say: () => state.question.trim() && openMust(state.question, state.decided) === 0, + sharpen: () => state.sharpened.trim() && checkRewrite(state.sharpened, state.decided).every((x) => x.stated), + name: () => state.inputs.length && state.inputs.every((x) => x.name && x.kind) && state.examples.length > 0, + steps: () => lastRun?.allPass, + code: () => false, + }; + let lastRun = null; + + function drawNav() { + clear(nav); + rungs.forEach((r, i) => { + const done = doneOf[r.id]?.(); + add(nav, el('li', { class: `pl-rung${i === state.rung ? ' is-current' : ''}${done ? ' is-done' : ''}` }, [ + el('button', { type: 'button', role: 'tab', 'aria-selected': i === state.rung ? 'true' : 'false', onclick: () => go(i) }, [ + el('span', { class: 'pl-rung-n', text: done ? '✓' : String(i + 1) }), + el('span', { class: 'pl-rung-t', text: r.short }), + ]), + ])); + }); + } + function go(i) { state.rung = Math.max(0, Math.min(i, rungs.length - 1)); save(); draw(); root.scrollIntoView?.({ block: 'nearest' }); } + function footer() { + const i = state.rung; + return el('div', { class: 'pl-rung-foot' }, [ + i > 0 ? button(`← ${rungs[i - 1].title}`, () => go(i - 1)) : el('span'), + i < rungs.length - 1 ? button(`${rungs[i + 1].title} →`, () => go(i + 1), { primary: true }) : el('span'), + ]); + } + function draw() { + drawNav(); + clear(body); + const r = rungs[state.rung]; + const view = { say: drawSay, sharpen: drawSharpen, name: drawName, steps: drawSteps, code: drawCode }[r.id]; + add(body, view()); + add(body, footer()); + } + + /* ---------- questions, shared by rungs 1 and 2 ---------- */ + + function questionCard(q, onChange, expanded = false) { + const chosen = state.decided[q.id]; + if (chosen != null && !expanded) { + const label = chosen === 'na' ? 'Does not apply here' : typeof chosen === 'object' ? chosen.other : q.options[chosen]?.label; + const card = el('div', { class: `pl-q is-settled is-compact${q.severity === 'must' ? ' is-must' : ''}`, dataset: { q: q.id } }, [ + el('span', { class: 'pl-check-icon', text: '✓', 'aria-hidden': 'true' }), + el('span', {}, [el('span', { class: 'pl-muted', text: `${q.question} ` }), el('strong', { text: label })]), + ]); + add(card, button('Change', () => { card.replaceWith(questionCard(q, onChange, true)); })); + return card; + } + const card = el('div', { class: `pl-q${chosen != null ? ' is-settled' : ''}${q.severity === 'must' ? ' is-must' : ''}`, dataset: { q: q.id } }); + add(card, el('p', { class: 'pl-q-title' }, [el('strong', { text: q.question }), q.severity !== 'must' ? el('span', { class: 'pl-muted', text: ' worth asking too' }) : null])); + add(card, el('details', { class: 'pl-q-why' }, [el('summary', { text: 'Why it matters' }), el('p', { text: q.why })])); + const opts = el('div', { class: 'pl-q-options', role: 'radiogroup', 'aria-label': q.question }); + const name = `q-${q.id}-${Math.random().toString(36).slice(2, 7)}`; + q.options.forEach((o, k) => { + const value = o.na ? 'na' : k; + const input = el('input', { type: 'radio', name, value: String(value) }); + if (chosen === value || (chosen === 'na' && o.na)) input.checked = true; + input.addEventListener('change', () => { state.decided[q.id] = value; save(); onChange(); }); + add(opts, el('label', { class: 'pl-q-opt' }, [input, el('span', { text: o.label })])); + }); + const other = el('input', { type: 'text', class: 'pl-input pl-q-other', placeholder: 'Something else: say it in your own words', value: typeof chosen === 'object' && chosen?.other ? chosen.other : '' }); + other.addEventListener('change', () => { if (other.value.trim()) { state.decided[q.id] = { other: other.value.trim() }; save(); onChange(); } }); + add(opts, other); + add(card, opts); + return card; + } + function markedSentence(text, qs) { + const p = el('p', { class: 'pl-marked' }); + for (const seg of segments(text, qs)) { + if (!seg.ids.length) { add(p, seg.text); continue; } + const open = seg.ids.filter((id) => state.decided[id] == null); + const mustOpen = open.filter((id) => QUESTIONS.find((q) => q.id === id)?.severity === 'must'); + add(p, el('mark', { class: `pl-vague${!open.length ? ' is-settled' : mustOpen.length ? '' : ' is-worth'}`, title: seg.ids.map((id) => QUESTIONS.find((q) => q.id === id)?.question).join('\n'), text: seg.text })); + } + return p; + } + + /* ---------- 1. Say it ---------- */ + + function drawSay() { + const wrap = el('div', { class: 'pl-rung-panel' }); + add(wrap, el('p', { class: 'pl-rung-lead', text: 'Write the problem the way you would say it to a colleague: one or two sentences, in your own words. Then answer the questions it raises. Each question is a choice the sentence leaves open, and a program has to make every one of them.' })); + const ta = el('textarea', { class: 'pl-textarea', rows: '3', 'aria-label': 'the problem in your words', placeholder: 'Where in this sequence does this shorter sequence occur?' }); + ta.value = state.question; + const live = el('div', { class: 'pl-say-live' }); + const refresh = () => { + clear(live); + const qs = detect(state.question); + if (!state.question.trim()) { add(live, el('p', { class: 'pl-muted', text: 'The questions appear as you type.' })); drawNav(); return; } + add(live, markedSentence(state.question, qs)); + const must = qs.filter((q) => q.severity === 'must'); + const worth = qs.filter((q) => q.severity !== 'must'); + const open = must.filter((q) => state.decided[q.id] == null).length; + add(live, el('p', { class: `pl-status${open ? '' : ' is-ok'}`, text: qs.length ? (open ? `${open === 1 ? 'One choice is' : `${open} choices are`} still open.` : `Every choice this sentence raises has an answer. Next, write the sentence again so that it says them.`) : 'No vague words found. Read it once more as a stranger would: could any word mean two things?' })); + for (const q of must) add(live, questionCard(q, refresh)); + if (worth.length) { + const more = el('details', { class: 'pl-q-more', open: worth.some((q) => state.decided[q.id] != null) ? '' : null }, [el('summary', { text: `${worth.length} more ${worth.length === 1 ? 'question' : 'questions'} worth asking about sequence data` })]); + for (const q of worth) add(more, questionCard(q, refresh)); + add(live, more); + } + drawNav(); + }; + ta.addEventListener('input', () => { state.question = ta.value; save(); refresh(); }); + add(wrap, [ta, live]); + refresh(); + return wrap; + } + + /* ---------- 2. Sharpen it ---------- */ + + function drawSharpen() { + const wrap = el('div', { class: 'pl-rung-panel' }); + add(wrap, el('p', { class: 'pl-rung-lead', text: 'Write the sentence again, so that it says every choice you made. A reader should need nothing else: no context, no guessing. It is still English; precise English.' })); + if (state.question.trim()) add(wrap, el('blockquote', { class: 'pl-said' }, [el('span', { class: 'pl-muted', text: 'You said: ' }), state.question])); + if (!state.sharpened) state.sharpened = state.question; + const ta = el('textarea', { class: 'pl-textarea', rows: '3', 'aria-label': 'the sharpened sentence' }); + ta.value = state.sharpened; + const live = el('div', {}); + const refresh = () => { + clear(live); + const checks = checkRewrite(state.sharpened, state.decided); + if (checks.length) { + const list = el('ul', { class: 'pl-checklist' }); + for (const c of checks) { + add(list, el('li', { class: c.stated ? 'is-ok' : 'is-open' }, [ + el('span', { class: 'pl-check-icon', text: c.stated ? '✓' : '○', 'aria-hidden': 'true' }), + el('span', {}, [el('strong', { text: c.choice }), c.na ? ` (${c.question.replace(/\?$/, '').toLowerCase()})` : '', !c.stated && !c.na ? el('span', { class: 'pl-muted', text: ` The sentence does not say this yet. Add something like "${c.suggest}".` }) : null]), + ])); + } + add(live, [el('p', { class: 'pl-sub', text: 'The choices you made' }), list]); + } else add(live, el('p', { class: 'pl-muted', text: 'No choices made yet. Go back to "Say it" and answer its questions, or sharpen the sentence here directly.' })); + const still = detect(state.sharpened).filter((q) => state.decided[q.id] == null && q.severity === 'must'); + if (still.length) { + add(live, el('p', { class: 'pl-sub', text: 'Still open in this sentence' })); + add(live, markedSentence(state.sharpened, still)); + for (const q of still) add(live, questionCard(q, refresh)); + } else if (checks.length && checks.every((c) => c.stated)) { + add(live, el('p', { class: 'pl-status is-ok', text: 'The sentence says every choice. It is precise enough to follow by hand: take an example, and check that the sentence alone decides every step.' })); + } + drawNav(); + }; + ta.addEventListener('input', () => { state.sharpened = ta.value; save(); refresh(); }); + add(wrap, [ta, live]); + refresh(); + return wrap; + } + + /* ---------- 3. Name the pieces ---------- */ + + function drawName() { + const wrap = el('div', { class: 'pl-rung-panel' }); + add(wrap, el('p', { class: 'pl-rung-lead', text: 'Give every input a short name and say what kind of value it is. Then work examples by hand: for each, the inputs, and the answer the sentence says is right. The examples are how you will know the steps work.' })); + const sentence = state.sharpened || state.question; + if (sentence.trim()) add(wrap, el('blockquote', { class: 'pl-said', text: sentence })); + // inputs + const inputsBox = el('div', { class: 'pl-pieces' }); + const exBox = el('div', {}); + const drawInputs = () => { + clear(inputsBox); + const table = el('table', { class: 'pl-piece-table' }, [el('tr', {}, [el('th', { text: 'Name' }), el('th', { text: 'Kind' }), el('th', { text: 'What it is' }), el('th')])]); + state.inputs.forEach((x, i) => { + const name = el('input', { class: 'pl-input pl-name-input', value: x.name, 'aria-label': 'input name', placeholder: 'T' }); + name.addEventListener('change', () => { + const old = x.name; x.name = name.value.trim(); + for (const ex of state.examples) if (old in ex.values) { ex.values[x.name] = ex.values[old]; delete ex.values[old]; } + save(); drawExamples(); drawNav(); + }); + const kind = kindSelect(x.kind, (v) => { x.kind = v; save(); drawExamples(); drawNav(); }); + const meaning = el('input', { class: 'pl-input', value: x.meaning || '', placeholder: 'the sequence to search', 'aria-label': 'what it is' }); + meaning.addEventListener('change', () => { x.meaning = meaning.value; save(); }); + add(table, el('tr', {}, [el('td', {}, [name]), el('td', {}, [kind]), el('td', {}, [meaning]), el('td', {}, [state.inputs.length > 1 ? button('Remove', () => { state.inputs.splice(i, 1); save(); drawInputs(); drawExamples(); }) : null])])); + }); + add(inputsBox, [el('p', { class: 'pl-sub', text: 'The inputs' }), table, button('Add an input', () => { state.inputs.push({ name: '', kind: 'a text', meaning: '' }); save(); drawInputs(); drawExamples(); })]); + const ans = kindSelect(state.answer.kind, (v) => { state.answer.kind = v; save(); drawExamples(); }, true); + const am = el('input', { class: 'pl-input', value: state.answer.meaning || '', placeholder: 'every position where P starts, left to right', 'aria-label': 'what the answer is' }); + am.addEventListener('change', () => { state.answer.meaning = am.value; save(); }); + add(inputsBox, [el('p', { class: 'pl-sub', text: 'The answer' }), el('div', { class: 'pl-answer-row' }, [ans, am])]); + const clash = state.inputs.map((x) => x.name).filter((n, i, a) => n && a.indexOf(n) !== i); + if (clash.length) add(inputsBox, el('p', { class: 'pl-error-inline', text: `${clash[0]} names two inputs. Each needs its own name.` })); + const badName = state.inputs.find((x) => x.name && !/^[A-Za-z][A-Za-z0-9]*$/.test(x.name)); + if (badName) add(inputsBox, el('p', { class: 'pl-error-inline', text: `"${badName.name}" is not a usable name. Use one letter (T, P, K) or one word starting with a capital (Read, Primer).` })); + }; + const drawExamples = () => { + clear(exBox); + const names = state.inputs.map((x) => x.name).filter(Boolean); + const table = el('table', { class: 'pl-piece-table pl-examples' }, [el('tr', {}, [...names.map((n) => el('th', { text: n })), el('th', { text: 'The answer' }), el('th')])]); + state.examples.forEach((ex, i) => { + const row = el('tr', {}); + for (const n of names) { + const kind = state.inputs.find((x) => x.name === n)?.kind; + const inp = el('input', { class: 'pl-input', value: ex.values[n] ?? '', placeholder: placeholderFor(kind), 'aria-label': `${n} in example ${i + 1}` }); + const msg = checkValue(ex.values[n], kind); + if (msg) inp.classList.add('is-bad'); + inp.title = msg || ''; + inp.addEventListener('change', () => { ex.values[n] = inp.value; save(); drawExamples(); }); + add(row, el('td', {}, [inp])); + } + const exp = el('input', { class: 'pl-input', value: ex.expect ?? '', placeholder: placeholderFor(state.answer.kind), 'aria-label': `answer in example ${i + 1}` }); + const msg = checkValue(ex.expect, state.answer.kind); + if (msg) exp.classList.add('is-bad'); + exp.title = msg || ''; + exp.addEventListener('change', () => { ex.expect = exp.value; save(); drawExamples(); drawNav(); }); + add(row, [el('td', {}, [exp]), el('td', {}, [button('Remove', () => { state.examples.splice(i, 1); save(); drawExamples(); drawNav(); })])]); + add(table, row); + }); + add(exBox, [el('p', { class: 'pl-sub', text: 'Examples, worked by hand' }), table, button('Add an example', () => { state.examples.push({ values: {}, expect: '' }); save(); drawExamples(); drawNav(); })]); + const bad = state.examples.flatMap((ex) => [...names.map((n) => checkValue(ex.values[n], state.inputs.find((x) => x.name === n)?.kind)), checkValue(ex.expect, state.answer.kind)]).filter(Boolean); + if (bad.length) add(exBox, el('p', { class: 'pl-error-inline', text: bad[0] })); + const hints = exampleHints(state.decided); + if (hints.length) add(exBox, el('div', { class: 'pl-hints-box' }, [el('p', { class: 'pl-sub', text: 'Examples worth having, from the choices you made' }), el('ul', {}, hints.map((h) => el('li', { text: h })))])); + }; + add(wrap, [inputsBox, exBox]); + drawInputs(); drawExamples(); + return wrap; + } + + function kindSelect(value, onChange, forAnswer = false) { + const sel = el('select', { class: 'pl-select', 'aria-label': forAnswer ? 'the kind of answer' : 'the kind of input' }); + const kinds = forAnswer ? ['', ...KINDS, 'nothing: it shows its results'] : KINDS; + for (const k of kinds) { const o = el('option', { value: k, text: k || 'choose a kind' }); if (k === value) o.selected = true; add(sel, o); } + sel.addEventListener('change', () => onChange(sel.value)); + return sel; + } + + /* ---------- 4. Write the steps ---------- */ + + let ed = null; + function drawSteps() { + const wrap = el('div', { class: 'pl-rung-panel pl-steps' }); + add(wrap, el('p', { class: 'pl-rung-lead', text: 'Write the steps in plain English, one to a line, indented under the line they belong to. The tool reads every sentence literally. When it meets a phrase it does not know, define that phrase on a line of its own, in sharper words, and keep going until every word is one the tool knows. Then run the examples.' })); + const sentence = state.sharpened || state.question; + if (sentence.trim()) add(wrap, el('blockquote', { class: 'pl-said', text: sentence })); + const findings = el('div', { class: 'pl-findings', 'aria-live': 'polite' }); + const tree = el('div', { class: 'pl-breakdown-box' }); + const results = el('div', { class: 'pl-results' }); + const stepper = el('div', { class: 'pl-estepper' }); + ed = editor({ + value: state.steps, minLines: 10, label: 'the steps in plain English', highlight: englishLineHtml, opens: /:\s*(\/\/.*)?$/, + onChange: (v) => { state.steps = v; save(); check(); }, + onRun: () => runAll(), + }); + const formsBox = sentenceForms((text) => { ed.insert(text); state.steps = ed.value; save(); check(); }); + const toolbar = el('div', { class: 'pl-lab-toolbar' }, [ + button('Run the examples', () => runAll(), { primary: true, title: 'Ctrl+Enter' }), + spec.reference ? button('Check on many inputs', () => runMany()) : null, + button('Sentences the tool reads', () => { formsBox.hidden = !formsBox.hidden; }), + ]); + add(wrap, [toolbar, formsBox, ed.node, findings, tree, results, stepper]); + formsBox.hidden = true; + + let current = null; + function check() { + current = compile(); + clear(findings); + clear(tree); + const bd = breakdown(current); + if (bd) add(tree, bd); + const ds = current.diags.filter((d) => d.severity !== 'note' || d.code !== 'auto-name'); + ed.setDiagnostics(ds.map((d) => ({ ...d }))); + if (!ds.length) { add(findings, el('p', { class: 'pl-status is-ok', text: 'Every sentence reads. Run the examples.' })); return current; } + for (const d of ds.slice(0, 8)) add(findings, findingNode(d)); + if (ds.length > 8) add(findings, el('p', { class: 'pl-muted', text: `${ds.length - 8} more below these; fix the first ones first.` })); + return current; + } + function findingNode(d) { + const node = el('div', { class: `pl-finding is-${d.severity}` }, [ + el('button', { type: 'button', class: 'pl-finding-line', text: `Line ${d.line}`, onclick: () => ed.goToLine(d.line) }), + el('span', { text: d.message }), + ]); + if (d.question) { + const opts = el('div', { class: 'pl-finding-opts' }, [el('span', { class: 'pl-muted', text: `${d.question.prompt} ` })]); + for (const o of d.question.options) add(opts, button(o.label, () => { applyOption(d, o); })); + add(node, opts); + } + return node; + } + function applyOption(d, o) { + const lines = state.steps.split('\n'); + if (o.replace) { + const L = lines[d.line - 1] ?? ''; + const a = (o.replace.col ?? 1) - 1; const b = (o.replace.endCol ?? L.length + 1) - 1; + lines[d.line - 1] = L.slice(0, a) + o.replace.text + L.slice(b); + } else if (o.insert) { + while (lines.length && !lines[lines.length - 1].trim()) lines.pop(); + lines.push('', o.insert.text); + } + state.steps = lines.join('\n'); + ed.value = state.steps; + save(); check(); + if (o.insert) ed.goToLine(lines.length); + } + function compile() { + const inputs = {}; + for (const x of state.inputs) if (x.name && x.kind) { const t = kindFromWords(x.kind.split(/\s+/)); if (t) inputs[x.name] = t; } + return readSteps(state.steps, { inputs }); + } + + function examplesFor(r) { + const out = []; + for (const ex of state.examples) out.push({ values: ex.values, expect: ex.expect, from: 'rung' }); + for (const ex of r.examples) out.push({ inline: ex, from: 'text' }); + return out; + } + function runAll() { + const r = check(); + clear(results); clear(stepper); + lastRun = null; + if (!r.ok) { add(results, el('p', { class: 'pl-status', text: 'Fix the findings above first; the steps run once every sentence reads.' })); drawNav(); return; } + const exs = examplesFor(r); + if (!exs.length) { add(results, el('p', { class: 'pl-status', text: 'There are no examples to run. Add some in "Name the pieces", or write one below the steps: Example: gives .' })); drawNav(); return; } + const table = el('table', { class: 'pl-result-table' }, [el('tr', {}, [el('th', { text: 'Example' }), el('th', { text: 'Expected' }), el('th', { text: 'The steps give' }), el('th')])]); + let pass = 0; + const rows = []; + exs.forEach((ex, i) => { + let call; let expect; let label; let problem = null; + try { + if (ex.inline) { + const vals = {}; ex.inline.routine.params.forEach((p, k) => { vals[p.display] = valueOfNode(ex.inline.args[k]); }); + call = callFor(r, vals, ex.inline.routine); + expect = ex.inline.shows ? { shows: String(valueOfNode(ex.inline.shows)) } : { value: valueOfNode(ex.inline.expect) }; + label = ex.inline.text.replace(/\s+(gives|shows|returns|->).*$/i, ''); + } else { + call = callFor(r, ex.values); + const main = r.main; + expect = { value: ex.expect === '' ? undefined : parseValue(ex.expect, resolvedAnswerType(r)) }; + label = main.params.map((p, k) => `${p.display} = ${englishValue(call.args[k], p.type)}`).join(', '); + } + } catch (err) { problem = err.message; } + if (problem) { rows.push({ i, ok: false }); add(table, el('tr', { class: 'is-bad' }, [el('td', { text: label || `example ${i + 1}` }), el('td', { colspan: '3', class: 'pl-error-inline', text: problem })])); return; } + const t = run(r.study.program, { call, record: true, maxSteps: 20000 }); + let got; let ok; + if (t.error) { const e = englishError(r, t.error); got = `stopped at line ${e.line}: ${e.message}`; ok = false; } + else if (expect.shows != null) { got = `shows ${JSON.stringify(t.output.join('\n'))}`; ok = t.output.join('\n') === expect.shows; } + else if (expect.value === undefined) { got = englishValue(t.result, r.main.ret); ok = null; } + else { got = englishValue(t.result, r.main.ret); ok = sameValue(t.result, expect.value); } + if (ok) pass++; + rows.push({ i, ok, t, call, r }); + add(table, el('tr', { class: ok === false ? 'is-bad' : ok ? 'is-ok' : '' }, [ + el('td', { text: label }), + el('td', { text: expect.shows != null ? `shows ${JSON.stringify(expect.shows)}` : expect.value === undefined ? '(not given)' : englishValue(expect.value, r.main.ret) }), + el('td', {}, [el('span', { class: 'pl-result-mark', text: ok === false ? '✗ ' : ok ? '✓ ' : '' }), got]), + el('td', {}, [button('Step through', () => showStepper(r, t, label))]), + ])); + }); + const allPass = pass === exs.length; + lastRun = { allPass }; + add(results, [table, el('p', { class: `pl-status${allPass ? ' is-ok' : ''}`, text: allPass ? `${exs.length === 1 ? 'The example gives' : `All ${exs.length} examples give`} the expected answer.${spec.reference ? ' Now check on many inputs, including ones nobody worked by hand.' : ' See it as code next.'}` : `${pass} of ${exs.length} examples give the expected answer. Step through one that does not, and find the first step that does something you did not expect.` })]); + const firstBad = rows.find((x) => x.ok === false && x.t); + if (firstBad) showStepper(r, firstBad.t, `example ${firstBad.i + 1}`, true); + drawNav(); + } + function runMany() { + const r = check(); + clear(results); + if (!r.ok) { add(results, el('p', { class: 'pl-status', text: 'Fix the findings above first.' })); return; } + const ref = study(spec.reference.code, { coach: false }); + const refSub = ref.program.subs.get(String(spec.reference.name).toUpperCase()); + const main = r.main; + const byName = refSub.params.map((rp) => main.params.findIndex((mp) => mp.display.toLowerCase() === rp.name.toLowerCase())); + let n = 0; let first = null; + for (let k = 0; k < 1200 && !first && n < 800; k++) { + const args = inputsFor(refSub.params.map((rp) => rp.type), k); + const a = run(ref.program, { call: { name: refSub.name, args }, record: false, maxSteps: 50000 }); + if (a.error) continue; + const mine = main.params.map((_, j) => { const at = byName.indexOf(j); return at >= 0 ? args[at] : args[j]; }); + const b = run(r.study.program, { call: { name: main.pseudo, args: mine.map((x) => (Array.isArray(x) ? x.slice() : x)) }, record: false, maxSteps: 50000 }); + n++; + const agree = !b.error && sameValue(b.result, a.result); + if (!agree) first = { args: mine, want: a.result, got: b.error ? englishError(r, b.error).message : englishValue(b.result, main.ret) }; + } + if (first) { + const call = { name: main.pseudo, args: first.args }; + const t = run(r.study.program, { call, record: true, maxSteps: 20000 }); + const label = main.params.map((p, j) => `${p.display} = ${formatValue(first.args[j])}`).join(', '); + add(results, el('div', { class: 'pl-status' }, [el('strong', { text: 'An input where the steps and the model answer disagree: ' }), `${label}. The answer should be ${englishValue(first.want, main.ret)}; the steps give ${first.got}.`])); + showStepper(r, t, label, true); + } else add(results, el('p', { class: 'pl-status is-ok', text: `The steps agree with the model answer on all ${n} inputs tried, including every DNA string up to length 4 and hundreds of longer random ones.` })); + } + function showStepper(r, t, label, quiet = false) { + clear(stepper); + add(stepper, englishStepper(r, t, { title: `Stepping through ${label}` })); + if (!quiet) stepper.scrollIntoView?.({ block: 'nearest', behavior: 'smooth' }); + } + check(); + return wrap; + } + + /* ---------- 5. See it as code ---------- */ + + function drawCode() { + const wrap = el('div', { class: 'pl-rung-panel' }); + add(wrap, el('p', { class: 'pl-rung-lead', text: 'The same steps in the course\'s pseudocode, line for line. Each English sentence became one or more lines of notation; hover over either side to see which. The notation says the same thing in fewer words, and it is one small step from here to Mojo.' })); + const inputs = {}; + for (const x of state.inputs) if (x.name && x.kind) { const t = kindFromWords(x.kind.split(/\s+/)); if (t) inputs[x.name] = t; } + const r = readSteps(state.steps, { inputs }); + if (!r.ok) { add(wrap, el('p', { class: 'pl-status', text: 'The steps do not read yet. Go back to "Write the steps" and fix the findings first.' })); return wrap; } + const tabs = el('div', { class: 'pl-tabs', role: 'tablist' }); + const pane = el('div', {}); + const langs = [['pseudocode', 'Pseudocode'], ['mojo', 'Mojo'], ['python', 'Python']]; + let lang = 'pseudocode'; + const drawPane = () => { + clear(pane); clear(tabs); + for (const [k, t] of langs) add(tabs, el('button', { type: 'button', role: 'tab', class: `pl-tab${k === lang ? ' is-current' : ''}`, 'aria-selected': k === lang ? 'true' : 'false', text: t, onclick: () => { lang = k; drawPane(); } })); + add(pane, sideBySide(r, lang)); + }; + drawPane(); + add(wrap, [tabs, pane]); + const labUrl = spec.labUrl ?? '../lab/'; + const main = r.main; + let call = ''; + const ex = state.examples[0]; + if (ex && main) { + try { const c = callFor(r, ex.values); call = `${main.pseudo}(${c.args.map((a) => formatValue(a, undefined)).join(', ')})`; } catch { call = ''; } + } + const link = el('a', { class: 'pl-btn', href: `${labUrl}#lab=${encodeLab({ code: r.pseudo, call, viewSet: 'step through' })}`, text: 'Open the pseudocode in the lab' }); + add(wrap, el('p', {}, [link, el('span', { class: 'pl-muted', text: ' The lab can prove it correct, draw its data flow, and measure how its work grows.' })])); + return wrap; + } + + draw(); + return root; +} + +/* ------------------------------------------------------------------ */ +/* Pieces */ +/* ------------------------------------------------------------------ */ + +function openMust(text, decided) { return detect(text).filter((q) => q.severity === 'must' && decided[q.id] == null).length; } + +function placeholderFor(kind) { + return { 'a text': '"GATTACA"', 'a letter': "'A'", 'a whole number': '3', 'a decimal number': '0.5', 'yes or no': 'yes', 'a list of whole numbers': '[1, 4]', 'a list of texts': '["AT", "GC"]', 'a tally of texts': '{"AT": 2}' }[kind] || ''; +} +function checkValue(text, kind) { + if (text == null || String(text).trim() === '') return null; + const t = kindFromWords(String(kind || '').split(/\s+/)); + if (!t) return null; + try { + const v = parseValue(text, t); + const ok = { STRING: typeof v === 'string', CHAR: typeof v === 'string' && [...v].length === 1, INTEGER: Number.isInteger(v), FLOAT: typeof v === 'number', BOOL: typeof v === 'boolean', LIST: Array.isArray(v), HASHMAP: v instanceof Map, SET: v instanceof Set || Array.isArray(v) }[t.k]; + return ok === false ? `${text} is not ${kind}.` : null; + } catch (err) { return err.message; } +} +function resolvedAnswerType(r) { return r.main?.ret; } +function sameValue(a, b) { + if (a instanceof Map && b instanceof Map) return a.size === b.size && [...a].every(([k, v]) => b.has(k) && valuesEqual(v, b.get(k))); + if (a instanceof Map && b && typeof b === 'object' && !Array.isArray(b)) return sameValue(a, new Map(Object.entries(b))); + return sameResult(a, b); +} +function valueOfNode(e) { + if (!e) return undefined; + switch (e.kind) { + case 'Str': case 'Num': case 'Bool': return e.value; + case 'ListLit': return e.items.map(valueOfNode); + case 'SetLit': return new Set(e.items.map(valueOfNode)); + case 'Unary': return -valueOfNode(e.arg); + default: return parseValue(e.text || ''); + } +} + +/** + * How the idea breaks down: the routine at the top, each phrase it uses + * below it, and every phrase not yet defined marked open. A step is precise + * when everything under it bottoms out in words the tool already knows. + */ +export function breakdown(r) { + const owners = [...r.routines, ...r.defs]; + if (!owners.length) return null; + const spanOf = (o) => { + let last = o.line; + const walk = (items) => { for (const c of items || []) { last = Math.max(last, c.line); walk(c.children); } }; + walk(o.it?.children); + return [o.line, last]; + }; + const uses = new Map(owners.map((o) => [o, new Set()])); + const visit = (o, x) => { + if (!x || typeof x !== 'object') return; + if (Array.isArray(x)) { for (const y of x) visit(o, y); return; } + if (x.kind === 'DefCall' && x.def) uses.get(o).add(x.def); + if ((x.kind === 'RoutineCall' || x.kind === 'CallStep') && x.routine) uses.get(o).add(x.routine); + for (const [k, v] of Object.entries(x)) if (!['entry', 'def', 'routine', 'it', 'head', 'scope', 'attachTo', 'var', 'params', 'implicit', 'calls'].includes(k) && v && typeof v === 'object') visit(o, v); + }; + for (const o of owners) visit(o, o.kind === 'routine' ? o.body : o.kind === 'predicate' ? o.cond : o.pieces); + const open = new Map(owners.map((o) => [o, []])); + for (const d of r.diags) { + if (d.code !== 'unknown-phrase') continue; + const o = owners.find((x) => { const [a, b] = spanOf(x); return d.line >= a && d.line <= b; }); + if (o) open.get(o).push(d.phrase); + } + const usedBy = new Set([...uses.values()].flatMap((x) => [...x])); + const roots = owners.filter((o) => !usedBy.has(o)); + const node = (o, seen) => { + const kids = [...uses.get(o)].filter((x) => !seen.has(x)); + const opens = open.get(o); + const li = el('li', { class: opens.length ? 'is-open' : 'is-ok' }, [ + el('span', { class: 'pl-bd-mark', text: opens.length ? '?' : '✓', 'aria-hidden': 'true' }), + el('span', { class: 'pl-bd-head', text: o.kind === 'routine' ? `To ${o.head.text}` : o.head.text }), + el('span', { class: 'pl-muted', text: ` line ${o.line}` }), + ]); + const sub = el('ul', {}); + for (const k of kids) add(sub, node(k, new Set([...seen, k]))); + for (const ph of opens) add(sub, el('li', { class: 'is-open' }, [el('span', { class: 'pl-bd-mark', text: '?', 'aria-hidden': 'true' }), el('span', { class: 'pl-bd-head', text: ph }), el('span', { class: 'pl-muted', text: ' not defined yet' })])); + if (!kids.length && !opens.length) add(sub, el('li', { class: 'pl-bd-leaf' }, [el('span', { class: 'pl-muted', text: 'uses only words the tool knows' })])); + add(li, sub); + return li; + }; + const list = el('ul', { class: 'pl-breakdown' }); + for (const o of roots.length ? roots : owners.slice(0, 1)) add(list, node(o, new Set([o]))); + const openCount = [...open.values()].reduce((n, x) => n + x.length, 0); + return el('div', {}, [ + el('p', { class: 'pl-sub', text: 'How the idea breaks down' }), + list, + el('p', { class: `pl-status${openCount ? '' : ' is-ok'}`, text: openCount ? `${openCount} ${openCount === 1 ? 'phrase needs' : 'phrases need'} defining. Each definition may use new phrases of its own; keep going until every branch ends in words the tool knows.` : 'Every branch ends in words the tool knows: nothing is left to interpret.' }), + ]); +} + +/** Examples worth having, from the choices made in rung 1. */ +function exampleHints(decided) { + const h = []; + const d = (id) => decided[id]; + if (d('empty') != null) h.push('An empty input, or one where there is nothing to find: the answer you chose for that case.'); + if (d('overlap') === 0) h.push('An example where appearances overlap, such as "AA" in "AAAA": every overlapping appearance counts.'); + if (d('overlap') === 1) h.push('An example where appearances overlap, such as "AA" in "AAAA": they must not both count.'); + if (d('ties') != null) h.push('A tie: two positions with the same best value.'); + if (d('threshold') != null) h.push('A value exactly at the threshold, one just above, and one just below.'); + if (d('ends') != null) h.push('Values at both ends of the range.'); + if (d('which-position') != null) h.push('An appearance at the very start and one at the very end of the sequence.'); + if (d('unknown-base') != null) h.push('A sequence containing N.'); + if (d('case') != null) h.push('A sequence with a lowercase letter.'); + if (d('frame') != null) h.push('A sequence whose length is not a multiple of 3.'); + if (!h.length) h.push('A small ordinary case, an empty input, and a case at an edge: the first or last position.'); + return h; +} + +/** + * Inputs for "Check on many inputs", the k-th set. The first text runs + * through every DNA string up to length 4, then random longer ones. Every + * other input is chosen independently: half the time a text of the same + * length as the first (so two-read problems get pairs that can be + * compared), otherwise any text. Each input has its own random stream. + */ +export function inputsFor(types, k) { + const all = (n) => (n === 0 ? [''] : all(n - 1).flatMap((x) => [...'ACGT'].map((c) => x + c))); + const small = [0, 1, 2, 3, 4].flatMap((n) => all(n)); + const streams = types.map((_, j) => { let seed = (k + 1) * 7919 + j * 104729; return () => ((seed = (seed * 1103515245 + 12345) % 2147483648) / 2147483648); }); + const randomText = (rnd, n) => Array.from({ length: n }, () => 'ACGT'[Math.floor(rnd() * 4)]).join(''); + let firstText = null; + return types.map((t, j) => { + const rnd = streams[j]; + for (let w = 0; w < j * 3; w++) rnd(); + switch (t.k) { + case 'STRING': { + let v; + if (firstText === null) v = k < small.length ? small[k] : randomText(rnd, 5 + Math.floor(rnd() * 8)); + else v = rnd() < 0.5 ? randomText(rnd, [...firstText].length) : (rnd() < 0.5 ? small[Math.floor(rnd() * small.length)] : randomText(rnd, Math.floor(rnd() * 9))); + if (firstText === null) firstText = v; + return v; + } + case 'CHAR': return 'ACGT'[Math.floor(rnd() * 4)]; + case 'INTEGER': return Math.floor(rnd() * 12) - 2; + case 'FLOAT': return Math.round(rnd() * 1000) / 10; + case 'BOOL': return rnd() < 0.5; + case 'LIST': return Array.from({ length: Math.floor(rnd() * 7) }, () => Math.floor(rnd() * 40)); + default: return null; + } + }); +} + +function encodeLab(s) { + const bytes = new TextEncoder().encode(JSON.stringify(s)); + let bin = ''; + for (const b of bytes) bin += String.fromCharCode(b); + return btoa(bin).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} + +/** The sentences the tool reads, grouped, each one click away from the editor. */ +function sentenceForms(insert) { + const box = el('div', { class: 'pl-forms' }); + for (const g of SENTENCE_FORMS) { + const list = el('ul', {}); + for (const [form, example] of g.forms) { + add(list, el('li', {}, [ + el('code', { class: 'pl-form', text: form }), + example ? el('button', { type: 'button', class: 'pl-form-ex', title: 'Insert this example at the cursor', text: example, onclick: () => insert(example) }) : null, + ])); + } + add(box, el('div', { class: 'pl-form-group' }, [el('p', { class: 'pl-sub', text: g.group }), list])); + } + return box; +} + +/** English beside pseudocode, Mojo, or Python, with the matching lines lit together. */ +function sideBySide(r, lang) { + const eLines = r.source.replace(/\r\n?/g, '\n').split('\n'); + let right; + if (lang === 'pseudocode') right = r.pseudo.split('\n').map((text, i) => ({ text, eline: r.lineMap[i]?.line ?? null, html: lineHtml(text) })); + else { + const tr = translate(r.study.program, lang); + right = tr.lines.map((l) => ({ text: l.text, eline: l.from ? r.lineMap[l.from - 1]?.line ?? null : null, html: escHtml(l.text) })); + if (tr.notes?.length) right.notes = tr.notes; + } + const grid = el('div', { class: 'pl-sbs' }); + const left = el('div', { class: 'pl-sbs-col pl-sbs-en', 'aria-label': 'the steps in English' }); + const rightCol = el('div', { class: `pl-sbs-col pl-sbs-code`, 'aria-label': lang }); + const leftRows = eLines.map((text, i) => el('div', { class: 'pl-sbs-row', dataset: { e: String(i + 1) }, html: `${i + 1}${englishLineHtml(text) || '​'}` })); + const rightRows = right.map((x) => el('div', { class: 'pl-sbs-row is-code', dataset: { e: x.eline ? String(x.eline) : '' }, html: `${x.html || '​'}` })); + add(left, leftRows); add(rightCol, rightRows); + const light = (e) => { for (const n of [...leftRows, ...rightRows]) n.classList.toggle('is-lit', Boolean(e) && n.dataset.e === String(e)); }; + for (const n of [...leftRows, ...rightRows]) { n.addEventListener('mouseenter', () => light(n.dataset.e)); n.addEventListener('mouseleave', () => light(null)); } + add(grid, [left, rightCol]); + const out = el('div', {}, [grid]); + if (right.notes?.length) add(out, el('ul', { class: 'pl-notes' }, right.notes.map((n) => el('li', { text: n })))); + return out; +} +function escHtml(s) { return String(s).replace(/&/g, '&').replace(//g, '>'); } + +/** + * A run of English steps, one sentence at a time: the listing with the + * current line marked, what the step did in words, and the values. + */ +export function englishStepper(r, t, options = {}) { + const root = el('div', { class: 'pl-estep' }); + if (options.title) add(root, el('p', { class: 'pl-sub', text: options.title })); + // the steps worth showing: the English side skips DECLAREs and the inside of a tally + const shown = []; + t.steps.forEach((s, k) => { const text = narrate(r, s, t.states[k + 1], t.states[k]); if (text) shown.push({ k, text, line: r.lineMap[s.line - 1]?.line ?? null }); }); + const total = shown.length; + let p = 0; + const lines = r.source.replace(/\r\n?/g, '\n').split('\n'); + const listing = el('div', { class: 'pl-code pl-elisting' }); + const rows = lines.map((text, i) => el('div', { class: 'pl-sbs-row', html: `${i + 1}${englishLineHtml(text) || '​'}` })); + add(listing, rows); + const said = el('p', { class: 'pl-narration', 'aria-live': 'polite' }); + const readout = el('span', { class: 'pl-readout' }); + const slider = el('input', { type: 'range', min: '0', max: String(total), value: '0', class: 'pl-slider', 'aria-label': 'step' }); + let timer = null; + const playBtn = button('Play', () => (timer ? pause() : play())); + const bar = el('div', { class: 'pl-controls' }, [ + button('Reset', () => { pause(); goto(0); }), button('Back', () => { pause(); goto(p - 1); }), playBtn, + button('Step', () => { pause(); goto(p + 1); }, { primary: true }), button('End', () => { pause(); goto(total); }), slider, readout, + ]); + slider.addEventListener('input', () => { pause(); goto(Number(slider.value)); }); + const label = (n) => r.display.get(String(n).toUpperCase()) || n; + const ctx = { program: r.study.program, trace: t }; + const vars = variablesView(ctx, { + label, + frameTitle: (fname) => { const o = [...r.routines, ...r.defs].find((x) => x.pseudo.toUpperCase() === String(fname).toUpperCase()); return o ? `"${o.head.text}"` : fname; }, + kindText: (x) => (x.kind === 'param' ? `input, ${kindText(x.type)}` : x.kind === 'counter' || x.kind === 'each' ? `loop, ${kindText(x.type)}` : kindText(x.type)), + title: 'Values', + }); + const out = el('pre', { class: 'pl-output' }); + const err = t.error ? englishError(r, t.error) : null; + add(root, [bar, el('div', { class: 'pl-estep-grid' }, [panel('The steps', listing, 'pl-estep-code'), el('div', {}, [panel('What just happened', said, 'pl-estep-said'), vars.node, t.output.length ? panel('Shown', out) : null])])]); + function goto(n) { + p = Math.max(0, Math.min(n, total)); + const cur = p > 0 ? shown[p - 1] : null; + const next = p < total ? shown[p] : null; + rows.forEach((row, i) => { row.classList.toggle('is-current', cur?.line === i + 1); row.classList.toggle('is-next', !cur && next?.line === i + 1); }); + const atEnd = p === total; + said.textContent = cur ? cur.text : 'Ready. Step to begin.'; + if (atEnd && err) { said.textContent = `${cur ? `${cur.text} ` : ''}Then the run stopped at line ${err.line}: ${err.message}`; said.classList.add('is-error'); rows.forEach((row, i) => row.classList.toggle('is-error', err.line === i + 1)); } else { said.classList.remove('is-error'); rows.forEach((row) => row.classList.remove('is-error')); } + const tIndex = cur ? cur.k + 1 : 0; + vars.update(frameAt(t, tIndex)); + out.textContent = (t.states[tIndex]?.outputLength != null ? t.output.slice(0, t.states[tIndex].outputLength) : t.output).join('\n'); + slider.value = String(p); + readout.textContent = `step ${p} of ${total}`; + playBtn.textContent = timer ? 'Pause' : 'Play'; + const curRow = rows[(cur?.line ?? next?.line ?? 1) - 1]; + if (curRow && listing.scrollHeight > listing.clientHeight) listing.scrollTop = Math.max(0, curRow.offsetTop - listing.clientHeight / 2); + } + function play() { if (timer) return; if (p >= total) p = 0; timer = setInterval(() => (p >= total ? pause() : goto(p + 1)), options.speed ?? 700); goto(p); } + function pause() { if (timer) { clearInterval(timer); timer = null; goto(p); } } + root.tabIndex = 0; + root.addEventListener('keydown', (e) => { + if (/^(INPUT|TEXTAREA|SELECT)$/.test(e.target.tagName) && e.target !== slider) return; + if (e.key === 'ArrowRight') { pause(); goto(p + 1); e.preventDefault(); } + if (e.key === 'ArrowLeft') { pause(); goto(p - 1); e.preventDefault(); } + if (e.key === 'Home') { pause(); goto(0); e.preventDefault(); } + if (e.key === 'End') { pause(); goto(total); e.preventDefault(); } + }); + goto(options.start === 'end' ? total : 0); + return root; +} + +/** A widget that only runs English steps: for galleries and lessons. */ +export function stepsWidget(spec) { + const root = el('div', { class: 'pl pl-sharpen' }); + if (spec.title) add(root, el('h3', { class: 'pl-title', text: spec.title })); + if (spec.blurb) add(root, el('p', { class: 'pl-blurb', text: spec.blurb })); + const r = readSteps(spec.steps, { inputs: spec.inputs || {} }); + if (!r.ok) { + add(root, el('div', { class: 'pl-findings' }, r.diags.filter((d) => d.severity === 'error').map((d) => el('div', { class: 'pl-finding is-error' }, [el('span', { class: 'pl-finding-line', text: `Line ${d.line}` }), el('span', { text: d.message })])))); + return root; + } + try { + const call = callFor(r, spec.values || {}); + const t = run(r.study.program, { call, record: true, maxSteps: spec.maxSteps ?? 20000 }); + add(root, englishStepper(r, t, { title: spec.caption, start: spec.startAt })); + } catch (err) { add(root, el('p', { class: 'pl-error', text: err.message })); } + return root; +} diff --git a/src/engines/pseudo/solver.ts b/src/engines/pseudo/solver.ts new file mode 100644 index 0000000..cd41193 --- /dev/null +++ b/src/engines/pseudo/solver.ts @@ -0,0 +1,628 @@ +/** + * pseudo/solver.js + * + * A small decision procedure: given facts and a goal, either prove the goal + * follows from the facts, or say it could not. "Could not" is never "false": + * the verifier reports it as not proved, and the lab tests the program + * instead. + * + * prove(facts, goal, options) -> { status: 'proved' | 'unknown', reason } + * + * How it works, in the order it happens: + * 1. Quantifiers over [lo, hi) are extended one position at a time where + * another fact talks about [lo, hi - 1): this is the step that lets an + * invariant like Total == LENGTH({ K IN [0, I) : T[K] == B }) carry + * from one pass to the next. + * 2. The facts and the negated goal become a list of cases (disjunctive + * normal form). The goal is proved when every case is impossible. + * 3. Each case is checked with three cooperating parts: equalities between + * letters, strings, and lists (union-find); linear arithmetic over whole + * numbers (Fourier-Motzkin elimination, tightened for integers); and + * congruence, which merges LENGTH(T) with LENGTH(T) and T[I] with T[J] + * whenever the arithmetic shows I == J. + * + * Everything it concludes is sound: a "proved" is a proof. It is incomplete: + * multiplication of two unknowns, MOD, and most facts about quantifiers + * over collections are opaque to it. + */ + +import { + key, bodyKey, and, or, not, cmp, ite, TRUE, FALSE, numeric, linear, rebuild, + constDiff, extend, identity, num, add, substBound, Q, bgcd, work, metered, TOO_LARGE, +} from './logic.js'; +import type { Term, Op, App, Quant, Ite, CmpOp } from './logic.js'; + +/** What `prove` answers: a proof, or "could not", never "false". */ +export interface ProveResult { status: 'proved' | 'unknown'; reason?: string; witness?: Term[] } +/** depth: 1 for a question the prover asks itself (no quantifier instantiation); work: the budget, PROVE_WORK by default. */ +export interface ProveOptions { depth?: number; cap?: number; work?: number } + +/** The terms stored under some of a term's fields, in the order named, for walks that look inside any kind of term. */ +function fields(x: Term, names: readonly string[]): Term[] { + const r = x as unknown as Record; + const out: Term[] = []; + for (const n of names) { const v = r[n]; if (v && typeof v === 'object') out.push(v as Term); } + return out; +} +const argsIn = (x: Term): Term[] => ('args' in x ? x.args : []); +const ABC = ['a', 'b', 'c'] as const; + +/* ------------------------------------------------------------------ */ +/* Rational numbers */ +/* ------------------------------------------------------------------ */ + +// Q, the exact rational the constraints are written in, lives in logic.ts: linear() builds its coefficients with it. +const ZERO = new Q(0n); +const ONE = new Q(1n); + +/* ------------------------------------------------------------------ */ +/* Linear constraints */ +/* ------------------------------------------------------------------ */ + +/** + * { co: Map(atomKey -> Q), c: Q, op: '<=' | '<' | '==' } meaning + * sum(co[x] * x) + c op 0. + */ +type LinOp = '<=' | '<' | '=='; +/** sum(co[atom] * atom) + c op 0 */ +interface Constraint { co: Map; c: Q; op: LinOp } +function constraint(co: Map, c: Q, op: LinOp): Constraint { return { co, c, op }; } + +function tighten(k: Constraint, ints: Set): Constraint { + // Every atom whole-numbered: scale to whole coefficients, divide by their + // common factor, round the constant up, and turn < into <= -1. + if (k.op === '==' || ![...k.co.keys()].every((a) => ints.has(a))) return k; + let l = k.c.d; + for (const v of k.co.values()) l = (l * v.d) / bgcd(l, v.d); + const co = new Map(); let g = 0n; + for (const [a, v] of k.co) { const w = (v.n * l) / v.d; co.set(a, w); g = bgcd(g, w); } + let c = (k.c.n * l) / k.c.d; + if (k.op === '<') c += 1n; + if (!co.size) return constraint(new Map(), new Q(c), '<='); + // sum(w x) + c <= 0 with every w divisible by g: sum(w/g x) + ceil(c/g) <= 0 + const cg = c >= 0n ? (c + g - 1n) / g : -((-c) / g); + const out = new Map(); + for (const [a, w] of co) out.set(a, new Q(w / g)); + return constraint(out, new Q(cg), '<='); +} + +function trivial(k: Constraint): boolean | null { + if (k.co.size) return null; + const s = k.c.sign; + if (k.op === '<=') return s <= 0; + if (k.op === '<') return s < 0; + return s === 0; +} + +function ckey(k: Constraint): string { + const parts = [...k.co].sort((a, b) => (a[0] < b[0] ? -1 : 1)).map(([a, v]) => `${v}*${a}`); + return `${parts.join('+')}|${k.c}|${k.op}`; +} + +/** Is this set of constraints impossible? true: yes; false: possibly satisfiable. */ +function infeasible(list: Constraint[], ints: Set, budget = { n: 6000 }): boolean { + let cs = list.map((k) => tighten(k, ints)); + for (const k of cs) if (trivial(k) === false) return true; + // equalities first: solve for one atom and substitute + for (; ;) { + work(1 + cs.length); + const i = cs.findIndex((k) => k.op === '==' && k.co.size); + if (i < 0) break; + const eq = cs[i]; + const pivot = [...eq.co].find(([, v]) => !v.zero); + if (!pivot) { cs.splice(i, 1); continue; } + const [a, ca] = pivot; + const next: Constraint[] = []; + for (let j = 0; j < cs.length; j++) { + if (j === i) continue; + const k = cs[j]; + work(1 + k.co.size); + const f = k.co.get(a); + if (!f) { next.push(k); continue; } + // k - (f / ca) * eq + const m = f.div(ca); + const co = new Map(k.co); + for (const [b, v] of eq.co) { + const nv = (co.get(b) || ZERO).sub(m.mul(v)); + if (nv.zero) co.delete(b); else co.set(b, nv); + } + const nk = tighten(constraint(co, k.c.sub(m.mul(eq.c)), k.op), ints); + if (trivial(nk) === false) return true; + next.push(nk); + } + cs = next; + } + cs = cs.filter((k) => trivial(k) !== true); + if (cs.some((k) => trivial(k) === false)) return true; + // Fourier-Motzkin + const seen = new Set(cs.map(ckey)); + for (; ;) { + work(1 + cs.length); + const atoms = new Map(); + for (const k of cs) for (const [a, v] of k.co) { + let n = atoms.get(a); + if (!n) { n = { pos: 0, neg: 0 }; atoms.set(a, n); } + if (v.sign > 0) n.pos++; else n.neg++; + } + if (!atoms.size) return cs.some((k) => trivial(k) === false); + let best = ''; let cost = Infinity; + for (const [a, { pos, neg }] of atoms) { const c = pos * neg - pos - neg; if (c < cost) { cost = c; best = a; } } + const P: Constraint[] = []; const N: Constraint[] = []; const rest: Constraint[] = []; + for (const k of cs) { + const v = k.co.get(best); + if (!v) rest.push(k); else if (v.sign > 0) P.push(k); else N.push(k); + } + const out = rest; + for (const p of P) { + for (const n of N) { + if (--budget.n < 0) return false; + work(1 + p.co.size + n.co.size); + const a = p.co.get(best)!; const b = n.co.get(best)!.neg(); + const co = new Map(); + for (const [x, v] of p.co) if (x !== best) co.set(x, v.mul(b)); + for (const [x, v] of n.co) if (x !== best) { const nv = (co.get(x) || ZERO).add(v.mul(a)); if (nv.zero) co.delete(x); else co.set(x, nv); } + const nk = tighten(constraint(co, p.c.mul(b).add(n.c.mul(a)), p.op === '<' || n.op === '<' ? '<' : '<='), ints); + const t = trivial(nk); + if (t === false) return true; + if (t === true) continue; + const kk = ckey(nk); + if (seen.has(kk)) continue; + seen.add(kk); + out.push(nk); + } + } + cs = out; + if (cs.length > 3000) return false; + } +} + +/* ------------------------------------------------------------------ */ +/* From terms to constraints */ +/* ------------------------------------------------------------------ */ + +/** A numeric comparison as linear constraints (a list: == gives one, < gives one). */ +type Atoms = Map; +function toConstraints(op: CmpOp, a: Term, b: Term, atoms: Atoms): Constraint[] { + const la = linear(a)!; const lb = linear(b)!; + const register = (t: Term) => { + const lin = linear(t)!; + for (const k of lin.terms.keys()) if (!atoms.has(k)) atoms.set(k, findAtom(t, k)); + return lin; + }; + register(a); register(b); + const co = new Map(); + for (const [k, v] of la.terms) co.set(k, Q.of(v)); + for (const [k, v] of lb.terms) { const nv = (co.get(k) || ZERO).sub(Q.of(v)); if (nv.zero) co.delete(k); else co.set(k, nv); } + const c = Q.of(la.c).sub(Q.of(lb.c)); + // a - b op 0 + switch (op) { + case '<=': return [constraint(co, c, '<=')]; + case '<': return [constraint(co, c, '<')]; + case '>=': return [constraint(negate(co), c.neg(), '<=')]; + case '>': return [constraint(negate(co), c.neg(), '<')]; + case '==': return [constraint(co, c, '==')]; + default: return []; + } +} +function negate(co: Map): Map { const m = new Map(); for (const [k, v] of co) m.set(k, v.neg()); return m; } + +/** The subterm of `t` whose key is `k`. */ +function findAtom(t: Term, k: string): Term | null { + let found: Term | null = null; + const visit = (x: Term): void => { + if (found || !x || typeof x !== 'object') return; + work(); + if (key(x) === k) { found = x; return; } + if (x.k === 'op') x.args.forEach(visit); + }; + visit(t); + return found; +} + +/* ------------------------------------------------------------------ */ +/* Cases */ +/* ------------------------------------------------------------------ */ + +/** The first if-then-else inside an atom, outside any quantifier body. */ +function firstIte(x: Term | undefined): Ite | null { + if (!x || typeof x !== 'object') return null; + work(); + if (x.k === 'ite') return x; + if (x.k === 'q') return firstIte(x.lo) || firstIte(x.hi); + if (x.k === 'qc') return firstIte(x.dom); + for (const y of fields(x, ABC)) { const r = firstIte(y); if (r) return r; } + for (const y of argsIn(x)) { const r = firstIte(y); if (r) return r; } + return null; +} + +function replaceKey(x: Term, k: string, by: Term): Term { return rebuild(x, (y) => (key(y) === k ? by : null)); } + +/** Negation normal form, with if-then-else lifted out of atoms. */ +function nnf(f: Term, depth = 0): Term { + if (depth > 40) return f; + work(); + switch (f.k) { + case 'and': return and(f.args.map((x) => nnf(x, depth + 1))); + case 'or': return or(f.args.map((x) => nnf(x, depth + 1))); + case 'not': { + const a = f.a; + if (a.k === 'and') return or(a.args.map((x) => nnf(not(x), depth + 1))); + if (a.k === 'or') return and(a.args.map((x) => nnf(not(x), depth + 1))); + if (a.k === 'not') return nnf(a.a, depth + 1); + if (a.k === 'lit') return a.v ? FALSE : TRUE; + const it = firstIte(a); + if (it) return nnf(not(liftOne(a, it)), depth + 1); + if (a.k === 'cmp' && numeric(a.a.t) && a.op === '==') return or(cmp('<', a.a, a.b), cmp('>', a.a, a.b)); + return f; + } + case 'cmp': { + const it = firstIte(f); + if (it) return nnf(liftOne(f, it), depth + 1); + if (numeric(f.a.t) && f.op === '!=') return or(cmp('<', f.a, f.b), cmp('>', f.a, f.b)); + return f; + } + case 'app': { const it = firstIte(f); return it ? nnf(liftOne(f, it), depth + 1) : f; } + default: return f; + } +} +function liftOne(atom: Term, it: Ite): Term { + const k = key(it); + return or(and(it.c, replaceKey(atom, k, it.a)), and(not(it.c), replaceKey(atom, k, it.b))); +} + +/** Disjunctive normal form: a list of cases, each a list of literals. */ +function dnf(f: Term, cap: number): Term[][] | null { + if (f.k === 'and') { + let cases: Term[][] = [[]]; + for (const g of f.args) { + const gs = dnf(g, cap); + if (gs === null) return null; + const next: Term[][] = []; + for (const c of cases) for (const d of gs) { work(1 + c.length + d.length); next.push(c.concat(d)); if (next.length > cap) return null; } + cases = next; + } + return cases; + } + if (f.k === 'or') { + const out: Term[][] = []; + for (const g of f.args) { const gs = dnf(g, cap); if (gs === null) return null; work(1 + gs.length); out.push(...gs); if (out.length > cap) return null; } + return out; + } + if (f.k === 'lit' && f.t === 'bool') return f.v ? [[]] : []; + return [[f]]; +} + +/* ------------------------------------------------------------------ */ +/* One case */ +/* ------------------------------------------------------------------ */ + +class UF { + p = new Map(); + term = new Map(); + add(t: Term): string { const k = key(t); if (!this.p.has(k)) { this.p.set(k, k); this.term.set(k, t); } return k; } + find(k: string): string { let r = k; while (this.p.get(r) !== r) r = this.p.get(r)!; let x = k; while (this.p.get(x) !== r) { const n = this.p.get(x)!; this.p.set(x, r); x = n; } return r; } + union(a: Term, b: Term): boolean { const ra = this.find(this.add(a)); const rb = this.find(this.add(b)); if (ra !== rb) this.p.set(ra, rb); return ra !== rb; } + same(a: Term, b: Term): boolean { return this.find(this.add(a)) === this.find(this.add(b)); } +} + +/** Collect every compound subterm, for congruence. */ +type Compound = App | Quant; +function subterms(x: Term, out: Map): void { + if (!x || typeof x !== 'object') return; + work(); + if (x.k === 'app' || x.k === 'q') out.set(key(x), x); + if (x.k === 'q') { subterms(x.lo, out); subterms(x.hi, out); return; } + if (x.k === 'qc') return; + for (const y of fields(x, ABC)) subterms(y, out); + for (const y of argsIn(x)) subterms(y, out); +} + +function signature(t: Compound): string { + if (t.k === 'q') return `q:${t.q}:${bodyKey(t)}`; + return `app:${t.f}:${t.args.length}`; +} +function argsOf(t: Compound): Term[] { return t.k === 'q' ? [t.lo, t.hi] : t.args; } + +/** Can this case happen? false when a contradiction is found. */ +function caseSatisfiable(lits: Term[]): boolean { + const uf = new UF(); + const bools = new Map(); // atom key -> true/false + const boolTerms = new Map(); + const diseqs: Array<[Term, Term]> = []; + const lin: Constraint[] = []; + const atoms: Atoms = new Map(); // numeric atom key -> term + const all = new Map(); + + const literal = (atom: Term, pos: boolean): boolean => { + if (atom.k === 'lit') return atom.v === pos; + if (atom.k === 'cmp' && numeric(atom.a.t) && numeric(atom.b.t)) { + if (!pos) return true; // nnf removed negated numeric comparisons + const cs = toConstraints(atom.op, atom.a, atom.b, atoms); + if (cs) lin.push(...cs); + subterms(atom, all); + return true; + } + if (atom.k === 'cmp' && (atom.op === '==' || atom.op === '!=')) { + const eq = (atom.op === '==') === pos; + subterms(atom, all); + uf.add(atom.a); uf.add(atom.b); + if (eq) uf.union(atom.a, atom.b); else diseqs.push([atom.a, atom.b]); + return true; + } + const k = key(atom); + subterms(atom, all); + if (bools.has(k) && bools.get(k) !== pos) return false; + bools.set(k, pos); + boolTerms.set(k, atom); + return true; + }; + for (const l of lits) { + const pos = l.k !== 'not'; + const atom = l.k === 'not' ? l.a : l; + if (!literal(atom, pos)) return false; + } + // x DIV c and x MOD c, for a constant c above 0, are pinned by c * (x DIV c) <= x < c * (x DIV c) + c. + const divs = new Map(); + const findDivs = (x: Term): void => { + if (!x || typeof x !== 'object') return; + work(); + if (x.k === 'op' && (x.op === 'DIV' || x.op === 'MOD')) { const c = x.args[1]; if (c.k === 'num' && Number.isInteger(c.v) && c.v > 0) divs.set(key(x), x); } + if (x.k === 'q') { findDivs(x.lo); findDivs(x.hi); return; } + for (const y of fields(x, ABC)) findDivs(y); + for (const y of argsIn(x)) findDivs(y); + }; + for (const l of lits) findDivs(l); + for (const d of divs.values()) { + const [x, cTerm] = d.args; + if (cTerm.k !== 'num') continue; + const c = cTerm; + const q: Term = d.op === 'DIV' ? d : { k: 'op', op: 'DIV', args: [x, c], t: 'int' }; + // c * q <= x <= c * q + c - 1 + lin.push(...toConstraints('<=', { k: 'op', op: '*', args: [c, q], t: 'int' }, x, atoms)); + lin.push(...toConstraints('<=', x, { k: 'op', op: '+', args: [{ k: 'op', op: '*', args: [c, q], t: 'int' }, num(c.v - 1)], t: 'int' }, atoms)); + if (d.op === 'MOD') lin.push(...toConstraints('==', d, { k: 'op', op: '-', args: [x, { k: 'op', op: '*', args: [c, q], t: 'int' }], t: 'int' }, atoms)); + } + // Every length and every count is at least 0. + for (const [k, t] of [...all]) { + if ((t.k === 'app' && t.f === 'LENGTH') || (t.k === 'q' && t.q === 'SIZE')) { + lin.push(...toConstraints('>=', t, num(0), atoms)); + } + } + const ints = () => new Set([...atoms].filter(([, t]) => t && t.t === 'int').map(([k]) => k)); + + const conflict = () => { + for (const [a, b] of diseqs) if (uf.same(a, b)) return true; + // two different literal values in one class + const lits = new Map(); + for (const [k, t] of uf.term) { + if (t.k !== 'lit') continue; + const r = uf.find(k); + if (lits.has(r) && lits.get(r) !== k) return true; + lits.set(r, k); + } + return infeasible(lin, ints()); + }; + + const merged = new Set(); + for (let round = 0; round < 6; round++) { + if (conflict()) return false; + // congruence: same function, equal arguments, therefore equal results + const groups = new Map(); + for (const t of all.values()) { + const s = signature(t); + let g = groups.get(s); + if (!g) { g = []; groups.set(s, g); } + g.push(t); + } + let changed = false; + const intSet = ints(); + const entailedEqual = (a: Term, b: Term): boolean => { + if (key(a) === key(b)) return true; + if (!numeric(a.t) || !numeric(b.t)) return uf.same(a, b); + const d = constDiff(a, b); + if (d !== null) return d === 0; + const scratch = new Map(atoms); + const up = toConstraints('>', a, b, scratch); const dn = toConstraints('<', a, b, scratch); + const si = new Set([...scratch].filter(([, t]) => t && t.t === 'int').map(([k]) => k)); + work(1 + lin.length); + return infeasible([...lin, ...up], si, { n: 1500 }) && infeasible([...lin, ...dn], si, { n: 1500 }); + }; + for (const list of groups.values()) { + if (list.length < 2 || list.length > 12) continue; + for (let i = 0; i < list.length; i++) { + for (let j = i + 1; j < list.length; j++) { + const a = list[i]; const b = list[j]; + work(); + const pk = `${key(a)}~${key(b)}`; + if (merged.has(pk)) continue; + const xa = argsOf(a); const xb = argsOf(b); + if (xa.length !== xb.length) continue; + if (!xa.every((x, n) => entailedEqual(x, xb[n]))) continue; + merged.add(pk); + changed = true; + if (numeric(a.t)) { lin.push(...toConstraints('==', a, b, atoms)); } + else if (a.t === 'bool') { + const ka = key(a); const kb = key(b); + if (bools.has(ka) && bools.has(kb) && bools.get(ka) !== bools.get(kb)) return false; + if (bools.has(ka)) bools.set(kb, bools.get(ka)!); else if (bools.has(kb)) bools.set(ka, bools.get(kb)!); + uf.union(a, b); + } else uf.union(a, b); + } + } + } + // A range that ends at or before it starts is empty. + for (const t of all.values()) { + if (t.k !== 'q') continue; + const pk = `empty:${key(t)}`; + if (merged.has(pk)) continue; + const d = constDiff(t.hi, t.lo); + let empty = d !== null ? d <= 0 : false; + if (d === null) { + const scratch = new Map(atoms); + const over = toConstraints('>', t.hi, t.lo, scratch); + const si = new Set([...scratch].filter(([, x]) => x && x.t === 'int').map(([k]) => k)); + work(1 + lin.length); + empty = infeasible([...lin, ...over], si, { n: 1500 }); + } + if (!empty) continue; + merged.add(pk); + changed = true; + const id = identity(t.q, t.t); + if (!id) continue; + if (numeric(t.t)) lin.push(...toConstraints('==', t, id, atoms)); + else if (t.t === 'bool') { + const k = key(t); + const v = id.k === 'lit' && id.v === true; + if (bools.has(k) && bools.get(k) !== v) return false; + bools.set(k, v); + } else uf.union(t, id); + } + // Equalities between numeric atoms that the union-find learned. + for (const [k, t] of uf.term) { + if (!numeric(t.t)) continue; + const r = uf.find(k); + if (r !== k && numeric(uf.term.get(r)?.t)) { + const pk = `uf:${k}=${r}`; + if (!merged.has(pk)) { merged.add(pk); lin.push(...toConstraints('==', t, uf.term.get(r)!, atoms)); changed = true; } + } + } + // bool atoms that became the same class + if (!changed) return !conflict(); + } + return !conflict(); +} + +/* ------------------------------------------------------------------ */ +/* Quantifier ranges, one position at a time */ +/* ------------------------------------------------------------------ */ + +function quantifiers(x: Term, out: Map = new Map()): Map { + if (!x || typeof x !== 'object') return out; + work(); + if (x.k === 'q' && x.q !== 'SETOF') out.set(key(x), x); + if (x.k === 'q') { quantifiers(x.lo, out); quantifiers(x.hi, out); return out; } + for (const y of fields(x, ['a', 'b', 'c', 'dom'])) quantifiers(y, out); + for (const y of argsIn(x)) quantifiers(y, out); + return out; +} + +/** + * Rewrite Q over [lo, h + d) as Q over [lo, h) extended by d positions, + * whenever another quantifier of the same shape over [lo, h) is present and + * lo <= h follows from the facts. Also folds single-position ranges. + */ +function splitRanges(formulas: Term[], facts: Term[], depth: number): Term[] { + let fs = formulas; + for (let round = 0; round < 4; round++) { + const qs = new Map(); + for (const f of fs) quantifiers(f, qs); + const list = [...qs.values()]; + const replace = new Map(); + for (const a of list) { + if (replace.has(key(a))) continue; + // a single position + const span = constDiff(a.hi, a.lo); + if (span === 1) { const id = identity(a.q, a.t); const one = id && extend(a, id, a.lo); if (one) replace.set(key(a), one); continue; } + for (const b of list) { + work(); + if (a === b || a.q !== b.q || bodyKey(a) !== bodyKey(b) || constDiff(a.lo, b.lo) !== 0) continue; + const d = constDiff(a.hi, b.hi); + if (d === null || d < 1 || d > 3) continue; + if (depth > 0 || proveSimple(facts, cmp('<=', b.lo, b.hi)) !== 'proved') continue; + let v: Term | null = b; + for (let s = 0; s < d && v; s++) v = extend(a, v, add(b.hi, num(s))); + if (!v) continue; + replace.set(key(a), v); + break; + } + } + if (!replace.size) return fs; + fs = fs.map((f) => rebuild(f, (y) => (y.k === 'q' && replace.has(key(y)) ? replace.get(key(y))! : null))); + } + return fs; +} + +/* ------------------------------------------------------------------ */ +/* FORALL, used at particular positions */ +/* ------------------------------------------------------------------ */ + +/** + * A FORALL that holds (or an EXISTS that fails) says something about every + * position in its range. Try it at each whole-number term the case + * mentions: for each such t, either t is outside the range or the body + * holds at t. If every resulting case is impossible, so is this one. + */ +function instantiate(lits: Term[], cap: number): boolean { + const qs: Array<{ q: Quant; neg: boolean }> = []; + for (const l of lits) { + if (l.k === 'q' && l.q === 'FORALL') qs.push({ q: l, neg: false }); + if (l.k === 'not' && l.a.k === 'q' && l.a.q === 'EXISTS') qs.push({ q: l.a, neg: true }); + } + if (!qs.length) return false; + const cands = new Map(); + const visit = (x: Term, inBody: boolean): void => { + if (!x || typeof x !== 'object' || cands.size > 14) return; + work(); + if (x.k === 'sym' && x.t === 'int') cands.set(key(x), x); + if (x.k === 'app' && x.f === 'AT' && !inBody && numeric(x.args[1].t)) cands.set(key(x.args[1]), x.args[1]); + if (x.k === 'q') { visit(x.lo, inBody); visit(x.hi, inBody); visit(x.body, true); return; } + for (const y of fields(x, ABC)) visit(y, inBody); + for (const y of argsIn(x)) visit(y, inBody); + }; + for (const l of lits) visit(l, false); + if (!cands.size) return false; + // Use the quantifier only where the case shows t is inside its range: + // then the body at t is a plain fact, with no new case split. + const inst: Term[] = []; + for (const { q, neg } of qs) { + for (const t of cands.values()) { + const inside = and(cmp('<=', q.lo, t), cmp('<', t, q.hi)); + if (prove(lits, inside, { depth: 1 }).status !== 'proved') continue; + const body = substBound(q.body, q.v, t); + inst.push(neg ? not(body) : body); + } + } + if (!inst.length) return false; + const f = nnf(and([...lits, ...inst])); + const cases = dnf(f, cap * 2); + if (cases === null) return false; + return cases.every((c) => !caseSatisfiable(c)); +} + +/* ------------------------------------------------------------------ */ +/* Entry points */ +/* ------------------------------------------------------------------ */ + +function proveSimple(facts: Term[], goal: Term): ProveResult['status'] { return prove(facts, goal, { depth: 1 }).status; } + +/** + * The work one `prove` may do (logic.ts `work`): far past what any obligation of the corpus or of prove-fuzz takes, + * and reached in about a second (tests/prover-limits.test.mjs). Past it the answer is "unknown", too large to + * prove here. + */ +export const PROVE_WORK = 2_000_000; + +export function prove(facts: Term[], goal: Term, options: ProveOptions = {}): ProveResult { + if (goal.k === 'lit' && goal.v === true) return { status: 'proved', reason: 'the goal is TRUE as written' }; + return metered(options.work ?? PROVE_WORK, () => proveIn(facts, goal, options), () => ({ status: 'unknown', reason: TOO_LARGE })); +} + +function proveIn(facts: Term[], goal: Term, options: ProveOptions): ProveResult { + const depth = options.depth || 0; + const cap = options.cap || 512; + let fs = [...facts, not(goal)]; + fs = splitRanges(fs, facts, depth); + let f = and(fs.map((x) => nnf(x))); + if (f.k === 'lit') return f.v ? { status: 'unknown', reason: 'nothing contradicts the negated goal' } : { status: 'proved', reason: 'the facts contradict each other or the goal is trivial' }; + f = nnf(f); + const cases = dnf(f, cap); + if (cases === null) return { status: 'unknown', reason: 'too many cases to check' }; + for (const c of cases) { + if (!caseSatisfiable(c)) continue; + if (depth === 0 && instantiate(c, cap)) continue; + return { status: 'unknown', reason: 'at least one case could not be ruled out', witness: c }; + } + return { status: 'proved', reason: `${cases.length} ${cases.length === 1 ? 'case' : 'cases'} ruled out` }; +} + +export { Q, infeasible }; diff --git a/src/engines/pseudo/specs.js b/src/engines/pseudo/specs.js new file mode 100644 index 0000000..edbe04d --- /dev/null +++ b/src/engines/pseudo/specs.js @@ -0,0 +1,854 @@ +/** + * pseudo/specs.js + * + * The course's pseudocode, ready to use: example programs for the lab, + * widget specs for the pages, and exercises with model answers. Every entry + * is a plain dictionary, so a lesson can copy one, change a line, and get a + * working widget. + */ + +/* ------------------------------------------------------------------ */ +/* Programs */ +/* ------------------------------------------------------------------ */ + +export const COUNT_BASE = `// How many positions of T hold the base B. +FUNCTION COUNT_BASE(B : CHAR, T : STRING) RETURNS INTEGER + + // Set up + DECLARE N : INTEGER + DECLARE Total : INTEGER + N <- LENGTH(T) + Total <- 0 + + // Visit each position + FOR I <- 0 TO N + + // Test this position + IF T[I] == B THEN + + // Update the running total + Total <- Total + 1 + + ENDIF + NEXT I + + // Report + RETURN Total +ENDFUNCTION`; + +export const COUNT_BASE_PROVED = `// How many positions of T hold the base B, with its specification. +FUNCTION COUNT_BASE(B : CHAR, T : STRING) RETURNS INTEGER + ENSURE RESULT == LENGTH({ K IN [0, LENGTH(T)) : T[K] == B }) + + DECLARE N : INTEGER + DECLARE Total : INTEGER + N <- LENGTH(T) + Total <- 0 + FOR I <- 0 TO N + INVARIANT Total == LENGTH({ K IN [0, I) : T[K] == B }) + IF T[I] == B THEN + Total <- Total + 1 + ENDIF + NEXT I + RETURN Total +ENDFUNCTION`; + +export const MATCH = `// Every position where pattern P occurs exactly in text T. +FUNCTION MATCH(P : STRING, T : STRING) RETURNS LIST OF INTEGER + + // Set up + DECLARE N : INTEGER + DECLARE M : INTEGER + DECLARE Matched : BOOL + DECLARE Answer : LIST OF INTEGER + N <- LENGTH(T) + M <- LENGTH(P) + + // Visit each legal start + FOR I <- 0 TO N - M + 1 + + // Test: does the pattern match here? + Matched <- TRUE + FOR J <- 0 TO M + IF T[I + J] != P[J] THEN + Matched <- FALSE + BREAK + ENDIF + NEXT J + + // Update + IF Matched THEN + APPEND(Answer, I) + ENDIF + NEXT I + + RETURN Answer +ENDFUNCTION`; + +export const MATCH_PROVED = `// MATCH, with the specification from "Specifying the problem", +// and the two invariants that prove it. +FUNCTION MATCH(P : STRING, T : STRING) RETURNS LIST OF INTEGER + REQUIRE LENGTH(P) >= 1 + ENSURE RESULT == SORTED({ S IN [0, LENGTH(T) - LENGTH(P) + 1) : FORALL K IN [0, LENGTH(P)) : T[S + K] == P[K] }) + + DECLARE N : INTEGER + DECLARE M : INTEGER + DECLARE Matched : BOOL + DECLARE Answer : LIST OF INTEGER + N <- LENGTH(T) + M <- LENGTH(P) + FOR I <- 0 TO N - M + 1 + INVARIANT Answer == SORTED({ S IN [0, I) : FORALL K IN [0, M) : T[S + K] == P[K] }) + Matched <- TRUE + FOR J <- 0 TO M + INVARIANT Matched == (FORALL K IN [0, J) : T[I + K] == P[K]) + IF T[I + J] != P[J] THEN + Matched <- FALSE + BREAK + ENDIF + NEXT J + IF Matched THEN + APPEND(Answer, I) + ENDIF + NEXT I + RETURN Answer +ENDFUNCTION`; + +export const MATCH_OFF_BY_ONE = MATCH.replace('FOR I <- 0 TO N - M + 1', 'FOR I <- 0 TO N').replace('// Every position', '// One bound is wrong. Every position'); + +export const MATCH_RESET_BUG = `// Matched is reset in the wrong place. +FUNCTION MATCH(P : STRING, T : STRING) RETURNS LIST OF INTEGER + DECLARE N : INTEGER + DECLARE M : INTEGER + DECLARE Matched : BOOL + DECLARE Answer : LIST OF INTEGER + N <- LENGTH(T) + M <- LENGTH(P) + Matched <- TRUE + FOR I <- 0 TO N - M + 1 + FOR J <- 0 TO M + IF T[I + J] != P[J] THEN + Matched <- FALSE + BREAK + ENDIF + NEXT J + IF Matched THEN + APPEND(Answer, I) + ENDIF + NEXT I + RETURN Answer +ENDFUNCTION`; + +export const FIRST_UNKNOWN = `// The position of the first N in T, or LENGTH(T) when there is none. +FUNCTION FIRST_UNKNOWN(T : STRING) RETURNS INTEGER + ENSURE 0 <= RESULT AND RESULT <= LENGTH(T) + ENSURE FORALL K IN [0, RESULT) : T[K] != 'N' + DECLARE I : INTEGER + I <- 0 + WHILE I < LENGTH(T) AND T[I] != 'N' + INVARIANT FORALL K IN [0, I) : T[K] != 'N' + I <- I + 1 + ENDWHILE + RETURN I +ENDFUNCTION`; + +export const TALLY = `// How many times each base appears in T. +FUNCTION TALLY_BASES(T : STRING) RETURNS HASHMAP OF CHAR : INTEGER + DECLARE Tally : HASHMAP OF CHAR : INTEGER + DECLARE B : CHAR + FOR I <- 0 TO LENGTH(T) + B <- T[I] + IF B IN Tally THEN + Tally[B] <- Tally[B] + 1 + ELSE + Tally[B] <- 1 // first sighting of this base + ENDIF + NEXT I + RETURN Tally +ENDFUNCTION`; + +export const REVERSE_COMPLEMENT = `// The reverse complement of a strand: read backwards, each base swapped for its partner. +FUNCTION REVERSE_COMPLEMENT(T : STRING) RETURNS STRING + ENSURE LENGTH(RESULT) == LENGTH(T) + DECLARE Answer : STRING + DECLARE Partner : CHAR + Answer <- "" + FOR I <- LENGTH(T) - 1 TO -1 STEP -1 + CASE OF T[I] + 'A' : Partner <- 'T' + 'T' : Partner <- 'A' + 'G' : Partner <- 'C' + 'C' : Partner <- 'G' + OTHERWISE : Partner <- 'N' + ENDCASE + Answer <- Answer & Partner + NEXT I + RETURN Answer +ENDFUNCTION`; + +export const GC_FRACTION = `// The fraction of T that is G or C. +FUNCTION GC_FRACTION(T : STRING) RETURNS FLOAT + REQUIRE LENGTH(T) > 0 + DECLARE GC : INTEGER + GC <- 0 + FOR EACH B IN T + IF B == 'G' OR B == 'C' THEN + GC <- GC + 1 + ENDIF + NEXT B + RETURN GC / LENGTH(T) +ENDFUNCTION`; + +export const HAMMING = `// How many positions differ between two reads of the same length. +FUNCTION HAMMING(A : STRING, B : STRING) RETURNS INTEGER + REQUIRE LENGTH(A) == LENGTH(B) + ENSURE RESULT == LENGTH({ K IN [0, LENGTH(A)) : A[K] != B[K] }) + DECLARE Differences : INTEGER + Differences <- 0 + FOR I <- 0 TO LENGTH(A) + INVARIANT Differences == LENGTH({ K IN [0, I) : A[K] != B[K] }) + IF A[I] != B[I] THEN + Differences <- Differences + 1 + ENDIF + NEXT I + RETURN Differences +ENDFUNCTION`; + +export const BEST_POSITION = `// The position of the highest quality score; the first one, if several tie. +FUNCTION BEST_POSITION(Q : LIST OF INTEGER) RETURNS INTEGER + REQUIRE LENGTH(Q) >= 1 + ENSURE 0 <= RESULT AND RESULT < LENGTH(Q) + ENSURE FORALL K IN [0, LENGTH(Q)) : Q[K] <= Q[RESULT] + DECLARE Best : INTEGER + Best <- 0 + FOR I <- 1 TO LENGTH(Q) + INVARIANT 0 <= Best AND Best < I + INVARIANT FORALL K IN [0, I) : Q[K] <= Q[Best] + IF Q[I] > Q[Best] THEN + Best <- I + ENDIF + NEXT I + RETURN Best +ENDFUNCTION`; + +export const KMER_COUNTS = `// How many times each word of K letters occurs in T. +FUNCTION KMER_COUNTS(T : STRING, K : INTEGER) RETURNS HASHMAP OF STRING : INTEGER + REQUIRE K >= 1 + DECLARE Counts : HASHMAP OF STRING : INTEGER + DECLARE Word : STRING + FOR I <- 0 TO LENGTH(T) - K + 1 + Word <- SLICE(T, I, I + K) + IF Word IN Counts THEN + Counts[Word] <- Counts[Word] + 1 + ELSE + Counts[Word] <- 1 + ENDIF + NEXT I + RETURN Counts +ENDFUNCTION`; + +export const BINARY_SEARCH = `// Is X in the sorted list L? Halve the range that could hold it each time. +FUNCTION CONTAINS_SORTED(L : LIST OF INTEGER, X : INTEGER) RETURNS BOOL + REQUIRE FORALL K IN [0, LENGTH(L) - 1) : L[K] <= L[K + 1] + DECLARE Lo : INTEGER + DECLARE Hi : INTEGER + DECLARE Mid : INTEGER + Lo <- 0 + Hi <- LENGTH(L) + WHILE Lo < Hi + INVARIANT 0 <= Lo AND Lo <= Hi AND Hi <= LENGTH(L) + DECREASES Hi - Lo + Mid <- (Lo + Hi) DIV 2 + IF L[Mid] == X THEN + RETURN TRUE + ELSEIF L[Mid] < X THEN + Lo <- Mid + 1 + ELSE + Hi <- Mid + ENDIF + ENDWHILE + RETURN FALSE +ENDFUNCTION`; + +export const SWAP = `// Swap the values held by X and Y. +DECLARE X : INTEGER +DECLARE Y : INTEGER +DECLARE Temp : INTEGER +X <- 4 +Y <- 9 +Temp <- X // hold X, because the next line overwrites it +X <- Y +Y <- Temp +OUTPUT "X is ", X, " and Y is ", Y`; + +export const SWAP_PROCEDURE = `PROCEDURE SWAP(BYREF X : INTEGER, BYREF Y : INTEGER) + ENSURE X == OLD(Y) AND Y == OLD(X) + DECLARE Temp : INTEGER + Temp <- X + X <- Y + Y <- Temp +ENDPROCEDURE + +DECLARE A : INTEGER +DECLARE B : INTEGER +A <- 4 +B <- 9 +CALL SWAP(A, B) +OUTPUT "A is ", A, " and B is ", B`; + +export const ROW_TOTALS = `// Row totals and a grand total for a small table of counts. +DECLARE Counts : ARRAY[3, 4] OF INTEGER +DECLARE Total : INTEGER +FOR Row <- 0 TO 3 + FOR Column <- 0 TO 4 + Counts[Row, Column] <- Row * 4 + Column + NEXT Column +NEXT Row +Total <- 0 +FOR Row <- 0 TO 3 + DECLARE RowTotal : INTEGER + RowTotal <- 0 + FOR Column <- 0 TO 4 + RowTotal <- RowTotal + Counts[Row, Column] + NEXT Column + OUTPUT "Row ", Row, " total is ", RowTotal + Total <- Total + RowTotal +NEXT Row +OUTPUT "Grand total is ", Total`; + +export const THRESHOLD_INPUT = `// Ask until the answer is a legal quality threshold. +DECLARE Threshold : INTEGER +REPEAT + OUTPUT "Enter a quality threshold between 0 and 40" + INPUT Threshold +UNTIL Threshold >= 0 AND Threshold <= 40 +OUTPUT "Using ", Threshold`; + +export const TRANSLATE_CODONS = `// Translate a reading frame into amino acids, stopping at the first stop codon. +TYPE CodonSet = SET OF STRING +DEFINE Stops ("TAA", "TAG", "TGA") : CodonSet + +FUNCTION TRANSLATE(T : STRING) RETURNS STRING + DECLARE Table : HASHMAP OF STRING : CHAR + DECLARE Protein : STRING + DECLARE Codon : STRING + Table["ATG"] <- 'M' + Table["GCC"] <- 'A' + Table["AAA"] <- 'K' + Table["TTT"] <- 'F' + Table["GGC"] <- 'G' + Protein <- "" + FOR I <- 0 TO LENGTH(T) DIV 3 * 3 STEP 3 + Codon <- SLICE(T, I, I + 3) + IF Codon IN Stops THEN + BREAK + ELSEIF Codon IN Table THEN + Protein <- Protein & Table[Codon] + ELSE + Protein <- Protein & 'X' + ENDIF + NEXT I + RETURN Protein +ENDFUNCTION`; + +export const EDIT_DISTANCE = `// The fewest single-letter edits (substitute, insert, delete) that turn A into B. +FUNCTION EDIT_DISTANCE(A : STRING, B : STRING) RETURNS INTEGER + DECLARE N : INTEGER + DECLARE M : INTEGER + N <- LENGTH(A) + M <- LENGTH(B) + DECLARE D : ARRAY[N + 1, M + 1] OF INTEGER + FOR I <- 0 TO N + 1 + D[I, 0] <- I + NEXT I + FOR J <- 0 TO M + 1 + D[0, J] <- J + NEXT J + FOR I <- 1 TO N + 1 + FOR J <- 1 TO M + 1 + DECLARE Cost : INTEGER + IF A[I - 1] == B[J - 1] THEN + Cost <- 0 + ELSE + Cost <- 1 + ENDIF + D[I, J] <- MIN(D[I - 1, J] + 1, D[I, J - 1] + 1, D[I - 1, J - 1] + Cost) + NEXT J + NEXT I + RETURN D[N, M] +ENDFUNCTION`; + +export const FIVE_BUGS = `// Count the G and C bases in T. This version has five mistakes; +// the coach finds all of them before anything runs. +FUNCTION COUNT_GC(T : STRING) RETURNS INTEGER + DECLARE Total : INTEGER + DECLARE Unused : FLOAT + FOR I <- 0 TO LENGTH(T) - 1 + Total <- 0 + IF T[I] == "G" OR T[I] == 'C' THEN + Total = Total + 1 + ENDIF + NEXT I + RETURN Total +ENDFUNCTION`; + +/* ------------------------------------------------------------------ */ +/* Examples, for the lab's menu */ +/* ------------------------------------------------------------------ */ + +export const EXAMPLES = { + count: { title: 'Count a base (the counting lesson)', code: COUNT_BASE, call: 'COUNT_BASE(\'A\', "GATTACA")', tests: 'COUNT_BASE(\'A\', "GATTACA") -> 3\nCOUNT_BASE(\'G\', "") -> 0\nCOUNT_BASE(\'T\', "TTTT") -> 4' }, + countProved: { title: 'Count a base, proved correct', code: COUNT_BASE_PROVED, call: 'COUNT_BASE(\'A\', "GATTACA")', tests: 'COUNT_BASE(\'A\', "GATTACA") -> 3' }, + match: { title: 'Find a pattern (MATCH)', code: MATCH, call: 'MATCH("ATT", "GATTACA")', tests: 'MATCH("ATT", "GATTACA") -> [1]\nMATCH("A", "GATTACA") -> [1, 4, 6]\nMATCH("GATTACAT", "GATTACA") -> []' }, + matchProved: { title: 'MATCH, proved against its specification', code: MATCH_PROVED, call: 'MATCH("ATT", "GATTACA")' }, + matchBound: { title: 'Bug: one bound is wrong', code: MATCH_OFF_BY_ONE, call: 'MATCH("ATT", "GATTACA")' }, + matchReset: { title: 'Bug: a flag reset in the wrong loop', code: MATCH_RESET_BUG, call: 'MATCH("GC", "GATGCA")', tests: 'MATCH("GC", "GATGCA") -> [3]' }, + fiveBugs: { title: 'Bug hunt: five mistakes in one function', code: FIVE_BUGS, call: 'COUNT_GC("GATTACA")' }, + firstUnknown: { title: 'The first unknown base (WHILE)', code: FIRST_UNKNOWN, call: 'FIRST_UNKNOWN("GATNACA")' }, + tally: { title: 'Tally the bases (HASHMAP)', code: TALLY, call: 'TALLY_BASES("GATTACA")' }, + revcomp: { title: 'Reverse complement (CASE, a counting-down loop)', code: REVERSE_COMPLEMENT, call: 'REVERSE_COMPLEMENT("GATTACA")', tests: 'REVERSE_COMPLEMENT("GATTACA") -> "TGTAATC"\nREVERSE_COMPLEMENT("") -> ""' }, + gc: { title: 'GC fraction (FLOAT, REQUIRE)', code: GC_FRACTION, call: 'GC_FRACTION("GATTACA")' }, + hamming: { title: 'Hamming distance, proved', code: HAMMING, call: 'HAMMING("GATTACA", "GACTATA")' }, + best: { title: 'The best quality score, proved', code: BEST_POSITION, call: 'BEST_POSITION([12, 30, 25, 30, 8])' }, + kmers: { title: 'Count every k-mer (SLICE, HASHMAP)', code: KMER_COUNTS, call: 'KMER_COUNTS("GATTACAGATT", 3)' }, + binary: { title: 'Binary search (DECREASES)', code: BINARY_SEARCH, call: 'CONTAINS_SORTED([2, 5, 8, 13, 21], 13)' }, + swap: { title: 'Swap two values (the guide\'s trace)', code: SWAP }, + swapProc: { title: 'SWAP as a BYREF procedure', code: SWAP_PROCEDURE }, + rows: { title: 'Row totals (two-dimensional ARRAY)', code: ROW_TOTALS }, + threshold: { title: 'Validate input (REPEAT ... UNTIL)', code: THRESHOLD_INPUT, input: ['55', '-3', '20'] }, + translate: { title: 'Translate codons (SET, HASHMAP, STEP 3)', code: TRANSLATE_CODONS, call: 'TRANSLATE("ATGGCCAAATAGTTT")' }, + edit: { title: 'Edit distance (a table filled cell by cell)', code: EDIT_DISTANCE, call: 'EDIT_DISTANCE("GATTACA", "GCATGCA")' }, +}; + +/* ------------------------------------------------------------------ */ +/* Widgets for the pages */ +/* ------------------------------------------------------------------ */ + +export const specCountStep = { + title: 'Counting, one step at a time', + blurb: 'Step through COUNT_BASE on GATTACA. The trace table below is the one you would write by hand: one row per step, one column per variable.', + code: COUNT_BASE, call: 'COUNT_BASE(\'A\', "GATTACA")', + views: [['code', { roles: true }], 'variables', 'explain', ['trace', { watch: ['I', 'Total'] }]], +}; + +export const specMatchTrace = { + title: 'MATCH on GATTACA and ATT', + blurb: 'The pointers under T show where I and I + J sit. Watch Matched reset at the top of every outer pass, and BREAK end the inner loop at the first disagreeing letter.', + code: MATCH, call: 'MATCH("ATT", "GATTACA")', + views: ['code', 'variables', 'explain', ['trace', { watch: ['I', 'J', 'Matched', 'Answer'] }]], +}; + +export const specSwap = { + title: 'Three assignments and one temporary name', + code: SWAP, + views: ['code', 'variables', 'explain', ['trace', { watch: ['X', 'Y', 'Temp'] }], 'output'], +}; + +export const specOffByOne = { + title: 'One past the end', + blurb: 'The outer bound is N instead of N - M + 1. Run it to the end and read the error: it names the position asked for, the positions that exist, and the values that produced it.', + code: MATCH_OFF_BY_ONE, call: 'MATCH("ATT", "GATTACA")', start: 'end', + views: ['code', 'variables', 'explain', 'diagnostics'], +}; + +export const specResetBug = { + title: 'A flag reset in the wrong loop', + blurb: 'Matched is set to TRUE once, before the outer loop. The coach flags it before anything runs; the run shows the cost: once one start fails, every later start inherits FALSE.', + code: MATCH_RESET_BUG, call: 'MATCH("GC", "GATGCA")', + views: ['code', 'variables', 'diagnostics', ['trace', { watch: ['I', 'J', 'Matched', 'Answer'] }]], +}; + +export const specWhileScan = { + title: 'Scan until something is found', + code: FIRST_UNKNOWN, call: 'FIRST_UNKNOWN("GATNACA")', + views: ['code', 'variables', 'explain', ['trace', { watch: ['I'] }]], +}; + +export const specTally = { + title: 'A HASHMAP builds a tally', + code: TALLY, call: 'TALLY_BASES("GATTACA")', + views: ['code', 'variables', 'explain', ['trace', { watch: ['I', 'B', 'Tally'] }]], +}; + +export const specFlowMatch = { + title: 'How values flow through MATCH', + blurb: 'Solid arrows carry values; dashed arrows are tests that decide whether a line runs. Step through and watch which arrows light up.', + code: MATCH, call: 'MATCH("ATT", "GATTACA")', + views: ['code', 'variables', 'defuse', 'flow'], +}; + +export const specJobsMatch = { + title: 'The four jobs in MATCH', + code: MATCH, call: 'MATCH("ATT", "GATTACA")', + views: [['code', { roles: true }], 'jobs', 'explain'], +}; + +export const specCfgCount = { + title: 'COUNT_BASE as a flowchart', + code: COUNT_BASE, call: 'COUNT_BASE(\'A\', "GAT")', + views: ['code', 'cfg', 'explain'], +}; + +export const specProofCount = { + title: 'Proving COUNT_BASE correct', + blurb: 'The ENSURE says what the answer is; the INVARIANT says what is true at the top of every pass. The checker proves the invariant holds before the first pass, survives every pass, and turns into the ENSURE when the loop ends.', + code: COUNT_BASE_PROVED, call: 'COUNT_BASE(\'A\', "GATTACA")', + views: ['code', ['proof', { open: true }], 'explain', ['trace', { watch: ['I', 'Total'], checks: true }]], +}; + +export const specProofMatch = { + title: 'Proving MATCH against its specification', + code: MATCH_PROVED, call: 'MATCH("ATT", "GATTACA")', + views: ['code', 'proof'], +}; + +export const specProofBug = { + title: 'What the checker cannot prove, a run breaks', + blurb: 'The guard sits on the wrong side of AND. The checker cannot prove the read of T[I] is safe, and the generated inputs find one that breaks it.', + code: `FUNCTION FIRST_UNKNOWN(T : STRING) RETURNS INTEGER + DECLARE I : INTEGER + I <- 0 + WHILE T[I] != 'N' AND I < LENGTH(T) + I <- I + 1 + ENDWHILE + RETURN I +ENDFUNCTION`, + call: 'FIRST_UNKNOWN("GATNACA")', + views: ['code', ['proof', { open: true }], 'diagnostics'], +}; + +/** For the growth view: a random text of length n, and a pattern of length 4. */ +const randomDna = (n, rnd) => { let s = ''; for (let i = 0; i < n; i++) s += 'ACGT'[Math.floor(rnd() * 4)]; return s; }; +export const specGrowthMatch = { + title: 'How MATCH\'s work grows', + blurb: 'Letter comparisons for a 4-letter pattern in random DNA of growing length. The count grows in step with the text: each start fails after about 1.33 comparisons on average.', + code: MATCH, call: 'MATCH("GATT", "GATTACA")', + views: [['growth', { measure: 'letterCompares', sizes: [100, 200, 400, 800, 1600, 3200], args: (n, rnd) => ['GATT', randomDna(n, rnd)], xLabel: 'text length n' }]], + layout: 'one-column', +}; +export const specGrowthWorst = { + title: 'The worst case: a pattern of all A in a text of all A', + code: MATCH, call: 'MATCH("AAAA", "AAAAAAAA")', + views: [['growth', { measure: 'letterCompares', sizes: [4, 8, 16, 32, 64, 128], args: (n) => ['A'.repeat(n / 2), 'A'.repeat(n)], xLabel: 'text length n, with a pattern of length n / 2' }]], + layout: 'one-column', +}; + +export const specTranslateMatch = { + title: 'MATCH in Mojo, line for line', + code: MATCH, call: 'MATCH("ATT", "GATTACA")', + views: ['translate', 'variables', 'explain'], +}; + +export const specRows = { + title: 'A two-dimensional ARRAY', + code: ROW_TOTALS, + views: ['code', 'variables', 'output', ['trace', { watch: ['Row', 'Column', 'RowTotal', 'Total'] }]], +}; + +export const specEdit = { + title: 'Filling a table cell by cell', + blurb: 'Every cell of D is computed from three cells already filled: above, to the left, and diagonally. The variables panel shows the table as it fills.', + code: EDIT_DISTANCE, call: 'EDIT_DISTANCE("GATT", "GCAT")', + views: ['code', 'variables', 'explain'], +}; + +export const PAGE_SPECS = { + specCountStep, specMatchTrace, specSwap, specOffByOne, specResetBug, specWhileScan, specTally, + specFlowMatch, specJobsMatch, specCfgCount, specProofCount, specProofMatch, specProofBug, + specGrowthMatch, specGrowthWorst, specTranslateMatch, specRows, specEdit, +}; + +/* ------------------------------------------------------------------ */ +/* Exercises */ +/* ------------------------------------------------------------------ */ + +/** + * An exercise: a prompt, a starting point, tests the student can see, and a + * model answer the checker compares against on hundreds of inputs the + * student cannot see. `pattern` names the scan the task calls for, so the + * four-jobs report can say which job is missing. + */ +export const EXERCISES = { + countGC: { + title: 'Count the G and C bases', + prompt: ['Write COUNT_GC(T : STRING) RETURNS INTEGER: how many positions of T hold G or C.', 'Start from the plan below. The test needs two comparisons joined with OR.'], + name: 'COUNT_GC', pattern: 'count', + starter: `FUNCTION COUNT_GC(T : STRING) RETURNS INTEGER + // Set up + + // Visit each position + + // Test + + // Update + + // Report + +ENDFUNCTION`, + tests: 'COUNT_GC("GATTACA") -> 2\nCOUNT_GC("") -> 0\nCOUNT_GC("GGCC") -> 4', + reference: `FUNCTION COUNT_GC(T : STRING) RETURNS INTEGER + DECLARE Total : INTEGER + Total <- 0 + FOR I <- 0 TO LENGTH(T) + IF T[I] == 'G' OR T[I] == 'C' THEN + Total <- Total + 1 + ENDIF + NEXT I + RETURN Total +ENDFUNCTION`, + hints: ['A total that belongs to the whole scan is set to 0 before the loop, never inside it.', 'FOR I <- 0 TO LENGTH(T) visits every position: FOR stops one short of its bound.', "Each side of OR needs its own comparison: T[I] == 'G' OR T[I] == 'C'. A CHAR is written in single quotes."], + }, + cpg: { + title: 'Find every CpG site', + prompt: ['Write CPG_SITES(T : STRING) RETURNS LIST OF INTEGER: every position I where T[I] is C and the next base is G.', 'Careful with the loop bound: the test reads T[I + 1].'], + name: 'CPG_SITES', pattern: 'find-all', + starter: `FUNCTION CPG_SITES(T : STRING) RETURNS LIST OF INTEGER + DECLARE Answer : LIST OF INTEGER + // Visit every position where a two-letter site could start + + RETURN Answer +ENDFUNCTION`, + tests: 'CPG_SITES("ACGTCG") -> [1, 4]\nCPG_SITES("C") -> []\nCPG_SITES("") -> []', + reference: `FUNCTION CPG_SITES(T : STRING) RETURNS LIST OF INTEGER + DECLARE Answer : LIST OF INTEGER + FOR I <- 0 TO LENGTH(T) - 1 + IF T[I] == 'C' AND T[I + 1] == 'G' THEN + APPEND(Answer, I) + ENDIF + NEXT I + RETURN Answer +ENDFUNCTION`, + hints: ['A site starting at I uses positions I and I + 1, so I + 1 has to exist: I <= LENGTH(T) - 2.', 'The legal starts are [0, LENGTH(T) - 1), so the FOR bound is LENGTH(T) - 1.', 'An empty LIST is the right answer when there are no sites; it needs no special case.'], + }, + hamming: { + title: 'Hamming distance', + prompt: ['Write HAMMING(A : STRING, B : STRING) RETURNS INTEGER: how many positions differ between two reads of the same length.', 'State the assumption about the lengths with REQUIRE.'], + name: 'HAMMING', pattern: 'count', + starter: `FUNCTION HAMMING(A : STRING, B : STRING) RETURNS INTEGER + REQUIRE + +ENDFUNCTION`, + tests: 'HAMMING("GATTACA", "GACTATA") -> 2\nHAMMING("", "") -> 0\nHAMMING("AAAA", "TTTT") -> 4', + reference: HAMMING, + hints: ['REQUIRE LENGTH(A) == LENGTH(B) makes the promise explicit. The checker then holds every caller to it.', 'One position I serves both sequences: A[I] and B[I] sit side by side.'], + }, + revcomp: { + title: 'Reverse complement', + prompt: ['Write REVERSE_COMPLEMENT(T : STRING) RETURNS STRING. Read T from its last base to its first, and write each base\'s partner: A with T, G with C. Any other letter becomes N.'], + name: 'REVERSE_COMPLEMENT', pattern: 'build', + starter: `FUNCTION REVERSE_COMPLEMENT(T : STRING) RETURNS STRING + DECLARE Answer : STRING + Answer <- "" + + RETURN Answer +ENDFUNCTION`, + tests: 'REVERSE_COMPLEMENT("GATTACA") -> "TGTAATC"\nREVERSE_COMPLEMENT("") -> ""\nREVERSE_COMPLEMENT("NAC") -> "GTN"', + reference: REVERSE_COMPLEMENT, + hints: ['A loop that walks backwards: FOR I <- LENGTH(T) - 1 TO -1 STEP -1 visits LENGTH(T) - 1 down to 0, stopping one short of -1.', 'A CASE ... ENDCASE on T[I] lays the pairing out as a table; OTHERWISE catches every other letter.', 'Answer <- Answer & Partner adds one letter to the end.'], + }, + kmer: { + title: 'Count a word', + prompt: ['Write COUNT_WORD(T : STRING, W : STRING) RETURNS INTEGER: how many times the word W occurs in T, overlaps included.', 'COUNT_WORD("AAAA", "AA") is 3.'], + name: 'COUNT_WORD', pattern: 'count', + starter: `FUNCTION COUNT_WORD(T : STRING, W : STRING) RETURNS INTEGER + REQUIRE LENGTH(W) >= 1 + +ENDFUNCTION`, + tests: 'COUNT_WORD("AAAA", "AA") -> 3\nCOUNT_WORD("GATTACA", "TA") -> 1\nCOUNT_WORD("GA", "GATTACA") -> 0', + reference: `FUNCTION COUNT_WORD(T : STRING, W : STRING) RETURNS INTEGER + DECLARE Total : INTEGER + Total <- 0 + FOR I <- 0 TO LENGTH(T) - LENGTH(W) + 1 + IF SLICE(T, I, I + LENGTH(W)) == W THEN + Total <- Total + 1 + ENDIF + NEXT I + RETURN Total +ENDFUNCTION`, + hints: ['A word of length M starting at I covers [I, I + M): the last start that fits is LENGTH(T) - M.', 'SLICE(T, I, I + LENGTH(W)) is the window starting at I, as a STRING you can compare with W.', 'When W is longer than T, the bound is below 0 and the loop runs zero times, which is the right answer.'], + }, + tally2: { + title: 'Tally the dinucleotides', + prompt: ['Write TALLY_PAIRS(T : STRING) RETURNS HASHMAP OF STRING : INTEGER: how many times each two-letter word occurs in T.'], + name: 'TALLY_PAIRS', pattern: 'tally', + starter: `FUNCTION TALLY_PAIRS(T : STRING) RETURNS HASHMAP OF STRING : INTEGER + DECLARE Tally : HASHMAP OF STRING : INTEGER + + RETURN Tally +ENDFUNCTION`, + tests: 'TALLY_PAIRS("GATTA") -> {"AT": 1, "GA": 1, "TA": 1, "TT": 1}', + reference: KMER_COUNTS.replace(/KMER_COUNTS\(T : STRING, K : INTEGER\)/, 'TALLY_PAIRS(T : STRING)').replace(' REQUIRE K >= 1\n', '').replace(/\bK\b/g, '2').replace(/Counts/g, 'Tally').replace('How many times each word of 2 letters occurs', 'How many times each pair occurs'), + hints: ['A key that is not in the HASHMAP yet has no count to add to: test Pair IN Tally first.', 'The two-letter word at I is SLICE(T, I, I + 2); the last start is LENGTH(T) - 2.'], + }, + firstStop: { + title: 'The first stop codon', + prompt: ['Write FIRST_STOP(T : STRING) RETURNS INTEGER: the position of the first codon, reading in frame from position 0, that is TAA, TAG, or TGA. Return LENGTH(T) when there is none.'], + name: 'FIRST_STOP', pattern: 'find-first', + starter: `TYPE CodonSet = SET OF STRING +DEFINE Stops ("TAA", "TAG", "TGA") : CodonSet + +FUNCTION FIRST_STOP(T : STRING) RETURNS INTEGER + +ENDFUNCTION`, + tests: 'FIRST_STOP("ATGTAAGG") -> 3\nFIRST_STOP("ATGGCC") -> 6\nFIRST_STOP("AT") -> 2', + reference: `TYPE CodonSet = SET OF STRING +DEFINE Stops ("TAA", "TAG", "TGA") : CodonSet + +FUNCTION FIRST_STOP(T : STRING) RETURNS INTEGER + FOR I <- 0 TO LENGTH(T) DIV 3 * 3 STEP 3 + IF SLICE(T, I, I + 3) IN Stops THEN + RETURN I + ENDIF + NEXT I + RETURN LENGTH(T) +ENDFUNCTION`, + hints: ['STEP 3 visits codon starts: 0, 3, 6.', 'LENGTH(T) DIV 3 * 3 is the length of the whole codons; a bound of LENGTH(T) would let I + 3 run past the end.', 'RETURN inside the loop ends the function at the first stop, which is exactly the first one.'], + }, + longestRun: { + title: 'The longest run of one base', + prompt: ['Write LONGEST_RUN(T : STRING) RETURNS INTEGER: the length of the longest stretch of T in which every base is the same. LONGEST_RUN("GAAATTA") is 3.', 'This one needs two accumulators: the current run, and the best so far.'], + name: 'LONGEST_RUN', pattern: 'find-best', + starter: `FUNCTION LONGEST_RUN(T : STRING) RETURNS INTEGER + +ENDFUNCTION`, + tests: 'LONGEST_RUN("GAAATTA") -> 3\nLONGEST_RUN("") -> 0\nLONGEST_RUN("C") -> 1', + reference: `FUNCTION LONGEST_RUN(T : STRING) RETURNS INTEGER + DECLARE Best : INTEGER + DECLARE Run : INTEGER + Best <- 0 + Run <- 0 + FOR I <- 0 TO LENGTH(T) + IF I > 0 AND T[I] == T[I - 1] THEN + Run <- Run + 1 + ELSE + Run <- 1 + ENDIF + Best <- MAX(Best, Run) + NEXT I + RETURN Best +ENDFUNCTION`, + hints: ['Run is the length of the stretch ending at I: it grows by one when T[I] equals T[I - 1], and starts again at 1 when it does not.', 'At position 0 there is no T[I - 1]: put I > 0 on the left of AND, so the read never happens there.', 'Best <- MAX(Best, Run) after every position keeps the longest seen so far.'], + }, +}; + +/* ------------------------------------------------------------------ */ +/* Activities */ +/* ------------------------------------------------------------------ */ + +export const PARSONS_COUNT = { + title: 'Assemble COUNT_BASE', + prompt: 'Put the lines in order. Two of them do not belong. Any order that works is accepted: the lab runs what you build rather than comparing it with one answer.', + name: 'COUNT_BASE', + lines: [ + 'FUNCTION COUNT_BASE(B : CHAR, T : STRING) RETURNS INTEGER', + ' DECLARE Total : INTEGER', + ' Total <- 0', + ' FOR I <- 0 TO LENGTH(T)', + ' IF T[I] == B THEN', + ' Total <- Total + 1', + ' ENDIF', + ' NEXT I', + ' RETURN Total', + 'ENDFUNCTION', + ], + distractors: [' FOR I <- 0 TO LENGTH(T) - 1', ' Total <- 0'], + tests: 'COUNT_BASE(\'A\', "GATTACA") -> 3\nCOUNT_BASE(\'A\', "") -> 0\nCOUNT_BASE(\'A\', "CCA") -> 1', + reference: COUNT_BASE, +}; + +export const PREDICT_MATCH = { + title: 'Trace MATCH by hand', + prompt: 'Fill in the value of each variable after each step, the way the guide\'s tables do, then check. Blank cells mean the value did not change.', + code: MATCH, call: 'MATCH("ATT", "GATA")', + watch: ['I', 'J', 'Matched'], + kinds: ['assign', 'next', 'loop-start'], + maxRows: 16, +}; + +export const PREDICT_COUNT = { + title: 'Predict, then run', + prompt: 'What does each call return? Write your prediction, then check.', + code: COUNT_BASE, + calls: ['COUNT_BASE(\'A\', "GATTACA")', 'COUNT_BASE(\'A\', "")', 'COUNT_BASE(\'N\', "GATTACA")'], +}; + +export const FILL_MATCH = { + title: 'Fill in the bounds', + prompt: 'MATCH with its two loop bounds left blank. Fill them in so every test passes, and no position outside T is ever read.', + name: 'MATCH', + code: MATCH.replace('FOR I <- 0 TO N - M + 1', 'FOR I <- 0 TO ').replace('FOR J <- 0 TO M', 'FOR J <- 0 TO '), + tests: 'MATCH("ATT", "GATTACA") -> [1]\nMATCH("A", "GATTACA") -> [1, 4, 6]\nMATCH("GATTACAT", "GATTACA") -> []', + reference: MATCH, +}; + +export const ACTIVITIES = { PARSONS_COUNT, PREDICT_MATCH, PREDICT_COUNT, FILL_MATCH }; + +/* ------------------------------------------------------------------ */ +/* From a sentence to steps (sharpen.js) */ +/* ------------------------------------------------------------------ */ + +const MATCH_STEPS = `To find every start of P in T: + Start with an empty list called Starts. + For each position i from 0 up to and including the length of T minus the length of P: + If P matches at i, add i to Starts. + The answer is Starts. + +P matches at i when for every j from 0 up to but not including the length of P, + the letter of P at j is the letter of T at i + j. +`; + +export const SHARPEN = { + // an empty ladder, for students' own problems + blank: { id: 'sharpen-blank' }, + // the lesson's problem, worked all the way through + match: { + id: 'sharpen-match-worked', + title: 'Where does a pattern occur?', + question: 'Where in this sequence does this shorter sequence occur?', + decided: { 'answer-kind': 0, 'match-kind': 0, 'which-position': 0, empty: 0, overlap: 0, strand: 0, case: 2, 'unknown-base': 0 }, + sharpened: 'List every position of T at which P starts, when every letter of P matches exactly, including appearances that overlap, on the sequence as written, assuming the letters are uppercase, with N counted as no base; an empty list if there are none.', + inputs: [{ name: 'P', kind: 'a text', meaning: 'the pattern' }, { name: 'T', kind: 'a text', meaning: 'the sequence to search' }], + answer: { kind: 'a list of whole numbers', meaning: 'every start of P in T, left to right' }, + examples: [ + { values: { P: '"ATT"', T: '"GATTACA"' }, expect: '[1]' }, + { values: { P: '"AA"', T: '"AAAA"' }, expect: '[0, 1, 2]' }, + { values: { P: '"GATTACAT"', T: '"GAT"' }, expect: '[]' }, + ], + steps: MATCH_STEPS, + start: 'say', + }, + // the practice problems, as ladders that start from a sentence +}; + +// The practice problems, each as a ladder that starts from a loose sentence. +const ladder = (key, id, title, question, inputs, answer, examples) => ({ + id: `sharpen-${id}`, title, question, inputs, answer, examples, + reference: { code: EXERCISES[key].reference, name: EXERCISES[key].name }, start: 'say', +}); +Object.assign(SHARPEN, { + countGC: ladder('countGC', 'countgc', 'How GC-rich is this read?', 'How many G and C bases are in this read?', + [{ name: 'T', kind: 'a text', meaning: 'the read' }], { kind: 'a whole number', meaning: '' }, + [{ values: { T: '"GATTACA"' }, expect: '2' }, { values: { T: '""' }, expect: '0' }]), + cpg: ladder('cpg', 'cpg', 'Where are the CpG sites?', 'Find the CpG sites in this promoter.', + [{ name: 'T', kind: 'a text', meaning: 'the promoter' }], { kind: 'a list of whole numbers', meaning: '' }, + [{ values: { T: '"ACGTCG"' }, expect: '[1, 4]' }]), + hamming: ladder('hamming', 'hamming', 'How different are two reads?', 'How different are these two reads of the same region?', + [{ name: 'A', kind: 'a text', meaning: '' }, { name: 'B', kind: 'a text', meaning: '' }], { kind: 'a whole number', meaning: '' }, + [{ values: { A: '"GATTACA"', B: '"GACTATA"' }, expect: '2' }]), + revcomp: ladder('revcomp', 'revcomp', 'The other strand', 'Give me the other strand of this sequence.', + [{ name: 'T', kind: 'a text', meaning: 'the strand' }], { kind: 'a text', meaning: '' }, + [{ values: { T: '"GATTACA"' }, expect: '"TGTAATC"' }]), + kmer: ladder('kmer', 'kmer', 'How often does a word occur?', 'How often does this word occur in the genome?', + [{ name: 'T', kind: 'a text', meaning: 'the genome' }, { name: 'W', kind: 'a text', meaning: 'the word' }], { kind: 'a whole number', meaning: '' }, + [{ values: { T: '"AAAA"', W: '"AA"' }, expect: '3' }]), + firstStop: ladder('firstStop', 'firststop', 'Where does translation stop?', 'Where is the first stop codon in this gene?', + [{ name: 'T', kind: 'a text', meaning: 'the gene' }], { kind: 'a whole number', meaning: '' }, + [{ values: { T: '"ATGTAAGG"' }, expect: '3' }]), + longestRun: ladder('longestRun', 'longestrun', 'The longest run', 'Find the longest run of one base in this read.', + [{ name: 'T', kind: 'a text', meaning: 'the read' }], { kind: 'a whole number', meaning: '' }, + [{ values: { T: '"GAAATTA"' }, expect: '3' }]), +}); + +export const STEPS_EXAMPLES = { + match: { steps: MATCH_STEPS, values: { P: 'ATT', T: 'GATTACA' }, caption: 'find every start of "ATT" in "GATTACA"' }, +}; diff --git a/src/engines/pseudo/study.js b/src/engines/pseudo/study.js new file mode 100644 index 0000000..e0caa88 --- /dev/null +++ b/src/engines/pseudo/study.js @@ -0,0 +1,83 @@ +/** + * pseudo/study.js + * + * One call that reads a program the way the lab does: parse it, check its + * names and types, analyze its data flow, and let the coach look it over. + * + * study(source, options) -> { + * source, parsed, program, analysis, + * diags, every finding, errors first within each line + * errors, the findings that stop the program running + * blanks, template blanks still to fill + * ok, true when the program can run + * } + * + * options.coach: false skips the coach (the tests and the verifier do not need it). + */ + +import { parse } from './parse.js'; +import { check } from './check.js'; +import { analyze } from './analyze.js'; +import { coach } from './coach.js'; + +const RANK = { error: 0, warning: 1, note: 2 }; + +export function study(source, options = {}) { + const parsed = parse(source); + const { program, diags } = check(parsed); + const errors = diags.filter((d) => d.severity === 'error'); + let analysis = null; + let coached = []; + // Analysis needs names resolved; it still runs on a program with type + // errors, since the data flow is usually intact. + const structural = parsed.diags.some((d) => d.severity === 'error' && ['missing-closer', 'stray-closer', 'next-mismatch'].includes(d.code)); + if (!structural) { + try { + analysis = analyze(program); + if (options.coach !== false) coached = coach(program, analysis, options); + } catch (err) { + if (options.strict) throw err; + analysis = null; + } + } + // A read-before-assignment the coach can prove on every path is worth + // knowing about even when other errors exist; the rest wait until the + // errors are fixed, so a student sees one layer at a time. + const all = [...diags, ...coached].sort((a, b) => a.line - b.line || RANK[a.severity] - RANK[b.severity] || (a.col || 0) - (b.col || 0)); + return { + source, parsed, program, analysis, + diags: all, + errors, + blanks: program.blanks, + ok: errors.length === 0, + }; +} + +export function severityCounts(diags) { + const c = { error: 0, warning: 0, note: 0 }; + for (const d of diags) c[d.severity]++; + return c; +} + +/** + * Apply the mechanical fixes some findings carry (keyword case, = for <-, + * a CHAR in double quotes). Returns the new source and how many changed. + */ +export function applyFixes(source, diags) { + const lines = String(source).replace(/\r\n?/g, '\n').split('\n'); + const fixes = diags.filter((d) => d.fix && d.fix.col != null && d.fix.endCol != null) + .map((d) => d.fix) + .sort((a, b) => b.line - a.line || b.col - a.col); + let n = 0; + const done = new Set(); + for (const f of fixes) { + const key = `${f.line}:${f.col}`; + if (done.has(key)) continue; + done.add(key); + const l = lines[f.line - 1]; + if (l == null) continue; + lines[f.line - 1] = l.slice(0, f.col - 1) + f.text + l.slice(f.endCol - 1); + n++; + } + return { source: lines.join('\n'), applied: n }; +} diff --git a/src/engines/pseudo/testing.js b/src/engines/pseudo/testing.js new file mode 100644 index 0000000..249e530 --- /dev/null +++ b/src/engines/pseudo/testing.js @@ -0,0 +1,424 @@ +/** + * pseudo/testing.js + * + * Checking a program by running it: on cases a student writes down, on + * hundreds of generated inputs, against a model answer, and at growing + * sizes to see how the work grows. + * + * parseTests(text) -> [{ line, call | input, expected }] + * runTests(program, tests) -> [{ ..., status, got, error }] + * checkProperties(program, name, options) -> { checked, skipped, failures, cases } + * compareWith(program, reference, name, options) -> { checked, differences } + * measureGrowth(program, name, options) -> { points, fit } + * + * A test line is a call, an arrow, and the value it should give: + * + * COUNT('A', "GATTACA") -> 3 + * MATCH("ATT", "GATTACA") -> [1] + * MATCH("", "GATTACA") -> ERROR any runtime error counts as a pass + * + * A program with no subroutine is tested through its input and output: + * + * INPUT "GATTACA", 'A' -> OUTPUT "3" + */ + +import { parseExpression, sourceOf } from './parse.js'; +import { run, evaluateConstant, makeRandom, valueFits } from './interp.js'; +import { valuesEqual, formatValue, outputText, typeName, T, cloneValue, PArray, intRange, knownDims } from './lang.js'; + +const up = (s) => String(s).toUpperCase(); + +/* ------------------------------------------------------------------ */ +/* Written tests */ +/* ------------------------------------------------------------------ */ + +function splitArrow(line) { + let depth = 0; let q = null; + for (let i = 0; i < line.length - 1; i++) { + const c = line[i]; + if (q) { if (c === '\\') i++; else if (c === q) q = null; continue; } + if (c === '"' || c === "'") { q = c; continue; } + if ('([{'.includes(c)) depth++; + if (')]}'.includes(c)) depth--; + if (depth === 0 && c === '-' && line[i + 1] === '>') return [line.slice(0, i), line.slice(i + 2)]; + } + return null; +} + +/** Split a comma list at the top level, respecting quotes and brackets. */ +function splitTop(text) { + const out = []; let depth = 0; let q = null; let start = 0; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + if (q) { if (c === '\\') i++; else if (c === q) q = null; continue; } + if (c === '"' || c === "'") { q = c; continue; } + if ('([{'.includes(c)) depth++; + if (')]}'.includes(c)) depth--; + if (depth === 0 && c === ',') { out.push(text.slice(start, i)); start = i + 1; } + } + if (text.slice(start).trim().length) out.push(text.slice(start)); + return out.map((s) => s.trim()); +} + +export function parseTests(text) { + const tests = []; + String(text || '').replace(/\r\n?/g, '\n').split('\n').forEach((raw, i) => { + const line = raw.replace(/\/\/.*$/, '').trim(); + if (!line) return; + const parts = splitArrow(line); + const t = { line: i + 1, source: line }; + if (!parts) { t.problem = 'A test is a call, an arrow ->, and the value it should give: COUNT(\'A\', "GATTACA") -> 3.'; tests.push(t); return; } + const [lhs, rhs] = parts.map((s) => s.trim()); + if (/^INPUT\b/i.test(lhs) || /^OUTPUT\b/i.test(rhs)) { + t.kind = 'io'; + t.input = splitTop(lhs.replace(/^INPUT\b/i, '')).map(literalText); + t.output = splitTop(rhs.replace(/^OUTPUT\b/i, '')).map(literalText); + tests.push(t); + return; + } + const call = parseExpression(lhs); + if (call.diags.some((d) => d.severity === 'error') || call.expr.kind !== 'CallExpr') { + t.problem = `The left side of a test is a call, such as COUNT('A', "GATTACA"); "${lhs}" is not one.`; + tests.push(t); + return; + } + t.kind = 'call'; + t.call = call.expr; + if (/^ERROR\b/i.test(rhs)) { t.expectError = rhs.replace(/^ERROR\s*/i, '').trim().toLowerCase() || true; tests.push(t); return; } + const exp = parseExpression(rhs); + if (exp.diags.some((d) => d.severity === 'error')) { t.problem = `The value after -> could not be read: ${exp.diags[0].message}`; tests.push(t); return; } + t.expectedExpr = exp.expr; + tests.push(t); + }); + return tests; +} + +/** "3" and 3 both mean the text 3 in an INPUT or OUTPUT list. */ +function literalText(s) { + const t = s.trim(); + const m = /^"((?:[^"\\]|\\.)*)"$/.exec(t) || /^'((?:[^'\\]|\\.)*)'$/.exec(t); + if (m) return m[1].replace(/\\n/g, '\n').replace(/\\t/g, '\t').replace(/\\(.)/g, '$1'); + return t; +} + +/** Numbers match with a little room, since two FLOAT routes rarely agree to the last digit. */ +export function sameResult(a, b, type) { + if (typeof a === 'number' && typeof b === 'number') { + if (type?.k === 'FLOAT' || !Number.isInteger(a) || !Number.isInteger(b)) return Math.abs(a - b) <= 1e-9 * Math.max(1, Math.abs(a), Math.abs(b)); + return a === b; + } + if (Array.isArray(a) && Array.isArray(b)) return a.length === b.length && a.every((x, i) => sameResult(x, b[i], type?.of)); + return valuesEqual(a, b); +} + +export function runTests(program, tests, options = {}) { + return tests.map((t) => { + if (t.problem) return { ...t, status: 'invalid', message: t.problem }; + if (t.kind === 'io') { + const trace = run(program, { input: t.input, maxSteps: options.maxSteps ?? 200000, record: options.record ?? false, contracts: options.contracts ?? true }); + if (trace.error) return { ...t, status: 'error', error: trace.error, got: trace.output, trace, message: `Stopped with an error on line ${trace.error.line}: ${trace.error.message}` }; + const ok = trace.output.length === t.output.length && trace.output.every((l, i) => l === t.output[i]); + return { ...t, status: ok ? 'pass' : 'fail', got: trace.output, trace, message: ok ? 'Output matches.' : `Expected ${t.output.map((x) => JSON.stringify(x)).join(', ')}; the program showed ${trace.output.map((x) => JSON.stringify(x)).join(', ') || 'nothing'}.` }; + } + const sub = program.subs.get(up(t.call.name)); + if (!sub) return { ...t, status: 'invalid', message: `The program has no FUNCTION or PROCEDURE called ${t.call.name}.` }; + let args; + try { args = t.call.args.map((a) => evaluateConstant(program, typed(a, sub.params[t.call.args.indexOf(a)]?.type))); } catch (e) { return { ...t, status: 'invalid', message: `An argument could not be read: ${e.message}` }; } + if (args.length !== sub.params.length) return { ...t, status: 'invalid', message: `${sub.name} takes ${sub.params.length} values, and the test gives ${args.length}.` }; + args = args.map((v, i) => adapt(v, sub.params[i].type)); + const bad = sub.params.findIndex((p, i) => !valueFits(p.type, args[i])); + if (bad >= 0) return { ...t, status: 'invalid', message: `Parameter ${sub.params[bad].name} is ${typeName(sub.params[bad].type)}, and the test gives ${sourceOf(t.call.args[bad])}.` }; + const trace = run(program, { call: { name: sub.name, args }, maxSteps: options.maxSteps ?? 200000, record: options.record ?? false, contracts: options.contracts ?? true }); + if (t.expectError) { + const ok = Boolean(trace.error) && (t.expectError === true || trace.error.kind === t.expectError); + return { ...t, status: ok ? 'pass' : 'fail', got: trace.result, error: trace.error, trace, message: ok ? `Stopped with an error, as expected: ${trace.error.message}` : trace.error ? `Stopped with a ${trace.error.kind} error, and the test expected ${t.expectError}.` : `Expected an error; ${sub.name} returned ${formatValue(trace.result, sub.returns)}.` }; + } + if (trace.error) return { ...t, status: 'error', error: trace.error, trace, message: `Stopped with an error on line ${trace.error.line}: ${trace.error.message}` }; + let expected; + try { expected = adapt(evaluateConstant(program, t.expectedExpr), sub.returns); } catch (e) { return { ...t, status: 'invalid', message: `The expected value could not be read: ${e.message}` }; } + const got = sub.kind === 'Procedure' ? undefined : trace.result; + const ok = sameResult(got, expected, sub.returns); + return { + ...t, status: ok ? 'pass' : 'fail', got, expected, trace, + message: ok ? `${sub.name} returned ${formatValue(got, sub.returns)}.` : `Expected ${formatValue(expected, sub.returns)}; ${sub.name} returned ${formatValue(got, sub.returns)}.`, + }; + }); +} + +/** Literal arguments take the parameter's type, so [] is an empty LIST OF whatever it needs. */ +function typed(expr) { return expr; } +function adapt(v, type) { + if (!type) return v; + if (type.k === 'ARRAY' && Array.isArray(v)) { const a = new PArray([v.length]); a.data = v.slice(); return a; } + if (type.k === 'SET' && Array.isArray(v)) return new Set(v); + return v; +} + +/* ------------------------------------------------------------------ */ +/* Generated inputs */ +/* ------------------------------------------------------------------ */ + +const DNA = ['A', 'C', 'G', 'T']; + +/** + * A generator for one parameter, from its type and name. Sequences are DNA + * by default, since that is what this course's procedures work on; a + * generator named in options.generators overrides the guess. + */ +function generatorFor(param, rnd, opts) { + const named = opts.generators?.[param.name] || opts.generators?.[up(param.name)]; + if (typeof named === 'function') return () => named(rnd); + if (Array.isArray(named)) return () => named[Math.floor(rnd() * named.length)]; + const alphabet = opts.alphabet || DNA; + const maxLen = opts.maxLength ?? 9; + const pick = (xs) => xs[Math.floor(rnd() * xs.length)]; + const int = (lo, hi) => lo + Math.floor(rnd() * (hi - lo + 1)); + const make = (t) => { + switch (t?.k) { + case 'INTEGER': { + if (t.bits) { + // a sized type: small values, and the edges of its range, where overflow lives + const [lo, hi] = intRange(t); + const small = () => Math.min(hi, Math.max(lo, int(opts.intMin ?? -3, opts.intMax ?? 12))); + const edges = [lo, lo + 1, hi - 1, hi, 0, 1].filter((v) => v >= lo && v <= hi); + return () => (rnd() < 0.3 ? pick(edges) : small()); + } + return () => (rnd() < 0.3 ? pick([0, 1, 2, -1]) : int(opts.intMin ?? -3, opts.intMax ?? 12)); + } + case 'FLOAT': { + const f = () => Math.round((rnd() * 12 - 2) * 100) / 100; + return t.bits === 32 ? () => Math.fround(f()) : f; + } + case 'BOOL': return () => rnd() < 0.5; + case 'CHAR': return () => (rnd() < 0.08 ? 'N' : pick(alphabet)); + case 'STRING': return () => { + const n = rnd() < 0.12 ? int(0, 1) : int(0, maxLen); + let s = ''; + for (let i = 0; i < n; i++) s += pick(alphabet); + return s; + }; + // a collection of a type with no generator has none either (a LIST OF a RECORD), so the caller says so + case 'LIST': { const el = make(t.of); return el && (() => Array.from({ length: int(0, 6) }, el)); } + case 'ARRAY': { + // an ARRAY declared ARRAY[3] (or [2, 3]) is given exactly that many values, as the run requires + const el = make(t.of); + if (!el) return null; + const dims = knownDims(t); + return () => { const d = dims ?? [int(0, 6)]; const a = new PArray(d); a.data = Array.from({ length: a.data.length }, el); return a; }; + } + case 'SET': { const el = make(t.of); return el && (() => new Set(Array.from({ length: int(0, 5) }, el))); } + case 'HASHMAP': { const k = make(t.key); const v = make(t.val); return k && v && (() => new Map(Array.from({ length: int(0, 4) }, () => [k(), v()]))); } + default: return null; + } + }; + return make(param.type); +} + +/** Smaller versions of a value, for shrinking a failure down to its essentials. */ +function* smaller(v, type) { + if (typeof v === 'string' && type?.k === 'STRING') { + const cs = Array.from(v); + for (let i = 0; i < cs.length; i++) yield cs.slice(0, i).concat(cs.slice(i + 1)).join(''); + for (let i = 0; i < cs.length; i++) if (cs[i] !== 'A') yield cs.slice(0, i).concat(['A'], cs.slice(i + 1)).join(''); + } + if (typeof v === 'number' && Number.isInteger(v) && v !== 0) { yield 0; yield v > 0 ? v - 1 : v + 1; if (Math.abs(v) > 2) yield Math.trunc(v / 2); } + if (Array.isArray(v)) { + for (let i = 0; i < v.length; i++) yield v.slice(0, i).concat(v.slice(i + 1)); + for (let i = 0; i < v.length; i++) for (const s of smaller(v[i], type?.of)) { const c = v.slice(); c[i] = s; yield c; } + } +} + +/** Every DNA string up to a length, shortest first. */ +function* allStrings(maxLen, alphabet = DNA) { + yield ''; + let layer = ['']; + for (let n = 1; n <= maxLen; n++) { + const next = []; + for (const s of layer) for (const c of alphabet) { next.push(s + c); yield s + c; } + layer = next; + } +} + +/** + * Run one subroutine on many inputs and report every input on which a + * contract fails or the program stops with an error. Inputs that break a + * REQUIRE are skipped: the caller promised never to pass them. With a + * reference program, also report inputs where the two disagree. + * + * options = { count: 400, seed: 1, exhaustive: 4, reference, generators, alphabet, maxLength, budgetMs, maxWork } + * maxWork: the steps and quantifier positions (`visits`) all the runs together may take; past it no more inputs + * are tried (the inputs tried so far stand; `cutShort` says so), and a run cut short by it is not a failure. It is + * counted, so it ends at the same input on every machine: one run of a routine whose ENSURE nests quantifiers may + * visit 5,000,000 positions, and took 7 seconds. `work` in the result says how much was used. + * budgetMs: stop trying inputs after this long, the same way. Without either, every input is tried, however slow + * the routine: 150 runs of a routine that recurses 400 deep took 20 seconds. + */ +export function checkProperties(program, name, options = {}) { + const opts = { count: 400, seed: 1, exhaustive: 4, maxFailures: 3, ...options }; + const sub = program.subs.get(up(name)); + if (!sub) return { checked: 0, skipped: 0, failures: [], problem: `No subroutine called ${name}.` }; + const rnd = makeRandom(opts.seed); + const gens = sub.params.map((p) => generatorFor(p, rnd, opts)); + if (gens.some((g) => !g)) return { checked: 0, skipped: 0, failures: [], problem: `The lab cannot make up values of every parameter type of ${sub.name}. Write test cases for it instead.` }; + const inputs = []; + // Exhaustive for small text inputs: every string up to a small length. + const textParams = sub.params.map((p, i) => (p.type.k === 'STRING' ? i : -1)).filter((i) => i >= 0); + if (opts.exhaustive && textParams.length === 1 && sub.params.length <= 2 && !opts.generators) { + const other = sub.params.findIndex((p, i) => i !== textParams[0]); + const otherValues = other < 0 ? [null] : sub.params[other].type.k === 'CHAR' ? DNA : Array.from({ length: 6 }, () => gens[other]()); + for (const s of allStrings(opts.exhaustive, opts.alphabet || DNA)) { + for (const o of otherValues) { + const args = []; + args[textParams[0]] = s; + if (other >= 0) args[other] = o; + inputs.push(args); + } + } + } else if (opts.exhaustive && textParams.length === 2 && sub.params.length === 2 && !opts.generators) { + const short = [...allStrings(Math.min(3, opts.exhaustive))]; + for (const a of short) for (const b of short) if (a.length <= 2 || b.length <= 2) inputs.push([a, b]); + } + for (let i = 0; i < opts.count; i++) inputs.push(gens.map((g) => g())); + + const refSub = opts.reference ? (typeof opts.reference === 'function' ? null : opts.reference.subs.get(up(name))) : null; + let checked = 0; let skipped = 0; + const failures = []; + const seen = new Set(); + const maxSteps = opts.maxSteps ?? 50000; + let work = 0; + const judge = (args) => { + // within what is left of maxWork: a run stopped by that, and not by its own limits, is cut short, not failed + const left = opts.maxWork == null ? Infinity : opts.maxWork - work; + if (left <= 0) return { out: true }; + const steps = Math.min(maxSteps, left); + const visits = Math.min(maxSteps * 100, left); + const trace = run(program, { call: { name: sub.name, args: args.map(cloneValue) }, record: false, maxSteps: steps, maxVisits: visits }); + work += (trace.work?.steps ?? 0) + (trace.visits ?? 0); + if (trace.error && trace.error.kind === 'limit' && ((steps < maxSteps && trace.work.steps > steps) || (visits < maxSteps * 100 && trace.visits > visits))) return { out: true }; + if (trace.error && trace.error.kind === 'require') return { skip: true }; + if (trace.error) return { fail: true, kind: trace.error.kind, message: trace.error.message, line: trace.error.line, col: trace.error.col, phase: trace.error.phase }; + if (opts.reference) { + let want; + if (typeof opts.reference === 'function') want = opts.reference(...args.map(cloneValue)); + else { + const rt = run(opts.reference, { call: { name: refSub.name, args: args.map(cloneValue) }, record: false, contracts: false, maxSteps: 200000 }); + if (rt.error) return { skip: true }; + want = rt.result; + } + if (!sameResult(trace.result, want, sub.returns)) { + return { fail: true, kind: 'mismatch', message: `returned ${formatValue(trace.result, sub.returns)}, and the right answer is ${formatValue(want, sub.returns)}`, got: trace.result, want }; + } + } + return { ok: true, result: trace.result }; + }; + const show = (args) => `${sub.name}(${args.map((a, i) => formatValue(a, sub.params[i].type, { bareChar: true })).join(', ')})`; + const started = Date.now(); + let cutShort = false; + for (const args of inputs) { + if (opts.budgetMs && Date.now() - started > opts.budgetMs) { cutShort = true; break; } + const k = show(args); + if (seen.has(k)) continue; + seen.add(k); + const r = judge(args); + if (r.out) { cutShort = true; break; } + if (r.skip) { skipped++; continue; } + checked++; + if (r.fail) { + // shrink: keep taking a smaller input that still fails the same way + let best = args; let bestR = r; + for (let tries = 0; tries < 300; tries++) { + let improved = false; + for (let i = 0; i < best.length && !improved; i++) { + for (const s of smaller(best[i], sub.params[i].type)) { + const cand = best.slice(); cand[i] = s; + const cr = judge(cand); + if (cr.out) break; + if (cr.fail && cr.kind === bestR.kind) { best = cand; bestR = cr; improved = true; break; } + } + } + if (!improved) break; + } + const key = `${bestR.kind}@${bestR.line}:${bestR.col}:${bestR.phase || ''}`; + if (!failures.some((f) => f.key === key)) failures.push({ key, call: show(best), args: best, ...bestR, original: show(args) }); + if (failures.length >= opts.maxFailures) break; + } + } + return { checked, skipped, failures, name: sub.name, work, ...(cutShort ? { cutShort } : {}) }; +} + +/* ------------------------------------------------------------------ */ +/* How the work grows */ +/* ------------------------------------------------------------------ */ + +const MODELS = [ + { id: '1', label: 'the same at every size', f: () => 1 }, + { id: 'log n', label: 'grows like log n', f: (n) => Math.log2(Math.max(2, n)) }, + { id: 'n', label: 'grows in step with n', f: (n) => n }, + { id: 'n log n', label: 'grows like n log n', f: (n) => n * Math.log2(Math.max(2, n)) }, + { id: 'n^2', label: 'grows like n squared', f: (n) => n * n }, + { id: 'n^3', label: 'grows like n cubed', f: (n) => n * n * n }, + { id: '2^n', label: 'doubles with every step in n', f: (n) => 2 ** Math.min(n, 60) }, +]; + +/** + * Run a subroutine at growing sizes and count its work: steps, or letter + * comparisons, the unit the counting-the-work lesson uses. + * + * options = { + * sizes: [8, 16, 32, 64, 128, 256], + * args: (n, random) => [...], the input at size n + * measure: 'steps' | 'letterCompares' | 'compares' | 'reads', + * seed, repeats + * } + */ +export function measureGrowth(program, name, options = {}) { + const opts = { sizes: [8, 16, 32, 64, 128, 256], measure: 'steps', seed: 7, repeats: 1, ...options }; + const sub = program.subs.get(up(name)); + if (!sub) return { points: [], problem: `No subroutine called ${name}.` }; + const rnd = makeRandom(opts.seed); + const makeArgs = opts.args || ((n) => sub.params.map((p) => defaultAtSize(p.type, n, rnd))); + const points = []; + for (const n of opts.sizes) { + let total = 0; let ok = true; + for (let r = 0; r < opts.repeats; r++) { + const args = makeArgs(n, rnd); + const t = run(program, { call: { name: sub.name, args }, record: false, contracts: false, maxSteps: opts.maxSteps ?? 5000000 }); + if (t.error) { ok = false; points.push({ n, error: t.error.message }); break; } + total += t.work[opts.measure] ?? t.work.steps; + } + if (!ok) break; + points.push({ n, work: total / opts.repeats }); + } + return { points, fit: fitGrowth(points.filter((p) => p.work != null)), measure: opts.measure }; +} + +function defaultAtSize(type, n, rnd) { + switch (type?.k) { + case 'STRING': { let s = ''; for (let i = 0; i < n; i++) s += DNA[Math.floor(rnd() * 4)]; return s; } + case 'CHAR': return 'A'; + case 'INTEGER': return n; + case 'FLOAT': return n; + case 'LIST': return Array.from({ length: n }, () => (type.of?.k === 'INTEGER' ? Math.floor(rnd() * 100) : type.of?.k === 'FLOAT' ? rnd() * 100 : 'A')); + case 'BOOL': return true; + default: return null; + } +} + +/** The model whose shape best matches the counts, by the spread of count / model(n). */ +export function fitGrowth(points) { + if (points.length < 3) return null; + let best = null; + for (const m of MODELS) { + const ratios = points.map((p) => p.work / m.f(p.n)); + if (ratios.some((r) => !Number.isFinite(r) || r <= 0)) continue; + const logs = ratios.map(Math.log); + const mean = logs.reduce((a, b) => a + b, 0) / logs.length; + const spread = Math.sqrt(logs.reduce((a, b) => a + (b - mean) ** 2, 0) / logs.length); + // Prefer the simpler model when two fit about equally well. + const score = spread + MODELS.indexOf(m) * 0.002; + if (!best || score < best.score) best = { ...m, score, constant: Math.exp(mean), spread }; + } + return best && { id: best.id, label: best.label, constant: best.constant, spread: best.spread }; +} + +export { MODELS }; diff --git a/src/engines/pseudo/translate.js b/src/engines/pseudo/translate.js new file mode 100644 index 0000000..1372b66 --- /dev/null +++ b/src/engines/pseudo/translate.js @@ -0,0 +1,541 @@ +/** + * pseudo/translate.js + * + * Pseudocode to Mojo (the course's language) and to Python, construct by + * construct, following the table at the end of the pseudocode guide. The + * translation is deliberately literal: each line of pseudocode becomes the + * line or lines that say the same thing, so a student can read the two side + * by side. Where the Mojo spelling depends on the version, the line says so. + * + * translate(program, target) -> { code, lines: [{ text, from }], notes } + * + * `from` is the pseudocode line each output line came from, so a view can + * highlight both at once. + */ + +import { sourceOf } from './parse.js'; +import { walkStatements, rootName } from './analyze.js'; + +const up = (s) => String(s).toUpperCase(); + +/** GCCount -> gc_count, MaxOf -> max_of, I -> i, COUNT_BASE -> count_base. */ +export function snake(name) { + if (/^[A-Z0-9_]+$/.test(name)) return name.toLowerCase(); + return name + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1_$2') + .replace(/([a-z0-9])([A-Z])/g, '$1_$2') + .toLowerCase(); +} + +const PY_RESERVED = new Set(['def', 'class', 'in', 'is', 'lambda', 'pass', 'from', 'import', 'global', 'with', 'yield', 'try', 'except', 'raise', 'len', 'list', 'str', 'int', 'float', 'set', 'dict', 'max', 'min', 'sum', 'type', 'input', 'print', 'range', 'var', 'fn', 'ref', 'mut', 'out', 'owned', 'alias', 'struct', 'trait', 'inout', 'read']); + +class Translator { + constructor(program, target) { + this.program = program; + this.target = target; + this.mojo = target === 'mojo'; + this.out = []; + this.notes = new Map(); + this.imports = new Set(); + this.names = new Map(); + this.commentsByLine = new Map(); + for (const c of program.comments || []) this.commentsByLine.set(c.line, c); + this.usedComments = new Set(); + this.helpers = new Map(); + } + + /** Names of collections a block of statements changes in place. */ + changedIn(body) { + const out = new Set(); + walkStatements(body, (x) => { + if (x.kind === 'BuiltinCall' && x.args[0]) { const r = rootName(x.args[0]); if (r) out.add(up(r)); } + if ((x.kind === 'Assign' || x.kind === 'Input') && x.target.kind !== 'Name') { const r = rootName(x.target); if (r) out.add(up(r)); } + if (x.kind === 'Call') x.args.forEach((a, i) => { if (x.sub?.params?.[i]?.mode === 'BYREF' && a.kind !== 'Name') { const r = rootName(a); if (r) out.add(up(r)); } }); + }); + return out; + } + + /** A value that would share storage with its source, copied: M <- L gives M its own list. */ + copied(e, text) { + const k = e.type?.k; + if (!['LIST', 'ARRAY', 'HASHMAP', 'SET', 'RECORD'].includes(k) || !['Name', 'Field', 'Index'].includes(e.kind)) return text; + this.note('copy', this.mojo + ? 'In the notation, assigning or returning a collection gives a separate copy. Mojo spells that .copy(), because Mojo tracks which piece of code owns a value; the exact rules are version-specific.' + : 'In the notation, assigning a collection gives a separate copy. In Python both names would share one list, so the translation copies it.'); + if (this.mojo) return `${text}.copy()`; + const flat = k !== 'RECORD' && !(k === 'ARRAY' && e.type.dims === 2) && !['LIST', 'ARRAY', 'HASHMAP', 'SET', 'RECORD'].includes((e.type.of || e.type.val)?.k); + if (flat) return `${text}.copy()`; + this.imports.add('import copy'); + return `copy.deepcopy(${text})`; + } + + /** An INTEGER value going where a FLOAT belongs keeps the point, as it does in the notation. */ + widened(target, e, text) { + if (target?.k === 'FLOAT' && e.type?.k === 'INTEGER') return this.mojo ? `Float64(${text})` : `float(${text})`; + return text; + } + + name(n) { + const k = up(n); + if (this.names.has(k)) return this.names.get(k); + let s = snake(n); + if (PY_RESERVED.has(s)) s = `${s}_`; + this.names.set(k, s); + return s; + } + constName(n) { return snake(n).toUpperCase(); } + + note(key, text) { this.notes.set(key, text); } + emit(indent, text, from) { this.out.push({ text: `${' '.repeat(indent)}${text}`, from }); } + + /** Comments that stand on their own line, before `line`, not yet placed. */ + leadingComments(line, indent) { + for (const [l, c] of this.commentsByLine) { + if (l >= line || this.usedComments.has(l) || !c.standalone) continue; + this.usedComments.add(l); + this.emit(indent, `# ${c.text}`, l); + } + } + trailing(line) { + const c = this.commentsByLine.get(line); + if (!c || c.standalone || this.usedComments.has(line)) return ''; + this.usedComments.add(line); + return ` # ${c.text}`; + } + + /* ---------- types ---------- */ + type(t) { + if (!t) return this.mojo ? 'Int' : 'int'; + const M = this.mojo; + switch (t.k) { + case 'INTEGER': return M ? 'Int' : 'int'; + case 'FLOAT': return M ? 'Float64' : 'float'; + case 'CHAR': case 'STRING': return M ? 'String' : 'str'; + case 'BOOL': return M ? 'Bool' : 'bool'; + case 'ARRAY': if (t.dims === 2) return M ? `List[List[${this.type(t.of)}]]` : `list[list[${this.type(t.of)}]]`; + // falls through + case 'LIST': return M ? `List[${this.type(t.of)}]` : `list[${this.type(t.of)}]`; + case 'HASHMAP': return M ? `Dict[${this.type(t.key)}, ${this.type(t.val)}]` : `dict[${this.type(t.key)}, ${this.type(t.val)}]`; + case 'SET': return M ? `Set[${this.type(t.of)}]` : `set[${this.type(t.of)}]`; + case 'RECORD': return t.name; + default: return M ? 'Int' : 'object'; + } + } + + emptyOf(t) { + const M = this.mojo; + switch (t.k) { + case 'LIST': return M ? `${this.type(t)}()` : '[]'; + case 'HASHMAP': return M ? `${this.type(t)}()` : '{}'; + case 'SET': return M ? `${this.type(t)}()` : 'set()'; + case 'ARRAY': { + const dims = (t.dimExprs || []).map((d) => (M || ['Name', 'Num', 'CallExpr'].includes(d.kind) ? this.expr(d) : `(${this.expr(d)})`)); + const fill = defaultOf(t.of, M); + if (dims.length === 2) return M ? `List[List[${this.type(t.of)}]](length=${dims[0]}, fill=List[${this.type(t.of)}](length=${dims[1]}, fill=${fill}))` : `[[${fill}] * ${dims[1]} for _ in range(${dims[0]})]`; + return M ? `List[${this.type(t.of)}](length=${dims[0]}, fill=${fill})` : `[${fill}] * ${dims[0]}`; + } + case 'RECORD': return M ? null : null; + default: return null; + } + } + + /* ---------- program ---------- */ + run() { + const p = this.program; + const M = this.mojo; + // constants, sets, and records first + for (const c of p.constants.values()) { + this.leadingComments(c.line, 0); + this.emit(0, M ? `alias ${this.constName(c.name)} = ${this.expr(c.node.value)}${this.trailing(c.line)}` : `${this.constName(c.name)} = ${this.expr(c.node.value)}${this.trailing(c.line)}`, c.line); + this.names.set(up(c.name), this.constName(c.name)); + } + if (p.constants.size) this.note('alias', 'A CONSTANT becomes an alias in Mojo. Recent Mojo versions also spell compile-time values with comptime; check the course\'s Mojo reference.'); + for (const d of p.defines.values()) { + this.leadingComments(d.line, 0); + const vals = d.node.values.map((v) => this.expr(v)).join(', '); + if (M) { + this.emit(0, `def ${this.name(d.name)}() -> ${this.type(d.type)}:${this.trailing(d.line)}`, d.line); + this.emit(1, `return ${this.type(d.type)}(${vals})`, d.line); + this.note('define', 'A DEFINE set becomes a small function that builds the Set, because Mojo has no global variables; the code calls it where the pseudocode names the set.'); + } else { + this.emit(0, `${this.constName(d.name)} = {${vals}}${this.trailing(d.line)}`, d.line); + } + this.emit(0, '', null); + } + for (const t of p.types.values()) { + this.leadingComments(t.node.line, 0); + if (M) { + this.emit(0, '@fieldwise_init', t.node.line); + this.emit(0, `struct ${t.name}(Copyable, Movable):`, t.node.line); + this.note('struct', 'A record becomes a struct with fields. The traits a struct declares (Copyable, Movable) change between Mojo versions; check the course\'s Mojo reference.'); + } else { + this.imports.add('from dataclasses import dataclass'); + this.emit(0, '@dataclass', t.node.line); + this.emit(0, `class ${t.name}:`, t.node.line); + } + t.fieldNames.forEach((f) => { + const ft = t.fields.get(up(f)); + this.emit(1, M ? `var ${this.name(f)}: ${this.type(ft)}` : `${this.name(f)}: ${this.type(ft)}${['LIST', 'HASHMAP', 'SET'].includes(ft.k) ? ` = field(default_factory=${ft.k === 'LIST' ? 'list' : ft.k === 'SET' ? 'set' : 'dict'})` : ''}`, t.node.line); + if (!M && ['LIST', 'HASHMAP', 'SET'].includes(ft.k)) this.imports.add('from dataclasses import dataclass, field'); + }); + this.emit(0, '', null); + } + for (const sub of p.subs.values()) { this.subroutine(sub); this.emit(0, '', null); } + const hasMain = p.main.body.length; + if (hasMain) { + this.emit(0, M ? 'def main():' : 'def main():', p.main.body[0]?.line ?? null); + this.block(p.main.body, 1, { sub: null }); + if (!M) { this.emit(0, '', null); this.emit(0, 'if __name__ == "__main__":', null); this.emit(1, 'main()', null); } + } + // any comments left at the end + this.leadingComments(Infinity, 0); + const head = [...this.imports].sort().map((t) => ({ text: t, from: null })); + const helpers = [...this.helpers.values()].flatMap((h) => [...h.split('\n').map((text) => ({ text, from: null })), { text: '', from: null }]); + const lines = [...head, ...(head.length ? [{ text: '', from: null }] : []), ...helpers, ...this.out]; + while (lines.length && !lines[lines.length - 1].text.trim()) lines.pop(); + return { code: lines.map((l) => l.text).join('\n'), lines, notes: [...this.notes.values()], target: this.target }; + } + + subroutine(sub) { + const s = sub.node; + const M = this.mojo; + this.leadingComments(s.line, 0); + const changed = this.changedIn(s.body); + const params = sub.params.map((p) => { + const n = this.name(p.name); + if (p.mode === 'BYREF') { + if (M) { this.note('mut', 'BYREF becomes Mojo\'s mut argument convention, which lets the function change the caller\'s variable.'); return `mut ${n}: ${this.type(p.type)}`; } + this.note('byref', 'Python has no BYREF. A procedure that changes its caller\'s variables usually returns the new values instead, or changes a list in place.'); + return `${n}: ${this.type(p.type)}`; + } + if (M && changed.has(up(p.name)) && ['LIST', 'ARRAY', 'HASHMAP', 'SET', 'RECORD'].includes(p.type?.k)) { + this.note('var-param', 'A parameter the procedure changes, on its own copy, is declared var in Mojo, so the function owns a copy it may change. Older Mojo versions spell this owned.'); + return `var ${n}: ${this.type(p.type)}`; + } + return `${n}: ${this.type(p.type)}`; + }).join(', '); + const ret = s.kind === 'Function' ? ` -> ${this.type(sub.returns)}` : M ? '' : ' -> None'; + this.emit(0, `def ${this.name(sub.name)}(${params})${ret}:${this.trailing(s.line)}`, s.line); + for (const r of sub.requires) this.emit(1, `# REQUIRE ${sourceOf(r.expr)}`, r.line); + for (const e of sub.ensures) this.emit(1, `# ENSURE ${sourceOf(e.expr)}`, e.line); + if (sub.requires.length && !M) for (const r of sub.requires) { const c = this.tryExpr(r.expr); if (c) this.emit(1, `assert ${c}`, r.line); } + const before = this.out.length; + // A by-value collection the body changes is the subroutine's own copy. + if (!M) for (const p of sub.params) { + if (p.mode !== 'BYREF' && changed.has(up(p.name)) && ['LIST', 'ARRAY', 'HASHMAP', 'SET', 'RECORD'].includes(p.type?.k)) { + this.emit(1, `${this.name(p.name)} = ${this.copied({ kind: 'Name', type: p.type }, this.name(p.name))} # by value: changes stay inside`, s.line); + } + } + this.block(s.body, 1, { sub }); + if (this.out.length === before) this.emit(1, 'pass', s.line); + } + + tryExpr(e) { try { return this.expr(e); } catch { return null; } } + + block(body, d, ctx) { + const before = this.out.length; + for (const s of body) this.stmt(s, d, ctx); + if (this.out.length === before) this.emit(d, 'pass', null); + } + + stmt(s, d, ctx) { + const M = this.mojo; + this.leadingComments(s.line, d); + const tr = () => this.trailing(s.line); + switch (s.kind) { + case 'Declare': { + const t = s.resolved; + for (const n of s.names) { + const empty = this.emptyOf(t); + if (M) this.emit(d, empty ? `var ${this.name(n.name)} = ${empty}${tr()}` : `var ${this.name(n.name)}: ${this.type(t)}${tr()}`, s.line); + else if (empty) this.emit(d, `${this.name(n.name)}: ${this.type(t)} = ${empty}${tr()}`, s.line); + else if (t.k === 'RECORD') this.emit(d, `${this.name(n.name)} = ${t.name}(${t.fieldNames.map((f) => defaultOf(t.fields.get(up(f)), false)).join(', ')})${tr()}`, s.line); + else this.emit(d, `${this.name(n.name)}: ${this.type(t)}${tr()}`, s.line); + if (t.k === 'ARRAY') this.note('array', 'A fixed-length ARRAY becomes a list created at its full length. The way Mojo spells a list of a given length depends on the version; check the course\'s Mojo reference.'); + if (t.k === 'RECORD' && M) this.note('record-init', 'A struct in Mojo is created with a value for every field at once; the translation declares it, and the field assignments below have to be gathered into one constructor call.'); + } + return; + } + case 'Assign': this.emit(d, `${this.lhs(s.target)} = ${this.widened(s.target.type, s.value, this.copied(s.value, this.expr(s.value)))}${tr()}`, s.line); return; + case 'Input': { + const t = s.target.type; + const read = M ? 'input()' : 'input()'; + const conv = t?.k === 'INTEGER' ? (M ? `atol(${read})` : `int(${read})`) : t?.k === 'FLOAT' ? (M ? `atof(${read})` : `float(${read})`) : t?.k === 'BOOL' ? `${read} == "TRUE"` : read; + this.emit(d, `${this.lhs(s.target)} = ${conv}${tr()}`, s.line); + if (M) this.note('input', 'Reading input in Mojo depends on the version; the course reads data from files and arguments rather than typed input.'); + return; + } + case 'Output': { + const vals = s.values.map((v) => this.outputValue(v)); + this.emit(d, `print(${vals.join(', ')}${vals.length > 1 ? ', sep=""' : ''})${tr()}`, s.line); + return; + } + case 'If': + s.branches.forEach((b, i) => { + this.emit(d, `${i ? 'elif' : 'if'} ${this.expr(b.cond)}:${i ? this.trailing(b.line) : tr()}`, b.line); + this.block(b.body, d + 1, ctx); + }); + if (s.elseBody) { this.emit(d, 'else:', s.elseLine); this.block(s.elseBody, d + 1, ctx); } + return; + case 'Case': { + const subj = this.expr(s.subject); + s.arms.forEach((a, i) => { + const conds = a.labels.map((l) => (l.value ? `${subj} == ${this.expr(l.value)}` : `${this.expr(l.low)} <= ${subj} < ${this.expr(l.high)}`)); + this.emit(d, `${i ? 'elif' : 'if'} ${conds.join(' or ')}:`, a.line); + this.block(a.body, d + 1, ctx); + }); + if (s.otherwise) { this.emit(d, s.arms.length ? 'else:' : 'if True:', s.otherwiseLine); this.block(s.otherwise, d + 1, ctx); } + this.note('case', 'Neither Mojo nor this translation has a CASE statement: it becomes a chain of if and elif.'); + return; + } + case 'For': { + const i = this.name(s.var); + const a = this.expr(s.start); const b = this.expr(s.stop); + const r = s.step ? `range(${a}, ${b}, ${this.expr(s.step)})` : a === '0' ? `range(${b})` : `range(${a}, ${b})`; + for (const inv of s.invariants || []) this.emit(d, `# INVARIANT ${sourceOf(inv.expr)}`, inv.line); + if (s.parallel) this.emit(d, `# PARALLEL FOR: the passes are independent${this.mojo ? '; parallelize() can run them on every core' : ''}`, s.line); + this.emit(d, `for ${i} in ${r}:${tr()}`, s.line); + this.block(s.body, d + 1, ctx); + return; + } + case 'ForEach': { + const x = this.name(s.var); + const it = s.iterable; + const coll = this.expr(it); + if (it.type?.k === 'STRING') { + if (M) { + const k = `_${x}_pos`; + this.emit(d, `for ${k} in range(${coll}.count_codepoints()):${tr()}`, s.line); + this.emit(d + 1, `var ${x} = String(${coll}[codepoint=${k}])`, s.line); + } else this.emit(d, `for ${x} in ${coll}:${tr()}`, s.line); + } else if (it.type?.k === 'HASHMAP' || it.type?.k === 'SET') { + this.emit(d, M ? `for ${x} in ${coll}${it.type.k === 'HASHMAP' ? '.keys()' : ''}:${tr()}` : `for ${x} in sorted(${coll}):${tr()}`, s.line); + if (M) this.note('each-order', 'The lab visits SET members and HASHMAP keys smallest first. Mojo visits them in its own order, which the notation never promised; a program whose answer depends on that order is worth rewriting.'); + } else if (it.kind === 'Range') { + this.emit(d, `for ${x} in range(${this.expr(it.lo)}, ${this.expr(it.hi)}):${tr()}`, s.line); + } else { + // FOR EACH walks a snapshot, so a body that changes the list does not change the walk + const r = rootName(it); + const snap = r && this.changedIn(s.body).has(up(r)); + if (snap) this.note('each-snapshot', 'FOR EACH walks the collection as it was when the loop began. The body changes it, so the translation walks a copy.'); + this.emit(d, `for ${x} in ${snap ? (M ? `${coll}.copy()` : `list(${coll})`) : coll}:${tr()}`, s.line); + } + this.block(s.body, d + 1, ctx); + return; + } + case 'While': + for (const inv of s.invariants || []) this.emit(d, `# INVARIANT ${sourceOf(inv.expr)}`, inv.line); + if (s.decreases) this.emit(d, `# DECREASES ${sourceOf(s.decreases.expr)}`, s.decreases.line); + this.emit(d, `while ${this.expr(s.cond)}:${tr()}`, s.line); + this.block(s.body, d + 1, ctx); + return; + case 'Repeat': + this.emit(d, `while True:${tr()}`, s.line); + this.block(s.body, d + 1, ctx); + this.emit(d + 1, `if ${this.expr(s.cond)}:`, s.endLine); + this.emit(d + 2, 'break', s.endLine); + this.note('repeat', 'REPEAT ... UNTIL has no direct counterpart: it becomes while True, with a break at the end once the UNTIL condition holds.'); + return; + case 'Break': this.emit(d, `break${tr()}`, s.line); return; + case 'Continue': this.emit(d, `continue${tr()}`, s.line); return; + case 'Return': { + if (!s.value) { this.emit(d, `return${tr()}`, s.line); return; } + let v = this.expr(s.value); + if (M) v = this.copied(s.value, v); + this.emit(d, `return ${this.widened(ctx.sub?.returns, s.value, v)}${tr()}`, s.line); + return; + } + case 'Call': this.emit(d, `${this.name(s.name)}(${s.args.map((a) => this.expr(a)).join(', ')})${tr()}`, s.line); return; + case 'BuiltinCall': this.emit(d, `${this.builtinStmt(s)}${tr()}`, s.line); return; + case 'ExprStmt': this.emit(d, `${this.expr(s.expr)}${tr()}`, s.line); return; + case 'Assert': { + const c = this.tryExpr(s.expr); + this.emit(d, c && !hasQuant(s.expr) ? (M ? `debug_assert(${c}, "ASSERT on line ${s.line}")` : `assert ${c}`) : `# ASSERT ${sourceOf(s.expr)}`, s.line); + return; + } + case 'BlankStmt': this.emit(d, `# <${s.text}>`, s.line); this.emit(d, 'pass', s.line); return; + default: return; + } + } + + builtinStmt(s) { + const M = this.mojo; + const [a, b, c] = s.args.map((x) => this.expr(x)); + switch (s.name) { + case 'APPEND': return `${a}.append(${b})`; + case 'INSERT': return `${a}.insert(${b}, ${c})`; + case 'REMOVE': return M ? `_ = ${a}.pop(${b})` : `${a}.pop(${b})`; + case 'ADD': return `${a}.add(${b})`; + case 'DELETE': return s.args[0].type?.k === 'SET' ? (M ? `${a}.remove(${b})` : `${a}.discard(${b})`) : (M ? `_ = ${a}.pop(${b})` : `del ${a}[${b}]`); + case 'SORT': if (M) { this.imports.add('from builtin.sort import sort'); return `sort(${a})`; } return `${a}.sort()`; + default: return `${s.name.toLowerCase()}(${[a, b, c].filter(Boolean).join(', ')})`; + } + } + + lhs(t) { return this.expr(t, true); } + + outputValue(v) { + if (v.type?.k === 'BOOL') return `"TRUE" if ${this.expr(v)} else "FALSE"`.replace(/^/, '(') + ')'; + return this.expr(v); + } + + /* ---------- expressions ---------- */ + expr(e, isTarget = false) { + const M = this.mojo; + const P = (x) => (x && ['Binary', 'Chain', 'Quant', 'Unary'].includes(x.kind) ? `(${this.expr(x)})` : this.expr(x)); + switch (e.kind) { + case 'Num': { const t = String(e.value); return e.float && !/[.e]/.test(t) ? `${t}.0` : t; } + case 'Str': return JSON.stringify(e.value); + case 'Char': return JSON.stringify(e.value); + case 'Bool': return e.value ? 'True' : 'False'; + case 'Blank': return `... # <${e.text}>`; + case 'Name': + if (e.sym?.kind === 'const') return this.constName(e.name); + if (e.sym?.kind === 'define') return M ? `${this.name(e.name)}()` : this.constName(e.name); + if (up(e.name) === 'RESULT' && e.sym?.kind === 'result') return 'result'; + return this.name(e.name); + case 'Unary': return e.op === 'NOT' ? `not ${P(e.arg)}` : `-${P(e.arg)}`; + case 'Binary': { + const map = { AND: 'and', OR: 'or', DIV: '//', MOD: '%', '^': '**', '&': '+', IN: 'in' }; + if (e.op === '/' && M) return `Float64(${this.expr(e.left)}) / Float64(${this.expr(e.right)})`; + if (e.op === 'IN' && e.right.kind === 'Range') return `${this.expr(e.right.lo)} <= ${P(e.left)} < ${this.expr(e.right.hi)}`; + if (e.op === '&') { + const side = (x) => (x.type?.k === 'CHAR' || x.type?.k === 'STRING' ? P(x) : `String(${this.expr(x)})`); + return `${side(e.left)} + ${side(e.right)}`; + } + return `${sideExpr(this, e.left, e.op, 'l')} ${map[e.op] || e.op} ${sideExpr(this, e.right, e.op, 'r')}`; + } + case 'Chain': return e.operands.map((o, i) => (i ? `${e.ops[i - 1] === 'IN' ? 'in' : e.ops[i - 1]} ${P(o)}` : P(o))).join(' '); + case 'Index': { + const t = this.expr(e.target); + if (e.target.type?.k === 'STRING' && !isTarget) { + return M ? `String(${t}[codepoint=${this.expr(e.indices[0])}])` : `${t}[${this.expr(e.indices[0])}]`; + } + if (e.indices.length === 2) return `${t}[${this.expr(e.indices[0])}][${this.expr(e.indices[1])}]`; + return `${t}[${this.expr(e.indices[0])}]`; + } + case 'Field': return `${this.expr(e.target)}.${this.name(e.field)}`; + case 'CallExpr': return this.call(e); + case 'ListLit': return M ? `List[${this.type(e.type?.of)}](${e.items.map((x) => this.expr(x)).join(', ')})` : `[${e.items.map((x) => this.expr(x)).join(', ')}]`; + case 'SetLit': return M ? `Set[${this.type(e.type?.of)}](${e.items.map((x) => this.expr(x)).join(', ')})` : e.items.length ? `{${e.items.map((x) => this.expr(x)).join(', ')}}` : 'set()'; + case 'MapLit': return M ? `Dict[${this.type(e.type?.key)}, ${this.type(e.type?.val)}]() # ${sourceOf(e)}` : `{${e.entries.map((x) => `${this.expr(x.key)}: ${this.expr(x.value)}`).join(', ')}}`; + case 'Range': return `range(${this.expr(e.lo)}, ${this.expr(e.hi)})`; + case 'SetBuilder': { + const v = this.name(e.var); + const dom = e.domain.kind === 'Range' ? `range(${this.expr(e.domain.lo)}, ${this.expr(e.domain.hi)})` : this.expr(e.domain); + if (M) { this.note('comprehension', 'Mojo has no set comprehension, so a set written by rule becomes a loop that fills a Set.'); return `_set_where(${dom}) # { ${sourceOf(e)} }`; } + return `{${v} for ${v} in ${dom} if ${this.expr(e.body)}}`; + } + case 'Quant': { + const v = this.name(e.var); + const dom = e.domain.kind === 'Range' ? `range(${this.expr(e.domain.lo)}, ${this.expr(e.domain.hi)})` : this.expr(e.domain); + if (M) { this.note('quantifier', 'FORALL, EXISTS, and SUM have no one-line Mojo spelling; in code they become a loop with a flag or a running total.'); return `False # ${sourceOf(e)}`; } + const fn = e.q === 'FORALL' ? 'all' : e.q === 'EXISTS' ? 'any' : 'sum'; + return `${fn}(${this.expr(e.body)} for ${v} in ${dom})`; + } + default: return '...'; + } + } + + call(e) { + const M = this.mojo; + const args = e.args.map((a) => this.expr(a)); + if (e.isOld) return `old_${args[0]}`; + if (!e.builtin) return `${this.name(e.name)}(${args.map((x, i) => this.widened(e.sub?.params?.[i]?.type, e.args[i], x)).join(', ')})`; + const [a, b, c] = args; + const t0 = e.args[0]?.type?.k; + switch (e.builtin) { + case 'LENGTH': return t0 === 'STRING' && M ? `${a}.count_codepoints()` : `len(${a})`; + case 'ROWS': return `len(${a})`; + case 'COLUMNS': return `len(${a}[0])`; + case 'INT': return M ? `Int(${a})` : `int(${a})`; + case 'ROUND': + if (M) return `Int(round(${a}))`; + this.imports.add('import math'); + this.helpers.set('round', 'def round_half_away(x: float) -> int:\n # ROUND takes halves away from zero; Python\'s round() takes them to the even neighbor\n return int(math.copysign(math.floor(abs(x) + 0.5), x))'); + return `round_half_away(${a})`; + case 'ABS': return `abs(${a})`; + case 'MIN': case 'MAX': { + const mixed = e.type?.k === 'FLOAT' && e.args.length > 1 && e.args.some((x) => x.type?.k === 'INTEGER'); + return `${e.builtin.toLowerCase()}(${args.map((x, i) => (mixed ? this.widened(e.type, e.args[i], x) : x)).join(', ')})`; + } + case 'SQRT': case 'EXP': case 'LOG': case 'LOG2': { + const f = e.builtin.toLowerCase(); + this.imports.add(M ? `from math import ${f}` : 'import math'); + return M ? `${f}(Float64(${a}))` : `math.${f}(${a})`; + } + case 'RAND': + if (M) { this.imports.add('from random import random_float64'); return `random_float64() * ${a}`; } + this.imports.add('import random'); return `random.random() * ${a}`; + case 'RANDINT': + if (M) { this.imports.add('from random import random_si64'); return `Int(random_si64(${a}, ${b} - 1))`; } + this.imports.add('import random'); return `random.randrange(${a}, ${b})`; + case 'SLICE': + if (M && t0 === 'STRING') this.note('slice', 'Slicing a String in Mojo counts bytes, not characters. For DNA the two agree; for other text, check the course\'s Mojo reference.'); + return `${a}[${b}:${c}]`; + case 'UPPER': return `${a}.upper()`; + case 'LOWER': return `${a}.lower()`; + case 'FLOOR': return M ? `Int(floor(${a}))` : `math.floor(${a})`; + case 'CEIL': return M ? `Int(ceil(${a}))` : `math.ceil(${a})`; + case 'LOG10': case 'SIN': case 'COS': case 'TAN': return M ? `${name.toLowerCase()}(${a})` : `math.${name.toLowerCase()}(${a})`; + case 'ATAN2': return M ? `atan2(${a}, ${b})` : `math.atan2(${a}, ${b})`; + case 'PI': return M ? 'pi' : 'math.pi'; + case 'CHOOSE': return M ? `comb(${a}, ${b})` : `math.comb(${a}, ${b})`; + case 'RANDN': return M ? 'randn_float64()' : 'random.gauss(0.0, 1.0)'; + case 'SUM_OF': return `sum(${a})`; + case 'MEAN_OF': return M ? `mean(${a})` : `statistics.fmean(${a})`; + case 'VARIANCE_OF': return M ? `variance(${a})` : `statistics.variance(${a})`; + case 'STDEV_OF': return M ? `stdev(${a})` : `statistics.stdev(${a})`; + case 'MEDIAN_OF': return M ? `median(${a})` : `statistics.median(${a})`; + case 'DOT': return `sum(p * q for p, q in zip(${a}, ${b}))`; + case 'NORM_OF': return M ? `norm(${a})` : `math.hypot(*${a})`; + case 'FILL': return `[${b}] * ${a}`; + case 'SPLIT': return `${a}.split(${b})`; + case 'JOIN': return `${b}.join(${a})`; + case 'INDEX_OF': return M ? `${a}.find(${b})` : `${a}.find(${b})`; + case 'REPLACE': return `${a}.replace(${b}, ${c})`; + case 'ORD': return `ord(${a})`; + case 'CHR': return `chr(${a})`; + case 'TO_STRING': return M ? `String(${a})` : `str(${a})`; + case 'TO_INTEGER': return M ? `atol(${a})` : `int(${a})`; + case 'TO_FLOAT': return M ? `Float64(${t0 === 'STRING' ? `atof(${a})` : a})` : `float(${a})`; + case 'KEYS': return M ? `sorted_keys(${a})` : `sorted(${a})`; + case 'SORTED': return M ? `sorted_copy(${a})` : `sorted(${a})`; + default: return `${e.builtin.toLowerCase()}(${args.join(', ')})`; + } + } +} + +const BIND = { OR: 1, AND: 2, '==': 4, '!=': 4, '<': 4, '<=': 4, '>': 4, '>=': 4, IN: 4, '&': 5, '+': 6, '-': 6, '*': 7, '/': 7, DIV: 7, MOD: 7, '^': 9 }; +function sideExpr(tr, x, op, side) { + const s = tr.expr(x); + if (!x) return s; + if (x.kind === 'Chain' || x.kind === 'Quant') return `(${s})`; + if (x.kind === 'Unary' && x.op === 'NOT' && BIND[op] > 3) return `(${s})`; + if (x.kind !== 'Binary') return s; + const a = BIND[x.op]; const b = BIND[op]; + if (a < b || (a === b && side === 'r' && !(op === x.op && ['+', '*', 'AND', 'OR', '&'].includes(op))) || (a === 4 && b === 4)) return `(${s})`; + return s; +} + +function hasQuant(e) { + let hit = false; + const visit = (x) => { if (!x || typeof x !== 'object' || hit) return; if (x.kind === 'Quant' || x.kind === 'SetBuilder') { hit = true; return; } for (const v of Object.values(x)) if (v && typeof v === 'object') { if (Array.isArray(v)) v.forEach(visit); else if (v.kind) visit(v); } }; + visit(e); + return hit; +} + +function defaultOf(t, M) { + switch (t?.k) { + case 'INTEGER': return '0'; + case 'FLOAT': return '0.0'; + case 'BOOL': return M ? 'False' : 'False'; + case 'CHAR': case 'STRING': return '""'; + case 'LIST': return M ? 'List[Int]()' : '[]'; + default: return M ? '0' : 'None'; + } +} + +export function translate(program, target = 'mojo') { + return new Translator(program, target).run(); +} diff --git a/src/engines/pseudo/vague.js b/src/engines/pseudo/vague.js new file mode 100644 index 0000000..152138f --- /dev/null +++ b/src/engines/pseudo/vague.js @@ -0,0 +1,366 @@ +/** + * pseudo/vague.js + * + * Finds the words in a problem statement that leave something open, and + * asks the question each one leaves. The course's method is to sharpen a + * sentence in passes ("Where in this sequence does this shorter sequence + * occur?" -> "List every position at which the shorter sequence starts, + * when it appears exactly and completely."), and every pass settles a + * question someone could have answered two ways. This module makes those + * questions visible. + * + * detect(text) -> [{ id, question, why, options, spans, severity }] + * checkRewrite(text, decided) -> [{ id, question, choice, stated, suggest }] + * + * An option's `says` is a pattern the sharpened sentence must match to + * count as saying what the student decided. The detector is general: it + * works on any sentence, with a handful of entries for sequence problems. + * Nothing here understands the sentence; it recognizes words that usually + * hide a choice, and stays quiet when the sentence already makes it. + */ + +const W = (s) => new RegExp(`\\b(?:${s})\\b`, 'i'); + +export const QUESTIONS = [ + { + id: 'answer-kind', severity: 'must', + when: W('where|find|finds|locate|search(?:es)? for|look(?:s)? for|occurrences?|instances?|hits?|appearances?|identify|detect|report'), + unless: W('every|all|each|first|last|how many|number of|count|counts|whether|is there|are there|at least one|any|the positions|longest|shortest|highest|lowest|best|largest|smallest|most|least|max(?:imum)?|min(?:imum)?'), + skipIfStart: /^\s*(does|do|is|are|can|has|have|was|were|will)\b/i, + question: 'Which answer do you want back?', + why: 'Finding something can mean every place it happens, the first place, how many places, or only whether it happens at all. Each is a different algorithm with a different answer.', + options: [ + { label: 'Every place it happens: a list of positions', says: W('every|all|each|list'), adds: 'every position where' }, + { label: 'Only the first place', says: W('first'), adds: 'the first position where' }, + { label: 'How many times it happens: one number', says: W('how many|number of|count'), adds: 'the number of positions where' }, + { label: 'Only whether it happens at all: yes or no', says: W('whether|is there|are there|at least one|any'), adds: 'whether there is any position where' }, + ], + }, + { + id: 'match-kind', severity: 'must', + when: W('occurs?|occurrences?|appears?|appearances?|match(?:es|ing)?|contains?|binds?|found|similar|resembles?|close to|maps?|mapping|aligns?|alignment'), + unless: W('exact(?:ly)?|letter[- ]for[- ]letter|every letter|identical|mismatch(?:es)?|allowing|errors?|differences?|ignoring case'), + question: 'What counts as a match?', + why: '"Occurs" and "matches" can mean letter for letter, or close enough. A primer can bind with a mismatch; a restriction site cannot.', + options: [ + { label: 'Every letter the same, exactly', says: W('exact(?:ly)?|letter[- ]for[- ]letter|every letter|identical|completely|no mismatch(?:es)?|without mismatch(?:es)?'), adds: 'exactly, letter for letter' }, + { label: 'Close enough: allowing a few mismatches (say how many)', says: /\b(allow\w*|at most \w+|up to \w+|with (one|two|three|\d+|a few)) (mismatch|difference|error)/i, not: /\b(no|zero|without) (mismatch|difference|error)/i, adds: 'with at most 1 mismatch' }, + { label: 'The same, ignoring upper and lower case', says: /\bcase\b/i, adds: 'ignoring upper and lower case' }, + ], + }, + { + id: 'which-position', severity: 'must', + when: W('where|positions?|locations?|places?|sites?|index|indices'), + also: W('occurs?|occurrences?|appears?|appearances?|match(?:es)?|patterns?|motifs?|primers?|words?|binds?|k-?mers?|substrings?|shorter sequence|codons?|sites?'), + unless: W('starts?|starting|begins?|beginning|ends?|ending|first letter|last letter|center|center|middle'), + skipIfStart: /^\s*(does|do|is|are|can|has|have|was|were|will)\b/i, + question: 'Which position names an appearance: where it starts, or where it ends?', + why: 'An appearance of a pattern covers several positions. The answer has to name one of them, and everyone who reads the answer has to know which.', + options: [ + { label: 'Where it starts', says: W('starts?|starting|begins?|beginning|first letter'), adds: 'where it starts' }, + { label: 'Where it ends', says: /\bwhere (it|they|each one|each) ends?\b|\bend(ing)? positions?\b|\blast letter/i, adds: 'where it ends' }, + ], + }, + { + id: 'overlap', severity: 'worth', + when: W('occurs?|occurrences?|appears?|appearances?|matches|copies|repeats?|times|every|all|each|how many|count'), + also: W('occurs?|appears?|match(?:es)?|pattern|word|motif|primer|site|k-?mers?|substrings?'), + unless: /overlap/i, + question: 'Can two appearances share letters, like "AA" twice inside "AAA"?', + why: 'Some tools count overlapping appearances and some do not. The two answers differ, so the sentence has to say which.', + options: [ + { label: 'Yes: overlapping appearances each count', says: /\b(overlapping|overlaps?|overlap)\b/i, not: /\b(not|no|non|without|never|don't)[\s-]*(\w+\s)?overlap/i, adds: 'including appearances that overlap' }, + { label: 'No: appearances may not overlap', says: /\b(non-?overlapping|not overlap|without overlaps?|no overlaps?|do not overlap|don't overlap)\b/i, adds: 'without overlaps' }, + ], + }, + { + id: 'threshold', severity: 'must', + when: /(?≤≥]=?\s*-?\d|\d+\s*(%|percent|or more|or less|or fewer|or higher|or above|or below)|\b(higher|lower|more|less|greater|smaller|larger|longer|shorter|richer) than\b/i, + question: 'Which number, exactly? And does that number itself count?', + why: '"High", "long", or "rich" means nothing to a program until it is a number and a comparison. And "above 30" and "at least 30" disagree about 30 itself.', + options: [ + { label: 'At least N: N itself counts', says: /\b(at least|no less than)\s+-?\d|\d+\s*(or more|or higher|or above)|(≥|>=)\s*-?\d/i, adds: 'at least 30' }, + { label: 'More than N: N itself does not count', says: /\b(more than|greater than|above|over|exceeds?)\s+-?\d|>(?!=)\s*-?\d/i, adds: 'more than 30' }, + { label: 'At most N', says: /\b(at most|no more than)\s+-?\d|\d+\s*(or less|or fewer|or below)|(≤|<=)\s*-?\d/i, adds: 'at most 30' }, + { label: 'Less than N', says: /\b(less than|fewer than|below|under)\s+-?\d|<(?!=)\s*-?\d/i, adds: 'less than 30' }, + ], + }, + { + id: 'ties', severity: 'must', + when: /(? or >=.', + options: [ + { label: 'The first one', says: W('first|earliest|leftmost'), adds: 'the first one if several tie' }, + { label: 'The last one', says: W('last|latest|rightmost'), adds: 'the last one if several tie' }, + { label: 'All of them', says: W('all|every'), adds: 'all of them if several tie' }, + ], + }, + { + id: 'empty', severity: 'must', + when: /(? re.test(t))) continue; + if (q.skipIfStart && q.skipIfStart.test(t)) continue; + if (options.skip && options.skip.includes(q.id)) continue; + out.push({ ...q, options: [...q.options, NOT_APPLICABLE], spans: spansOf(q.when, t) }); + } + // questions that only matter once another is settled, and the pronoun last + out.sort((a, b) => (a.severity === b.severity ? 0 : a.severity === 'must' ? -1 : 1)); + return out; +} + +function spansOf(re, text) { + const g = new RegExp(re.source, re.flags.includes('g') ? re.flags : `${re.flags}g`); + const spans = []; + let m; + while ((m = g.exec(text))) { spans.push([m.index, m.index + m[0].length]); if (!m[0].length) g.lastIndex++; } + return spans; +} + +/** + * Whether a rewritten sentence states each decision. `decided` maps a + * question id to the chosen option's index (or 'na'). + */ +export function checkRewrite(text, decided) { + const t = String(text || ''); + const out = []; + for (const [id, choice] of Object.entries(decided || {})) { + const q = QUESTIONS.find((x) => x.id === id); + if (!q) continue; + if (choice === 'na' || choice == null) { out.push({ id, question: q.question, choice: 'Does not apply', stated: true, na: true }); continue; } + if (typeof choice === 'object' && choice.other) { out.push({ id, question: q.question, choice: choice.other, stated: true, custom: true }); continue; } + const opt = q.options[choice]; + if (!opt) continue; + const stated = opt.absent ? !opt.absent.test(t) : opt.says.test(t) && !(opt.not && opt.not.test(t)); + out.push({ id, question: q.question, choice: opt.label, stated, suggest: opt.adds }); + } + return out; +} + +/** The words of a sentence, marked where a question is still open: [{ text, ids }]. */ +export function segments(text, questions) { + const t = String(text || ''); + const marks = new Array(t.length).fill(null).map(() => []); + for (const q of questions) for (const [a, b] of q.spans) for (let i = a; i < b; i++) marks[i].push(q.id); + const out = []; + let cur = null; + for (let i = 0; i < t.length; i++) { + const key = marks[i].join(','); + if (!cur || cur.key !== key) { cur = { key, text: '', ids: marks[i] }; out.push(cur); } + cur.text += t[i]; + } + return out.map(({ text: x, ids }) => ({ text: x, ids })); +} diff --git a/src/engines/pseudo/vcgen.ts b/src/engines/pseudo/vcgen.ts new file mode 100644 index 0000000..428e4bb --- /dev/null +++ b/src/engines/pseudo/vcgen.ts @@ -0,0 +1,1398 @@ +/** + * pseudo/vcgen.js + * + * Proof obligations: every place where the program depends on something + * being true, stated as a fact to prove from what is known at that point. + * + * verify(program, options) -> { + * units: [{ name, kind, line, obligations, inferred, notes, partial }], + * counts: { proved, unknown, total }, + * } + * + * An obligation group is one place in the source (a line and a kind), with + * one instance per path that reaches it: + * { kind, line, col, label, goal, status, instances, shown } + * + * Kinds: 'index', 'divide', 'key', 'slice', 'argument', 'require', 'ensure', + * 'invariant-init', 'invariant-keep', 'decreases-bound', 'decreases-step', + * 'terminates', 'assert'. + * + * The method is symbolic execution. Each path keeps what it knows as a list + * of facts about symbols. A loop is cut at its invariant: at the top of the + * loop every variable the body changes becomes a fresh symbol that the + * invariants describe. Invariants come from the student (INVARIANT lines) + * and from the checker itself, which tries a few simple shapes + * (0 <= I, Total <= I, LENGTH(Hits) >= 0, ...) and keeps the ones it can + * prove hold on entry and survive every pass. + */ + +import { + sym, bv, num, lit, TRUE, FALSE, and, or, not, cmp, ite, app, at, at2, length, member, + arith, add, sub, neg, mkq, key, substSyms, show, symsOf, sortOf, numeric, implies, work, metered, unmetered, OutOfWork, TOO_LARGE, depthOf, +} from './logic.js'; +import type { Term, Sort, TypeLike, CmpOp } from './logic.js'; +import { prove } from './solver.js'; +import { sourceOf } from './parse.js'; +import { walkStatements, rootName, statementExprs, walkExpr } from './analyze.js'; +import { isSizedInt, isFloat32, intRange, knownDims } from './lang.js'; +import { namesIn } from './interp.js'; + +/* ------------------------------------------------------------------ */ +/* Types */ +/* ------------------------------------------------------------------ */ + +/** + * A node of the parsed program, as parse.js and check.js build it: a `kind` + * ('For', 'Binary', 'CallExpr', ...), a position, and the fields of that kind. + * The parser is still JavaScript, so the fields are not typed here. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type PNode = { kind: string; line?: number; col?: number; [field: string]: any }; +/** A subroutine as check.js records it (params, requires, ensures, decreases, node). */ +export type Routine = PNode; +/** The checked program: subs, main, constants, defines. */ +export type Program = PNode; + +/** The verifier's unit of work: one subroutine, or the main program. */ +export interface Unit { + name: string; + kind: string; + node: PNode | null; + body: PNode[]; + params: PNode[]; + symbols: PNode[]; + requires: PNode[]; + ensures: PNode[]; + sub?: Routine; +} + +export interface VerifyOptions { budgetMs?: number; maxPaths?: number; infer?: boolean; debug?: boolean } + +/** One obligation group as the report gives it: one place in the source, one kind. */ +export interface Obligation { + kind: string; + line: number | null; + col: number; + label: string; + status: 'proved' | 'unknown'; + paths: number; + target?: number; + measure?: string; + needsMeasure?: boolean; + recursive?: 'self' | 'mutual'; + shown: { facts: string[]; goal: string; trail: string[]; reason?: string; witness?: string[] } | null; +} +export interface Note { line: number; text: string; kind?: string } +export interface UnitReport { + name: string; + kind: string; + line: number; + obligations: Obligation[]; + inferred: Array<{ line: number; facts: string[] }>; + notes: Note[]; + partial: boolean; +} +export interface Report { units: UnitReport[]; counts: { proved: number; unknown: number; total: number } } + +/** A value in the symbolic store: a term, or the mark of a variable declared but not yet set. */ +const UNSET = { k: 'unset' } as const; +type Val = Term | typeof UNSET; +type Env = Map; + +/** What an expression is translated in: inside a spec, under quantifiers, at a call. */ +interface Ctx { + spec?: boolean; + bound?: Map; + result?: Term | null; + paramEnv?: Env; + oldEnv?: Env; + breaks?: State[]; + continues?: State[]; +} +/** One obligation as recorded, before grouping and proving. */ +interface RawOb { + kind: string; line: number | null; col: number; label: string; hyps: Term[]; goal: Term | null; trail: string[]; + target?: number; measure?: string; needsMeasure?: boolean; recursive?: 'self' | 'mutual'; + status?: string; reason?: string; witness?: Term[]; +} +/** A candidate invariant: its text, and the formula it makes over a store. */ +interface Cand { text: string; at: (env: Env) => Term } +/** How a loop kind lets the inference build its head and take a pass. */ +interface InferHooks { + entryEnv: (env: Env) => Env; + makeHead: (extra: (h: State) => Term[]) => { H: State; i?: Term }; + guardOf: ((i: Term) => Term) | null; + whileGuard?: (H: State) => Term; + advance: (st: State, i: Term) => State; + counter?: string; + start?: Term; + step?: number; +} + +const up = (s: unknown): string => String(s).toUpperCase(); +/** A stored value as a term: an unset variable reads as a fresh unknown. */ +const isSet = (v: Val | undefined): v is Term => !!v && v !== UNSET; +const asTerm = (v: Val | undefined, name: string, sort: Sort): Term => (isSet(v) ? v : sym(name, sort)); +/** A store value the code knows is a term (a loop counter, a variable the loop head made fresh). */ +const valOf = (env: Env, k: string): Term => env.get(k) as Term; + +/** The subroutines a subroutine's body calls directly. */ +const calleeCache = new WeakMap>(); +function calleesOf(routine: Routine): Set { + const hit = calleeCache.get(routine); + if (hit) return hit; + const out = new Set(); + const body = routine.node?.body || []; + const look = (s: PNode): void => { + if (s.kind === 'Call' && s.sub) out.add(s.sub); + for (const e of statementExprs(s)) walkExpr(e, (x: PNode) => { if (x.kind === 'CallExpr' && x.sub && !x.isOld) out.add(x.sub); }); + }; + walkStatements(body, look); + for (const c of [...(routine.requires || []), ...(routine.ensures || [])]) look(c); + calleeCache.set(routine, out); + return out; +} + +/** Whether calling `from` can lead, through any chain of calls, to `to` (from === to counts). */ +function callsReach(program: Program, from: Routine, to: Routine): boolean { + const seen = new Set(); + const stack: Routine[] = [from]; + while (stack.length) { + const r = stack.pop()!; + if (r === to) return true; + if (seen.has(r)) continue; + seen.add(r); + for (const c of calleesOf(r)) stack.push(c); + } + return false; +} +/** The built-ins that give a new number on every call. */ +const DRAWS = new Set(['RAND', 'RANDINT', 'RANDN']); +const directCache = new WeakMap(); +/** Whether a subroutine's own lines read INPUT or draw a random number. */ +function readsOutside(routine: Routine): boolean { + const hit = directCache.get(routine); + if (hit !== undefined) return hit; + let found = false; + walkStatements(routine.node?.body || [], (s: PNode) => { + if (s.kind === 'Input') found = true; + for (const e of statementExprs(s)) walkExpr(e, (x: PNode) => { if (x.kind === 'CallExpr' && x.builtin && DRAWS.has(x.builtin)) found = true; }); + }); + directCache.set(routine, found); + return found; +} +/** + * A subroutine whose answer is not a function of its arguments alone: it reads INPUT or draws a random number, itself + * or through a call. Two calls of it with the same arguments may differ (library/pseudo/prover/impure-function.pseudo). + */ +function impure(routine: Routine): boolean { + const seen = new Set(); + const stack: Routine[] = [routine]; + while (stack.length) { + const r = stack.pop()!; + if (seen.has(r)) continue; + seen.add(r); + if (readsOutside(r)) return true; + for (const c of calleesOf(r)) stack.push(c); + } + return false; +} +/** Whether an expression draws a random number, directly or by calling such a subroutine. */ +function drawsIn(e: PNode | null | undefined): boolean { + let found = false; + walkExpr(e, (x: PNode) => { if (x.kind === 'CallExpr' && !x.isOld && ((x.builtin && DRAWS.has(x.builtin)) || (x.sub && impure(x.sub)))) found = true; }); + return found; +} +/** The names a list of statements can change, and among them the ones it can replace whole (not just one position). */ +type Modified = Set & { whole: Set }; +let arrayIds = 0; + +/* ------------------------------------------------------------------ */ +/* State */ +/* ------------------------------------------------------------------ */ + +class State { + env: Env; + facts: Term[]; + trail: string[]; + constructor(env: Env = new Map(), facts: Term[] = [], trail: string[] = []) { this.env = env; this.facts = facts; this.trail = trail; } + clone(): State { work(1 + this.env.size + this.facts.length + this.trail.length); return new State(new Map(this.env), this.facts.slice(), this.trail.slice()); } + assume(f: Term | null | undefined, why?: string): State { + const s = this.clone(); + if (f && !(f.k === 'lit' && f.v === true)) s.facts.push(f); + if (why) s.trail.push(why); + return s; + } +} + +/** Every variable a list of statements can change. */ +function modifiedIn(body: PNode[]): Modified { + const out = Object.assign(new Set(), { whole: new Set() }) as Modified; + walkStatements(body, (x: PNode) => { + if (x.kind === 'Assign' || x.kind === 'Input') { const r = rootName(x.target); if (r) { out.add(up(r)); if (x.target.kind === 'Name') out.whole.add(up(r)); } } + if (x.kind === 'BuiltinCall' && x.args[0]) { const r = rootName(x.args[0]); if (r) out.add(up(r)); } + if (x.kind === 'Call') x.args.forEach((a: PNode, i: number) => { if (x.sub?.params?.[i]?.mode === 'BYREF' && a.kind === 'Name') { out.add(up(a.name)); out.whole.add(up(a.name)); } }); + if (x.kind === 'For' || x.kind === 'ForEach') out.add(up(x.var)); + if (x.kind === 'Declare') for (const n of x.names) { out.add(up(n.name)); out.whole.add(up(n.name)); } + }); + return out; +} + +/* ------------------------------------------------------------------ */ +/* The executor */ +/* ------------------------------------------------------------------ */ + +class Exec { + program: Program; + unit: Unit; + opts: Required> & VerifyOptions; + recording: boolean; + raw: RawOb[]; + notes: Note[]; + inferred: Array<{ line: number; facts: string[] }>; + partial: boolean; + types: Map; + consts: Map; + defines: Map; + depth: number; + quiet = 0; + deepNoted = false; + deadline = 0; + entryEnv: Env = new Map(); + intArith: unknown = false; + floatArith = false; + constructor(program: Program, unit: Unit, options: VerifyOptions) { + this.program = program; + this.unit = unit; + this.opts = { maxPaths: 48, infer: true, budgetMs: 4000, ...options }; + this.recording = true; + this.raw = []; + this.notes = []; + this.inferred = []; + this.partial = false; + this.types = new Map(); + for (const s of unit.symbols || []) this.types.set(up(s.name), s.type); + for (const p of unit.params || []) this.types.set(up(p.name), p.type); + this.consts = new Map(); + for (const c of program.constants.values()) this.consts.set(up(c.name), c); + this.defines = new Map([...program.defines.values()].map((d: PNode): [string, PNode] => [up(d.name), d])); + this.depth = 0; + } + + sortOfName(name: string): Sort { return sortOf(this.types.get(up(name))); } + + obligation(kind: string, node: PNode | null | undefined, label: string, st: State, guards: Term[], goal: Term | null, extra: Partial = {}): void { + if (!this.recording || this.quiet) return; + work(1 + st.facts.length + guards.length + st.trail.length); + this.raw.push({ kind, line: node?.line ?? null, col: node?.col ?? 1, label, hyps: [...st.facts, ...guards], goal, trail: st.trail.slice(), ...extra }); + } + + fresh(name: string, sort: Sort): Term { return sym(name, sort); } + + /** + * A fact learned while translating an expression. It holds where the expression is evaluated: on the right of an + * AND or OR that is only under the guards, so there it is guarded by them (library/pseudo/prover/shortcircuit-*.pseudo). + */ + learn(st: State, g: Term[], f: Term): void { st.facts.push(g.length ? implies(and(g), f) : f); } + + /** + * A collection going where an ARRAY declared with plain-number lengths is: the run stops unless it has those + * lengths (interp.js `fixedShape`), so there is an obligation, and after it the lengths are a fact. When the + * lengths are plain to see (a list written out, an ARRAY declared ARRAY[3]) the goal folds to TRUE and no + * obligation is shown; the checker has already reported a mismatch it could see. + */ + arrayFit(type: TypeLike, v: Term, node: PNode, st: State, g: Term[], what: string, ctx: Ctx = {}): void { + const dims = knownDims(type); + if (!dims || ctx.spec) return; + const goal = and(cmp('==', length(v), num(dims[0])), ...(dims.length > 1 ? [cmp('==', columns(v), num(dims[1]))] : [])); + if (goal !== TRUE) this.obligation('argument', node, `${what} has the ${dims.join(' by ')} positions its ARRAY is declared with`, st, g, goal); + this.learn(st, g, goal); + } + + /** The range of a sized whole-number type as a fact about a term: lowest <= v <= highest. */ + rangeOf(type: TypeLike, v: Term): Term { + const [lo, hi] = intRange(type); + return and(cmp('<=', num(lo), v), cmp('<=', v, num(hi))); + } + + /** + * A value going into a sized type (a store, a parameter, a RETURN, a conversion, an operator's result). A whole + * number gets an obligation that it fits: the run stops right there when it does not, so from then on the + * range is a fact. A number going into a FLOAT32 is rounded: a constant is rounded here, exactly; anything + * else that is not already a FLOAT32 becomes TOFLOAT32(v), about which nothing is assumed. + */ + fit(type: TypeLike | undefined, v: Term, from: TypeLike | undefined, node: PNode, st: State, g: Term[], what: string, ctx: Ctx = {}): Term { + if (!type) return v; + if ((type as any).k === 'ARRAY') { this.arrayFit(type, v, node, st, g, what, ctx); return v; } + if (isSizedInt(type)) { + if (isSizedInt(from) && (from as any).bits === (type as any).bits && (from as any).signed === (type as any).signed) return v; + if (!ctx.spec) { + this.obligation('overflow', node, `${what} fits ${(type as any).name}`, st, g, this.rangeOf(type, v)); + this.learn(st, g, this.rangeOf(type, v)); + } + return v; + } + if (isFloat32(type)) { + if (isFloat32(from)) return v; + if (v.k === 'num') return num(Math.fround(v.v), 'float'); + return app('TOFLOAT32', [v], 'float'); + } + return v; + } + + /** + * A value deeper than MAX_DEPTH becomes an unknown of its sort: the prover's walks over a term go as deep as the + * term, and 100 lines of `T <- T + A[T] + ...` made one 10,000 deep, past the stack of the engine underneath. + * Knowing less about a value never proves more; inside a quantifier's body, where a term may be about the + * quantifier's own name, nothing is replaced (a body is as deep as its expression, which the parser bounds). + */ + shallow(v: Term, at: PNode | null | undefined, name: string, ctx: Ctx = {}): Term { + if (ctx.bound?.size || depthOf(v) <= MAX_DEPTH) return v; + if (!this.deepNoted) { + this.deepNoted = true; + this.notes.push({ line: at?.line ?? this.unit.node?.line ?? 1, kind: 'too-deep', text: `The value on line ${at?.line ?? '?'} is built from more than ${MAX_DEPTH.toLocaleString('en-US')} operations one inside another, more than the checker follows: from there on it treats such a value as one it knows nothing about, so what depends on it stays open.` }); + } + return this.fresh(name, v.t); + } + + /* ---------- expressions ---------- */ + tr(e: PNode, st: State, g: Term[] = [], ctx: Ctx = {}): Term { + work(); + return this.shallow(this.trIn(e, st, g, ctx), e, 'value', ctx); + } + trIn(e: PNode, st: State, g: Term[], ctx: Ctx): Term { + switch (e.kind) { + case 'Num': return isFloat32(e.type) ? num(Math.fround(e.value), 'float') : num(e.value, e.float ? 'float' : 'int'); + case 'TypeRef': return this.fresh(e.name, 'any'); + case 'Str': return lit(e.value, 'string'); + case 'Char': return lit(e.value, 'char'); + case 'Bool': return e.value ? TRUE : FALSE; + case 'Blank': case 'Error': return this.fresh('?', 'any'); + case 'Name': return this.name(e, st, ctx); + case 'Unary': { + const a = this.tr(e.arg, st, g, ctx); + if (e.op === 'NOT') return not(a); + if (isFloat32(e.type)) return neg(a); // negation is exact in floating point + const r = neg(a); + if (isSizedInt(e.type) && !ctx.spec) { + this.obligation('overflow', e, `${sourceOf(e)} fits ${e.type.name}`, st, g, this.rangeOf(e.type, r)); + this.learn(st, g, this.rangeOf(e.type, r)); + } + return r; + } + case 'Binary': return this.binary(e, st, g, ctx); + case 'Chain': { + const parts: Term[] = []; + let prev: Term | null = this.tr(e.operands[0], st, g, ctx); + for (let i = 0; i < e.ops.length; i++) { + const nextNode = e.operands[i + 1]; + if (e.ops[i] === 'IN') { parts.push(this.membership(prev ?? this.fresh('?', 'any'), nextNode, st, [...g, ...parts], ctx)); prev = null; continue; } + const nx = this.tr(nextNode, st, [...g, ...parts], ctx); + parts.push(cmp(e.ops[i] as CmpOp, prev ?? this.fresh('?', 'any'), nx)); + prev = nx; + } + return and(parts); + } + case 'Index': return this.index(e, st, g, ctx); + case 'Field': { + const r = this.tr(e.target, st, g, ctx); + return fieldRead(r, e.field, sortOf(e.type)); + } + case 'CallExpr': return this.call(e, st, g, ctx); + case 'ListLit': return app('LIST', e.items.map((x: PNode) => this.tr(x, st, g, ctx)), 'list'); + case 'SetLit': return app('SET', e.items.map((x: PNode) => this.tr(x, st, g, ctx)), 'set'); + case 'MapLit': return e.entries.reduce((m: Term, x: PNode) => app('MAPPUT', [m, this.tr(x.key, st, g, ctx), this.tr(x.value, st, g, ctx)], 'map'), app('EMPTY', [], 'map')); + case 'Range': return app('RANGE', [this.tr(e.lo, st, g, ctx), this.tr(e.hi, st, g, ctx)], 'range'); + case 'SetBuilder': case 'Quant': return this.binder(e, st, g, ctx); + default: return this.fresh('?', 'any'); + } + } + + name(e: PNode, st: State, ctx: Ctx): Term { + const k = up(e.name); + if (ctx.bound && ctx.bound.has(k)) return bv(ctx.bound.get(k)!, sortOf(e.type)); + if (ctx.result && k === 'RESULT' && e.sym?.kind === 'result') return ctx.result; + if (ctx.paramEnv && ctx.paramEnv.has(k)) return asTerm(ctx.paramEnv.get(k), e.name, sortOf(e.type)); + if (st.env.has(k)) { + const v = st.env.get(k); + if (!isSet(v)) return this.fresh(e.name, sortOf(e.type)); + return v; + } + const c = this.consts.get(k); + if (c) return this.tr(c.node.value, new State(), [], {}); + const d = this.defines.get(k); + if (d) return app('SET', d.node.values.map((v: PNode) => this.tr(v, new State(), [], {})), 'set'); + return this.fresh(e.name, sortOf(e.type)); + } + + binary(e: PNode, st: State, g: Term[], ctx: Ctx): Term { + const op = e.op; + if (op === 'AND') { const l = this.tr(e.left, st, g, ctx); return and(l, this.tr(e.right, st, [...g, l], ctx)); } + if (op === 'OR') { const l = this.tr(e.left, st, g, ctx); return or(l, this.tr(e.right, st, [...g, not(l)], ctx)); } + if (op === 'IN') return this.membership(this.tr(e.left, st, g, ctx), e.right, st, g, ctx); + const l = this.tr(e.left, st, g, ctx); + const r = this.tr(e.right, st, g, ctx); + if (['/', 'DIV', 'MOD'].includes(op) && !ctx.spec) { + this.obligation('divide', e.right, `${sourceOf(e.right)} is not 0`, st, g, cmp('!=', r, num(0, r.t))); + } + if (['==', '!=', '<', '<=', '>', '>='].includes(op)) return cmp(op as CmpOp, l, r); + // FLOAT arithmetic rounds at every step, so 0.2 + 0.1 - 0.2 is not 0.1 + // and X + 1.0 can equal X. The checker knows nothing about a rounded + // result except that the same operation on the same values gives it + // again; comparisons of FLOAT values stay exact. + if (['+', '-', '*', '/', '^'].includes(op) && (op === '/' || sortOf(e.type) === 'float' || e.left.type?.k === 'FLOAT' || e.right.type?.k === 'FLOAT')) { + if (!ctx.spec) this.floatArith = true; + // FLOAT32 arithmetic is a different function: the same values rounded to single precision + return app(isFloat32(e.type) ? `F32${op}` : `FLOAT${op}`, [l, r], 'float'); + } + if (!ctx.spec && ['+', '-', '*', '^'].includes(op) && sortOf(e.type) === 'int' && !isSizedInt(e.type)) this.intArith = e.line ?? e.left?.line ?? true; + const res = arith(op, l, r, sortOf(e.type)); + // a sized whole number: the exact result, and an obligation that it fits (the run stops where it does not) + if (isSizedInt(e.type) && !ctx.spec && ['+', '-', '*', '^', 'DIV', 'MOD'].includes(op)) { + this.obligation('overflow', e, `${sourceOf(e)} fits ${e.type.name}`, st, g, this.rangeOf(e.type, res)); + this.learn(st, g, this.rangeOf(e.type, res)); + } + return res; + } + + membership(x: Term, rightNode: PNode, st: State, g: Term[], ctx: Ctx): Term { + if (rightNode.kind === 'Range') { + const lo = this.tr(rightNode.lo, st, g, ctx); const hi = this.tr(rightNode.hi, st, g, ctx); + // only a whole number is in a range of positions; for a FLOAT the answer is left open + if (x.t !== 'int') return app('IN', [x, app('RANGE', [lo, hi], 'range')], 'bool'); + return and(cmp('<=', lo, x), cmp('<', x, hi)); + } + return member(x, this.tr(rightNode, st, g, ctx)); + } + + index(e: PNode, st: State, g: Term[], ctx: Ctx): Term { + const c = this.tr(e.target, st, g, ctx); + const idx: Term[] = e.indices.map((i: PNode) => this.tr(i, st, g, ctx)); + const tk = e.target.type?.k; + const s = sortOf(e.type); + const quiet = ctx.spec || ctx.bound; + if (tk === 'HASHMAP') { + if (!quiet) this.obligation('key', e, `${sourceOf(e.indices[0])} is a key of ${sourceOf(e.target)}`, st, g, member(idx[0], c)); + return at(c, idx[0], s); + } + if (tk === 'ARRAY' && idx.length === 2) { + if (!quiet) { + this.obligation('index', e, `row ${sourceOf(e.indices[0])} and column ${sourceOf(e.indices[1])} of ${sourceOf(e.target)} exist`, st, g, + and(cmp('<=', num(0), idx[0]), cmp('<', idx[0], length(c)), cmp('<=', num(0), idx[1]), cmp('<', idx[1], columns(c)))); + } + return at2(c, idx[0], idx[1], s); + } + if (!quiet) { + this.obligation('index', e, `position ${sourceOf(e.indices[0])} of ${sourceOf(e.target)} exists`, st, g, + and(cmp('<=', num(0), idx[0]), cmp('<', idx[0], length(c)))); + } + return at(c, idx[0], s); + } + + binder(e: PNode, st: State, g: Term[], ctx: Ctx): Term { + const bound = new Map(ctx.bound || []); + const name = `${e.var}`; + bound.set(up(e.var), name); + const inner: Ctx = { ...ctx, bound, spec: true }; + const q = e.kind === 'SetBuilder' ? 'SETOF' : e.q; + const t: Sort = e.kind === 'SetBuilder' ? 'set' : e.q === 'SUM' ? sortOf(e.type) : 'bool'; + // a body that draws a random number draws again for every value: nothing is known about the result + if (drawsIn(e.body)) return this.fresh(q, t); + if (e.domain?.kind === 'Range') { + const lo = this.tr(e.domain.lo, st, g, ctx); const hi = this.tr(e.domain.hi, st, g, ctx); + const body = this.tr(e.body, st, [], inner); + return mkq(q, name, lo, hi, body, t); + } + const dom = this.tr(e.domain, st, g, ctx); + return { k: 'qc', q, v: name, dom, body: this.tr(e.body, st, [], inner), t }; + } + + call(e: PNode, st: State, g: Term[], ctx: Ctx): Term { + if (e.isOld) { + const n = e.args[0]; + if (ctx.oldEnv && n.kind === 'Name' && ctx.oldEnv.has(up(n.name))) return asTerm(ctx.oldEnv.get(up(n.name)), n.name, sortOf(n.type)); + return this.tr(n, st, g, ctx); + } + const args: Term[] = e.args.map((a: PNode) => this.tr(a, st, g, ctx)); + const s = sortOf(e.type); + if (e.convert) { + const from = e.args[0]?.type; + const x = args[0] ?? this.fresh('?', 'any'); + if (isSizedInt(e.type)) return this.fit(e.type, from?.k === 'FLOAT' ? app('TRUNC', [x], 'int') : x, from?.k === 'FLOAT' ? undefined : from, e, st, g, sourceOf(e), ctx); + if (isFloat32(e.type)) return this.fit(e.type, x, from, e, st, g, sourceOf(e), ctx); + return x; + } + if (e.builtin) { + // MIN, MAX and CLAMP of a FLOAT32 and a whole number give a FLOAT32: the run rounds the whole number to one + if (isFloat32(e.type) && args.length >= 2 && ['MIN', 'MAX', 'CLAMP'].includes(e.builtin)) args.forEach((a, i) => { args[i] = this.fit(e.type, a, e.args[i]?.type, e.args[i], st, g, sourceOf(e.args[i]), ctx); }); + const [x, y, z] = args; + // A built-in the run stops on for some arguments has an obligation that the arguments are allowed + // (kind 'argument'); without USE IEEE these are the FLOAT functions with a domain. + const domain = (goal: Term, label: string) => { if (!ctx.spec) this.obligation('argument', e, label, st, g, goal); }; + const call = `${e.builtin}(${e.args.map(sourceOf).join(', ')})`; + const zero = num(0, x?.t === 'float' ? 'float' : 'int'); + switch (e.builtin) { + case 'SQRT': domain(cmp('>=', x, zero), `${call} takes a number of 0 or more`); return app('SQRT', args, s); + case 'LOG': case 'LOG2': case 'LOG10': domain(cmp('>', x, zero), `${call} takes a number above 0`); return app(e.builtin, args, s); + case 'ASIN': case 'ACOS': domain(and(cmp('<=', num(-1), x), cmp('<=', x, num(1))), `${call} takes a number in [-1, 1]`); return app(e.builtin, args, s); + case 'LOG1P': domain(cmp('>', x, num(-1)), `${call} takes a number above -1`); return app(e.builtin, args, s); + case 'CLAMP': domain(cmp('<=', y, z), `${call} has its low end at or below its high end`); return ite(cmp('<', x, y), y, ite(cmp('>', x, z), z, x)); + case 'SHIFT_LEFT': case 'SHIFT_RIGHT': { + domain(cmp('>=', y, num(0)), `${call} moves bits by 0 or more places`); + const r = app(e.builtin, args, s); + if (e.builtin === 'SHIFT_LEFT') this.fitResult(e, r, st, g, ctx); + return r; + } + case 'LCM': { const r = app('LCM', args, s); this.fitResult(e, r, st, g, ctx); return r; } + // GCD(-128, 0) is 128 and every bit of a UINT64 flipped is past what a run holds: the run stops where the + // answer does not fit its sized type, so each has the obligation (tests/numeric.test.mjs) + case 'GCD': case 'BIT_NOT': { const r = app(e.builtin, args, s); this.fitResult(e, r, st, g, ctx); return r; } + case 'MAX_VALUE': case 'MIN_VALUE': case 'EPSILON': { + const ty = e.args[0]?.kind === 'TypeRef' ? e.type : null; + if (!ty) return app(e.builtin, args, s); + if (e.builtin === 'EPSILON') return num(isFloat32(ty) ? 2 ** -23 : Number.EPSILON, 'float'); + if (ty.k === 'INTEGER') { const [lo, hi] = intRange(ty); return num(e.builtin === 'MAX_VALUE' ? hi : lo, 'int'); } + const m = isFloat32(ty) ? 3.4028234663852886e38 : Number.MAX_VALUE; + return num(e.builtin === 'MAX_VALUE' ? m : -m, 'float'); + } + case 'SUM_OF': case 'DOT': if (isSizedInt(e.type)) { const r = app(e.builtin, args, s); this.fitResult(e, r, st, g, ctx); return r; } break; + default: break; + } + switch (e.builtin) { + case 'LENGTH': return length(x); + case 'ROWS': return length(x); + case 'COLUMNS': return columns(x); + case 'MIN': return args.length === 2 ? ite(cmp('<=', x, y), x, y) : app('MIN', args, s); + case 'MAX': return args.length === 2 ? ite(cmp('>=', x, y), x, y) : app('MAX', args, s); + // ABS of a sized type's lowest value does not fit it (ABS of the INT8 -128 is 128), and the run stops there + case 'ABS': { const r = ite(cmp('>=', x, num(0, x.t)), x, neg(x)); this.fitResult(e, r, st, g, ctx); return r; } + case 'SLICE': + if (!ctx.spec) this.obligation('slice', e, `SLICE(${e.args.map(sourceOf).join(', ')}) stays inside ${sourceOf(e.args[0])}`, st, g, and(cmp('<=', num(0), y), cmp('<=', y, z), cmp('<=', z, length(x)))); + return app('SLICE', [x, y, z], s); + case 'SORTED': + // every value of a two-dimensional ARRAY, rows times columns of them: nothing is said about how many + if (e.args[0]?.type?.k === 'ARRAY' && e.args[0].type.dims === 2) return app('SORTEDALL', [x], 'list'); + if (x.k === 'q' && x.q === 'SETOF') return { ...x, q: 'SORTED', t: 'list' }; + if (x.k === 'app' && x.f === 'EMPTY') return app('LIST', [], 'list'); + return app('SORTEDOF', [x], 'list'); + // A fact pushed here holds for every value the built-in can return, or + // it is guarded by what the call needs: an unguarded fact that a run can + // break leaves the checker holding a contradiction, and then it proves + // anything (library/pseudo/prover/rand-*.pseudo). Where the interpreter stops the + // program on a bad argument, that is an obligation (kind 'argument'). + case 'FILL': { + if (!ctx.spec) this.obligation('argument', e, `FILL(${e.args.map(sourceOf).join(', ')}) asks for 0 or more copies`, st, g, cmp('<=', num(0), x)); + const r = app('FILL', args, s); + this.learn(st, g, or(cmp('<', x, num(0)), cmp('==', length(r), x))); + return r; + } + // -1 when the piece is not there, else a position; an empty piece is found at position 0, of an empty text too + case 'INDEX_OF': { const r = app('INDEX_OF', args, 'int'); this.learn(st, g, and(cmp('<=', num(-1), r), or(cmp('<', r, length(x)), cmp('==', r, num(0))))); return r; } + case 'RAND': { + // RAND(X) is a random number in [0, 1) times X: [0, X) for X > 0, (X, 0] for X < 0, 0 for X = 0. + const r = this.fresh('RAND', 'float'); + const zero = num(0, 'float'); + st.facts.push(or( + and(cmp('>', x, zero), cmp('<=', zero, r), cmp('<', r, x)), + and(cmp('==', x, zero), cmp('==', r, zero)), + and(cmp('<', x, zero), cmp('<', x, r), cmp('<=', r, zero)), + )); + return r; + } + // a new draw on every call: two RANDN() are two unknowns + case 'RANDN': return this.fresh('RANDN', 'float'); + case 'RANDINT': { + if (!ctx.spec) this.obligation('argument', e, `RANDINT(${e.args.map(sourceOf).join(', ')}) picks from a range that is not empty`, st, g, cmp('<', x, y)); + const r = this.fresh('RANDINT', 'int'); + st.facts.push(or(cmp('>=', x, y), and(cmp('<=', x, r), cmp('<', r, y)))); + return r; + } + default: return app(e.builtin, args, s); + } + } + const sub = e.sub; + if (!sub) return this.fresh(e.name, s); + sub.params.forEach((p: PNode, i: number) => { if (args[i] && e.args[i]) args[i] = this.fit(p.type, args[i], e.args[i].type, e.args[i], st, g, `the value for ${p.name}`, ctx); }); + // the same call gives the same result only when the function reads nothing but its arguments + const result = impure(sub) ? this.fresh(sub.name, s) : app(`FN:${sub.name}`, args, s); + // A FUNCTION that RETURNS an ARRAY of plain-number lengths hands back one of those lengths: its RETURN is checked + // in a run (and has its own obligation in the function), so at the call the lengths are a fact. Without it, + // `G <- LIFE_STEP(G)` left "has the 3 by 3 positions" open in library/pseudo/programs/15-game-of-life. + const returned = knownDims(sub.returns); + if (returned && !ctx.bound?.size) this.learn(st, g, and(cmp('==', length(result), num(returned[0])), ...(returned.length > 1 ? [cmp('==', columns(result), num(returned[1]))] : []))); + const paramEnv: Env = new Map(sub.params.map((p: PNode, i: number): [string, Val] => [up(p.name), args[i]])); + if (!ctx.spec) { + for (const r of sub.requires) { + const goal = this.tr(r.expr, st, g, { paramEnv, spec: true }); + this.obligation('require', e, `this call to ${sub.name} meets its REQUIRE, ${sourceOf(r.expr)}`, st, g, goal, { target: r.line }); + } + this.recursion(sub, e, args, st, g); + } + // An ENSURE is a fact where the call is made: guarded under a short-circuit, and not stated at all inside a + // quantifier's body, where an argument may be the quantifier's own name and the fact would be about no value + // in particular (library/pseudo/prover/quantifier-ensure.pseudo). + if (!ctx.bound?.size) for (const en of sub.ensures) this.learn(st, g, this.tr(en.expr, st, g, { paramEnv, result, spec: true, oldEnv: paramEnv })); + return result; + } + + /** A built-in's sized whole-number result: an obligation that it fits, then the range as a fact. */ + fitResult(e: PNode, r: Term, st: State, g: Term[], ctx: Ctx): void { + if (!isSizedInt(e.type) || ctx.spec) return; + this.obligation('overflow', e, `${sourceOf(e)} fits ${e.type.name}`, st, g, this.rangeOf(e.type, r)); + this.learn(st, g, this.rangeOf(e.type, r)); + } + + /** A recursive call has to make the DECREASES measure smaller, and keep it at 0 or above. */ + recursion(sub: Routine, at: PNode, args: Term[], st: State, g: Term[]): void { + const cur = this.unit.sub; + if (!cur || !callsReach(this.program, sub, cur)) return; + // Only a subroutine calling itself, with a DECREASES, has a measure to + // check. Any other cycle of calls stays open: nothing shows it ends. + if (sub !== cur || !sub.decreases) { + this.obligation('terminates', at, sub === cur ? `the recursive calls to ${sub.name} end` : `the calls from ${cur.name} to ${sub.name} and back end`, st, g, null, { needsMeasure: true, recursive: sub === cur ? 'self' : 'mutual' }); + return; + } + const paramEnv: Env = new Map(sub.params.map((p: PNode, i: number): [string, Val] => [up(p.name), args[i]])); + // the run compares with the measure this call started with, whatever the body has done to its parameters since + const now = this.tr(sub.decreases.expr, st, [], { paramEnv: this.entryEnv, spec: true }); + const then = this.tr(sub.decreases.expr, st, [], { paramEnv, spec: true }); + this.obligation('decreases-step', at, `the recursive call makes ${sourceOf(sub.decreases.expr)} smaller`, st, g, and(cmp('<', then, now), cmp('>=', then, num(0)))); + } + + /* ---------- statements ---------- */ + run(body: PNode[], states: State[], ctx: Ctx): State[] { + let cur = states; + for (const s of body) { + if (!cur.length) break; + const next: State[] = []; + for (const st of cur) next.push(...this.stmt(s, st, ctx)); + if (next.length > this.opts.maxPaths) { this.partial = true; cur = next.slice(0, this.opts.maxPaths); } else cur = next; + if (this.deadline && Date.now() > this.deadline) { this.partial = true; return []; } + } + return cur; + } + + stmt(s: PNode, st: State, ctx: Ctx): State[] { + switch (s.kind) { + case 'Declare': { + const n = st.clone(); + for (const nm of s.names) { + const k = s.resolved?.k; + let v: Val = UNSET; + if (k === 'LIST') v = app('LIST', [], 'list'); + else if (k === 'HASHMAP' || k === 'SET') v = app('EMPTY', [], k === 'SET' ? 'set' : 'map'); + else if (k === 'ARRAY') { + const dims: Term[] = (s.resolved.dimExprs || []).map((d: PNode) => this.tr(d, st, [])); + v = app('NEWARRAY', [num(++arrayIds), ...dims], 'array'); + if (dims.length) n.facts.push(cmp('>=', dims[0], num(0))); + } else if (k === 'RECORD') v = this.fresh(nm.name, 'record'); + n.env.set(up(nm.name), v); + } + return [n]; + } + case 'Assign': { + const v = this.fit(s.target.type, this.tr(s.value, st, []), s.value.type, s.value, st, [], sourceOf(s.value)); + const n = st.clone(); + this.store(s.target, v, n, st); + return [n]; + } + case 'Input': { + const raw = this.fresh(rootName(s.target) || 'input', sortOf(s.target.type)); + const v = this.fit(s.target.type, raw, undefined, s, st, [], `the value INPUT reads into ${sourceOf(s.target)}`); + const n = st.clone(); + this.store(s.target, v, n, st); + return [n]; + } + case 'Output': case 'ExprStmt': { + for (const v of s.values || [s.expr]) this.tr(v, st, []); + return [st]; + } + case 'If': return this.ifStmt(s, st, ctx); + case 'Case': return this.caseStmt(s, st, ctx); + case 'For': return this.forStmt(s, st, ctx); + case 'ForEach': return this.forEach(s, st, ctx); + case 'While': return this.whileStmt(s, st, ctx); + case 'Repeat': return this.repeatStmt(s, st, ctx); + case 'Break': ctx.breaks?.push(st); return []; + case 'Continue': ctx.continues?.push(st); return []; + case 'Return': { + const v = s.value ? this.fit(this.unit.sub?.returns, this.tr(s.value, st, []), s.value.type, s.value, st, [], sourceOf(s.value)) : null; + this.ensure(s, st, v); + return []; + } + case 'Call': return [this.procCall(s, st)]; + case 'BuiltinCall': return [this.builtin(s, st)]; + case 'Assert': { + const goal = this.tr(s.expr, st, [], { spec: true }); + this.obligation('assert', s, `ASSERT ${sourceOf(s.expr)}`, st, [], goal); + return [st.assume(goal)]; + } + default: return [st]; + } + } + + /** Write a value into a target, updating the symbolic store. */ + store(target: PNode, value: Term, n: State, st: State): void { + const v = this.shallow(value, target, rootName(target) || 'value'); + if (target.kind === 'Name') { n.env.set(up(target.name), v); return; } + const root = rootName(target); + const build = (t: PNode, value: Term): Term => { + if (t.kind === 'Name') return value; + if (t.kind === 'Index') { + const container = this.tr(t.target, n, []); + const idx: Term[] = t.indices.map((i: PNode) => this.tr(i, st, [])); + const tk = t.target.type?.k; + if (tk === 'HASHMAP') return build(t.target, app('MAPPUT', [container, idx[0], value], 'map')); + if (idx.length === 2) { + this.obligation('index', t, `row ${sourceOf(t.indices[0])} and column ${sourceOf(t.indices[1])} of ${sourceOf(t.target)} exist`, st, [], + and(cmp('<=', num(0), idx[0]), cmp('<', idx[0], length(container)), cmp('<=', num(0), idx[1]), cmp('<', idx[1], columns(container)))); + return build(t.target, app('STORE2', [container, idx[0], idx[1], value], 'array')); + } + this.obligation('index', t, `position ${sourceOf(t.indices[0])} of ${sourceOf(t.target)} exists`, st, [], and(cmp('<=', num(0), idx[0]), cmp('<', idx[0], length(container)))); + return build(t.target, app('STORE', [container, idx[0], value], container.t)); + } + if (t.kind === 'Field') { + const r = this.tr(t.target, n, []); + return build(t.target, app(`FSTORE.${up(t.field)}`, [r, value], 'record')); + } + return value; + }; + n.env.set(up(root), this.shallow(build(target, v), target, root)); + } + + builtin(s: PNode, st: State): State { + const [first, ...rest]: PNode[] = s.args; + const vals: Term[] = rest.map((a) => this.tr(a, st, [])); + const coll = this.tr(first, st, []); + const n = st.clone(); + let nv: Term; + switch (s.name) { + // a value going into a list of a sized type is stored as that type: checked, or rounded to a FLOAT32 + case 'APPEND': nv = app('APPENDED', [coll, this.fit(first.type?.of, vals[0], rest[0]?.type, rest[0], st, [], sourceOf(rest[0]))], 'list'); break; + case 'ADD': nv = app('SETADD', [coll, vals[0]], 'set'); break; + case 'DELETE': nv = app(coll.t === 'map' ? 'MAPDEL' : 'SETDEL', [coll, vals[0]], coll.t); break; + case 'INSERT': { + this.obligation('index', s, `INSERT's position ${sourceOf(rest[0])} lies in [0, LENGTH(${sourceOf(first)})]`, st, [], and(cmp('<=', num(0), vals[0]), cmp('<=', vals[0], length(coll)))); + if (rest[1]) this.fit(first.type?.of, vals[1], rest[1].type, rest[1], st, [], sourceOf(rest[1])); + nv = this.fresh(sourceOf(first), 'list'); + n.facts.push(cmp('==', length(nv), add(length(coll), num(1)))); + break; + } + case 'REMOVE': { + this.obligation('index', s, `position ${sourceOf(rest[0])} of ${sourceOf(first)} exists`, st, [], and(cmp('<=', num(0), vals[0]), cmp('<', vals[0], length(coll)))); + nv = this.fresh(sourceOf(first), 'list'); + n.facts.push(cmp('==', length(nv), sub(length(coll), num(1)))); + break; + } + case 'SORT': nv = app('SORTLIST', [coll], coll.t); break; + default: nv = this.fresh(sourceOf(first), coll.t); + } + this.store(first, nv, n, st); + return n; + } + + procCall(s: PNode, st: State): State { + const sub = s.sub; + if (!sub) return st; + const args: Term[] = s.args.map((a: PNode) => this.tr(a, st, [])); + sub.params.forEach((p: PNode, i: number) => { + if (!args[i] || !s.args[i]) return; + if (p.mode !== 'BYREF') args[i] = this.fit(p.type, args[i], s.args[i].type, s.args[i], st, [], `the value for ${p.name}`); + else this.arrayFit(p.type, args[i], s.args[i], st, [], `the ARRAY passed for ${p.name}`); + }); + const paramEnv: Env = new Map(sub.params.map((p: PNode, i: number): [string, Val] => [up(p.name), args[i]])); + for (const r of sub.requires) { + this.obligation('require', s, `this call to ${sub.name} meets its REQUIRE, ${sourceOf(r.expr)}`, st, [], this.tr(r.expr, st, [], { paramEnv, spec: true }), { target: r.line }); + } + this.recursion(sub, s, args, st, []); + const n = st.clone(); + const after = new Map(paramEnv); + sub.params.forEach((p: PNode, i: number) => { + if (p.mode === 'BYREF' && s.args[i].kind === 'Name') { + const fresh = this.fresh(s.args[i].name, sortOf(p.type)); + n.env.set(up(s.args[i].name), fresh); + after.set(up(p.name), fresh); + } + }); + for (const en of sub.ensures) n.facts.push(this.tr(en.expr, n, [], { paramEnv: after, oldEnv: paramEnv, spec: true })); + return n; + } + + ensure(at: PNode, st: State, value: Term | null): void { + const u = this.unit; + if (!u.ensures?.length) return; + const paramEnv: Env = new Map(); + for (const p of u.params || []) { + const k = up(p.name); + paramEnv.set(k, (p.mode === 'BYREF' ? st.env.get(k) : this.entryEnv.get(k)) ?? UNSET); + } + for (const en of u.ensures) { + const goal = this.tr(en.expr, st, [], { paramEnv, result: value, oldEnv: this.entryEnv, spec: true }); + this.obligation('ensure', at, `${u.name} keeps its ENSURE, ${sourceOf(en.expr)}`, st, [], goal, { target: en.line }); + } + } + + ifStmt(s: PNode, st: State, ctx: Ctx): State[] { + const out: State[] = []; + let rest = st; + for (let i = 0; i < s.branches.length; i++) { + const b = s.branches[i]; + const c = this.tr(b.cond, rest, []); + out.push(...this.run(b.body, [rest.assume(c, `line ${b.line}: ${sourceOf(b.cond)}`)], ctx)); + rest = rest.assume(not(c), `line ${b.line}: NOT (${sourceOf(b.cond)})`); + } + if (s.elseBody) out.push(...this.run(s.elseBody, [rest], ctx)); + else out.push(rest); + return out; + } + + caseStmt(s: PNode, st: State, ctx: Ctx): State[] { + const v = s.subject ? this.tr(s.subject, st, []) : this.fresh('case', 'any'); + const out: State[] = []; + let rest = st; + for (const arm of s.arms) { + // a label is evaluated only when no label before it in the arm matched: what translating it learns is guarded by that + const conds: Term[] = []; + for (const l of arm.labels as PNode[]) { + const g = conds.map((x) => not(x)); + conds.push(l.value ? cmp('==', v, this.tr(l.value, rest, g)) : and(cmp('<=', this.tr(l.low, rest, g), v), cmp('<', v, this.tr(l.high, rest, g)))); + } + const c = or(conds); + out.push(...this.run(arm.body, [rest.assume(c, `line ${arm.line}`)], ctx)); + rest = rest.assume(not(c)); + } + if (s.otherwise) out.push(...this.run(s.otherwise, [rest], ctx)); + else out.push(rest); + return out; + } + + /* ---------- loops ---------- */ + + /** The state at the top of a loop: changed variables become fresh symbols. */ + havoc(st: State, modified: Modified, loopLine: number | undefined): State { + const n = st.clone(); + for (const k of modified) { + const old = st.env.get(k); + const name = this.types.get(k) ? [...(this.unit.symbols || []), ...(this.unit.params || [])].find((x) => up(x.name) === k)?.name || k : k; + const known = isSet(old) ? old : null; + // the unknown has the sort of the variable's type, not of the value it happened to hold: a FLOAT that starts at 1 + // is not a whole number on the next pass (library/pseudo/prover/havoc-float-sort.pseudo) + const fresh = this.fresh(name, this.types.get(k) ? this.sortOfName(k) : known && known.t ? known.t : this.sortOfName(k)); + n.env.set(k, fresh); + // an ARRAY keeps its dimensions however its contents change, unless the body can replace it whole (A <- B) + if (known && known.t === 'array' && !modified.whole.has(k)) n.facts.push(cmp('==', length(fresh), length(known)), cmp('==', columns(fresh), columns(known))); + // a sized whole number stays in its range: every store into it is checked, and the run stops otherwise + const ty = this.types.get(k); + if (isSizedInt(ty)) n.facts.push(this.rangeOf(ty!, fresh)); + } + n.trail.push(`the top of the loop on line ${loopLine}`); + return n; + } + + userInvariants(s: PNode, st: State, env?: Env): Array<{ node: PNode; term: Term }> { + const view = new State(env || st.env, st.facts, st.trail); + return (s.invariants || []).map((inv: PNode) => ({ node: inv, term: this.tr(inv.expr, view, [], { spec: true }) })); + } + + forStmt(s: PNode, E: State, ctx: Ctx): State[] { + const a = this.tr(s.start, E, []); + const b = this.tr(s.stop, E, []); + const step: number | null = s.step ? (s.step.kind === 'Num' ? s.step.value : null) : 1; + const I = up(s.var); + const modified = modifiedIn(s.body); + modified.delete(I); + // entry: the invariant with the counter at its start + const entryEnv = new Map(E.env); entryEnv.set(I, a); + for (const inv of this.userInvariants(s, E, entryEnv)) { + this.obligation('invariant-init', inv.node, `the INVARIANT holds before the first pass`, E, [], inv.term); + } + const counterFacts = (i: Term): Term => { + if (step === null) return TRUE; + // The counter never passes the stop by a whole step or more, unless the + // loop never ran at all and it still holds its start. + if (step > 0) return and(cmp('<=', a, i), or(cmp('<', i, add(b, num(step))), cmp('==', i, a))); + return and(cmp('>=', a, i), or(cmp('>', i, add(b, num(step))), cmp('==', i, a))); + }; + const guardOf = (i: Term): Term => (step === null ? this.fresh('guard', 'bool') : step > 0 ? cmp('<', i, b) : cmp('>', i, b)); + const makeHead = (extra: (h: State) => Term[]) => { + const H = this.havoc(E, modified, s.line); + const i = this.fresh(s.var, 'int'); + H.env.set(I, i); + H.facts.push(counterFacts(i)); + if (step !== null && Math.abs(step) > 1) { + // the counter moves in whole steps from its start: I == start + STEP * Passes + const passes = this.fresh('passes', 'int'); + H.facts.push(cmp('==', i, add(a, arith('*', num(step), passes))), cmp('>=', passes, num(0))); + } + for (const f of extra(H)) H.facts.push(f); + for (const inv of this.userInvariants(s, H)) H.facts.push(inv.term); + return { H, i }; + }; + // a STEP that is not a plain number moves the counter by an amount the checker does not follow: an unknown + const advance = (st: State, _i: Term): State => { const n = st.clone(); n.env.set(I, step === null ? this.fresh(s.var, 'int') : add(asTerm(n.env.get(I), s.var, 'int'), num(step))); return n; }; + const cands = this.opts.infer ? this.infer(s, E, modified, { + entryEnv: (env: Env) => { const m = new Map(env); m.set(I, a); return m; }, + makeHead, guardOf, advance, counter: step === null ? undefined : I, start: a, step: step ?? 1, + }) : []; + const { H, i } = makeHead((h) => cands.map((c) => c.at(h.env))); + const g = guardOf(i); + const inner: { breaks: State[]; continues: State[] } = { breaks: [], continues: [] }; + const top = H.assume(g, `line ${s.line}: ${s.var} < ${sourceOf(s.stop)}`); + // a counter declared with a sized type: the run checks at the top of each pass that it fits, and stops if not + const declared = s.declaredCounter?.type; + if (isSizedInt(declared)) this.fit(declared, i, undefined, s, top, [], s.var); + const ends = this.run(s.body, [top], inner); + for (const st of [...ends, ...inner.continues]) { + const n = advance(st, i); + for (const inv of this.userInvariants(s, n)) this.obligation('invariant-keep', inv.node, `each pass keeps the INVARIANT`, n, [], inv.term); + } + this.decreases(s, H, g, [...ends, ...inner.continues].map((st) => advance(st, i))); + const exit = H.assume(not(g), `line ${s.line}: the loop has ended`); + const after = [exit, ...inner.breaks].map((st) => { + const n = st.clone(); + if (s.declaredCounter) n.env.set(I, UNSET); else n.env.delete(I); + return n; + }); + return after; + } + + forEach(s: PNode, E: State, ctx: Ctx): State[] { + const coll = this.tr(s.iterable, E, []); + const X = up(s.var); + const modified = modifiedIn(s.body); + modified.delete(X); + // An INVARIANT that names the loop's own variable is about the value a pass is about to visit, and the run checks + // it at the top of each pass. One that does not is about the state between passes: the run checks it at the top of + // each pass and once more after the last (or instead of any, for an empty collection), so it is known on the way out. + const invs: PNode[] = s.invariants || []; + const names = (inv: PNode): boolean => [...namesIn(inv.expr)].some((n) => up(n) === X); + const own = invs.filter((i) => !names(i)); + const each = invs.filter(names); + const inColl = (x: Term): Term | null => { + if (coll.k === 'app' && coll.f === 'RANGE') return and(cmp('<=', coll.args[0], x), cmp('<', x, coll.args[1])); + return coll.t === 'list' || coll.t === 'set' || coll.t === 'map' ? member(x, coll) : null; + }; + /** The state about to start a pass on some value of the collection. */ + const next = (st: State): State => { const n = st.clone(); const x = this.fresh(s.var, sortOf(s.eachSym?.type)); n.env.set(X, x); return n.assume(inColl(x)); }; + const ask = (kind: string, label: string, st: State): void => { + for (const inv of own) this.obligation(kind, inv, label, st, [], this.tr(inv.expr, st, [], { spec: true })); + if (!each.length) return; + const n = next(st); + for (const inv of each) this.obligation(kind, inv, label, n, [], this.tr(inv.expr, n, [], { spec: true })); + }; + ask('invariant-init', `the INVARIANT holds before the first pass`, E); + const H = this.havoc(E, modified, s.line); + H.env.delete(X); + for (const inv of own) H.facts.push(this.tr(inv.expr, H, [], { spec: true })); + // X is in the collection only inside a pass. The loop also ends without + // any pass when the collection is empty, so the way out must not know it: + // for an empty list that fact is FALSE, and the checker would then prove + // anything after the loop (library/pseudo/prover/foreach-empty.pseudo). + const inPass = next(H); + for (const inv of each) inPass.facts.push(this.tr(inv.expr, inPass, [], { spec: true })); + const inner: { breaks: State[]; continues: State[] } = { breaks: [], continues: [] }; + const ends = this.run(s.body, [inPass], inner); + for (const st of [...ends, ...inner.continues]) ask('invariant-keep', `each pass keeps the INVARIANT`, st); + const out = [H, ...inner.breaks].map((st) => { const n = st.clone(); n.env.delete(X); return n; }); + return out; + } + + whileStmt(s: PNode, E: State, ctx: Ctx): State[] { + const modified = modifiedIn(s.body); + for (const inv of this.userInvariants(s, E)) this.obligation('invariant-init', inv.node, `the INVARIANT holds before the first pass`, E, [], inv.term); + const makeHead = (extra: (h: State) => Term[]) => { + const H = this.havoc(E, modified, s.line); + for (const f of extra(H)) H.facts.push(f); + for (const inv of this.userInvariants(s, H)) H.facts.push(inv.term); + return { H }; + }; + const guardOf = (H: State): Term => this.withQuiet(() => this.tr(s.cond, H, [])); + const cands = this.opts.infer ? this.infer(s, E, modified, { + entryEnv: (env: Env) => env, makeHead, guardOf: null, whileGuard: guardOf, advance: (st: State) => st, + }) : []; + const { H } = makeHead((h) => cands.map((c) => c.at(h.env))); + const g = this.tr(s.cond, H, []); + const inner: { breaks: State[]; continues: State[] } = { breaks: [], continues: [] }; + const ends = this.run(s.body, [H.assume(g, `line ${s.line}: ${sourceOf(s.cond)}`)], inner); + const back = [...ends, ...inner.continues]; + for (const st of back) { + for (const inv of this.userInvariants(s, st)) this.obligation('invariant-keep', inv.node, `each pass keeps the INVARIANT`, st, [], inv.term); + } + this.decreases(s, H, g, back, true, cands); + return [H.assume(not(g), `line ${s.line}: NOT (${sourceOf(s.cond)})`), ...inner.breaks]; + } + + repeatStmt(s: PNode, E: State, ctx: Ctx): State[] { + const modified = modifiedIn(s.body); + for (const inv of this.userInvariants(s, E)) this.obligation('invariant-init', inv.node, `the INVARIANT holds before the first pass`, E, [], inv.term); + const H = this.havoc(E, modified, s.line); + for (const inv of this.userInvariants(s, H)) H.facts.push(inv.term); + const inner: { breaks: State[]; continues: State[] } = { breaks: [], continues: [] }; + const ends = this.run(s.body, [H], inner); + const exits: State[] = []; + const again: State[] = []; + for (const st of [...ends, ...inner.continues]) { + const c = s.cond ? this.tr(s.cond, st, []) : TRUE; + const done = st.assume(c, `line ${s.endLine}: UNTIL ${sourceOf(s.cond)}`); + exits.push(done); + // the run checks the INVARIANT once more after the last pass + for (const inv of this.userInvariants(s, done)) this.obligation('invariant-keep', inv.node, `each pass keeps the INVARIANT`, done, [], inv.term); + const n = st.assume(not(c)); + again.push(n); + for (const inv of this.userInvariants(s, n)) this.obligation('invariant-keep', inv.node, `each pass keeps the INVARIANT`, n, [], inv.term); + } + this.decreases(s, H, TRUE, again, s.kind !== 'For'); + return [...exits, ...inner.breaks]; + } + + withQuiet(fn: () => T): T { this.quiet = (this.quiet || 0) + 1; try { return fn(); } finally { this.quiet--; } } + + /** + * Termination. A DECREASES line gives two obligations: the measure is at + * least 0 whenever the loop continues, and every pass makes it smaller. + * Without one, a WHILE loop gets a guess from its condition (N - I for + * I < N) that is kept only if both obligations prove. + */ + decreases(s: PNode, H: State, guard: Term, backStates: State[], needed = false, _cands: Cand[] = []): void { + if (s.decreases) { + const d0 = this.tr(s.decreases.expr, H, [], { spec: true }); + this.obligation('decreases-bound', s.decreases, `${sourceOf(s.decreases.expr)} is at least 0 while the loop continues`, H, [guard], cmp('>=', d0, num(0))); + for (const st of backStates) { + const d1 = this.tr(s.decreases.expr, st, [], { spec: true }); + this.obligation('decreases-step', s.decreases, `each pass makes ${sourceOf(s.decreases.expr)} smaller`, st, [], cmp('<', d1, d0)); + } + return; + } + if (!needed || s.kind === 'For' || !this.recording) return; + // guess a measure from the condition + const guesses: PNode[] = []; + const conds = s.cond ? conjuncts(s.cond) : []; + for (const c of conds) { + if (c.kind !== 'Binary') continue; + const [a, b] = [c.left, c.right]; + if (c.op === '<') guesses.push({ kind: 'Binary', op: '-', left: b, right: a, type: a.type }); + if (c.op === '<=') guesses.push({ kind: 'Binary', op: '+', left: { kind: 'Binary', op: '-', left: b, right: a, type: a.type }, right: { kind: 'Num', value: 1 }, type: a.type }); + if (c.op === '>') guesses.push({ kind: 'Binary', op: '-', left: a, right: b, type: a.type }); + if (c.op === '>=') guesses.push({ kind: 'Binary', op: '+', left: { kind: 'Binary', op: '-', left: a, right: b, type: a.type }, right: { kind: 'Num', value: 1 }, type: a.type }); + } + for (const gs of guesses) { + if (gs.left?.type?.k !== 'INTEGER' && gs.type?.k !== 'INTEGER') continue; + const d0 = this.withQuiet(() => this.tr(gs, H, [], { spec: true })); + const okBound = prove([...H.facts, guard], cmp('>=', d0, num(0))).status === 'proved'; + const okStep = okBound && backStates.every((st) => prove(st.facts, cmp('<', this.withQuiet(() => this.tr(gs, st, [], { spec: true })), d0)).status === 'proved'); + if (okBound && okStep) { + this.notes.push({ line: s.line as number, text: `The ${s.kind === 'While' ? 'WHILE' : 'REPEAT'} loop on line ${s.line} ends: ${sourceOf(gs)} is at least 0 whenever the loop continues, and every pass makes it smaller. The checker found this measure itself; writing it as DECREASES ${sourceOf(gs)} states it for a reader.` }); + this.obligation('terminates', s, `the ${s.kind === 'While' ? 'WHILE' : 'REPEAT'} loop on line ${s.line} ends`, H, [], TRUE, { measure: sourceOf(gs) }); + return; + } + } + this.obligation('terminates', s, `the ${s.kind === 'While' ? 'WHILE' : 'REPEAT'} loop on line ${s.line} ends`, H, [], null, { needsMeasure: true }); + } + + /* ---------- invariant inference ---------- */ + + /** + * Houdini: start from a list of simple candidate invariants, and throw out + * every one that fails on entry or fails to survive a pass, until what is + * left survives. What is left is proved, and the loop's real analysis + * assumes it. + */ + infer(s: PNode, E: State, modified: Set, h: InferHooks): Cand[] { + if (this.depth > 2) return []; + const cands: Cand[] = []; + const entry = h.entryEnv(E.env); + const counter = h.counter; + const startTerm = h.start ?? num(0); + const loopLine = s.line; + for (const k of modified) { + const v0 = E.env.get(k); + const nm = [...(this.unit.symbols || [])].find((x) => up(x.name) === k)?.name || k; + if (!v0 || v0 === UNSET) continue; + if (!isSet(v0)) continue; + const t = v0.t; + if (numeric(t)) { + cands.push({ text: `${nm} >= ${show(v0)}`, at: (env) => cmp('>=', valOf(env, k), v0) }); + cands.push({ text: `${nm} <= ${show(v0)}`, at: (env) => cmp('<=', valOf(env, k), v0) }); + if (counter) cands.push({ text: `${nm} <= ${show(v0)} + ${counter} - ${show(startTerm)}`, at: (env) => cmp('<=', valOf(env, k), add(v0, sub(valOf(env, counter), startTerm))) }); + } + if (t === 'list' || t === 'string') { + cands.push({ text: `LENGTH(${nm}) >= LENGTH(before the loop)`, at: (env) => cmp('>=', length(valOf(env, k)), length(v0)) }); + if (counter) { + const passes = (env: Env): Term => ((h.step ?? 1) < 0 ? sub(startTerm, valOf(env, counter)) : sub(valOf(env, counter), startTerm)); + cands.push({ text: `LENGTH(${nm}) <= LENGTH(before) + passes`, at: (env) => cmp('<=', length(valOf(env, k)), add(length(v0), passes(env))) }); + cands.push({ text: `LENGTH(${nm}) == LENGTH(before) + passes`, at: (env) => cmp('==', length(valOf(env, k)), add(length(v0), passes(env))) }); + } + } + } + // from a WHILE condition: I < N suggests I <= N + if (s.kind === 'While') { + for (const c of conjuncts(s.cond)) { + if (c.kind !== 'Binary' || !['<', '<=', '>', '>='].includes(c.op)) continue; + const l = c.left; const r = c.right; + if (l.kind === 'Name' && modified.has(up(l.name)) && l.type?.k === 'INTEGER') { + const op = c.op === '<' ? '<=' : c.op === '>' ? '>=' : c.op; + const extra = c.op === '<=' ? 1 : c.op === '>=' ? -1 : 0; + cands.push({ text: `${l.name} ${op} ${sourceOf(r)}${extra ? (extra > 0 ? ' + 1' : ' - 1') : ''}`, at: (env) => { const st = new State(env, []); const R = this.withQuiet(() => this.tr(r, st, [], { spec: true })); return cmp(op as CmpOp, valOf(env, up(l.name)), extra ? add(R, num(extra)) : R); } }); + } + } + } + if (!cands.length) return []; + let lastHead: State | null = null; + let live = cands.filter((c) => { + try { return prove(E.facts, c.at(entry)).status === 'proved'; } catch { return false; } + }); + const saved = this.recording; + this.recording = false; + this.depth++; + // Only a set that survives a whole pass unchanged is inductive. Running + // out of rounds while candidates are still being dropped proves nothing + // about the ones left, so then nothing is kept (library/pseudo/prover/houdini-cap). + let stable = live.length === 0; + try { + for (let round = 0; round < 8 && live.length; round++) { + const { H, i } = h.makeHead((hh) => live.map((c) => c.at(hh.env))); + lastHead = H; + let start = H; + if (h.guardOf) start = H.assume(h.guardOf(i as Term)); + else if (h.whileGuard) start = H.assume(h.whileGuard(H)); + const inner = { breaks: [], continues: [] }; + const ends = this.run(s.body, [start], inner).concat(inner.continues).map((st) => h.advance(st, i as Term)); + const keep = live.filter((c) => ends.every((st) => { + try { return prove(st.facts, c.at(st.env)).status === 'proved'; } catch { return false; } + })); + if (keep.length === live.length) { stable = true; break; } + live = keep; + if (!live.length) stable = true; + } + } finally { + this.recording = saved; + this.depth--; + } + if (!stable) live = []; + if (saved && live.length) { + const env = lastHead ? lastHead.env : entry; + const texts = live.filter((c) => { try { return prove([], c.at(env)).status !== 'proved'; } catch { return true; } }) + .map((c) => { try { return show(c.at(env)); } catch { return c.text; } }); + this.inferred.push({ line: loopLine as number, facts: [...new Set(texts)] }); + } + return live; + } + + /* ---------- a whole unit ---------- */ + runUnit(): void { + const u = this.unit; + const st = new State(); + this.entryEnv = new Map(); + for (const p of u.params || []) { + const v = this.fresh(p.name, sortOf(p.type)); + st.env.set(up(p.name), v); + this.entryEnv.set(up(p.name), v); + // a parameter of a sized type holds a value of that type: the call checked it on the way in + if (isSizedInt(p.type)) st.facts.push(this.rangeOf(p.type, v)); + // and an ARRAY parameter has the lengths it is declared with (interp.js `fixedShape`) + const dims = knownDims(p.type); + if (dims) { st.facts.push(cmp('==', length(v), num(dims[0]))); if (dims.length > 1) st.facts.push(cmp('==', columns(v), num(dims[1]))); } + } + for (const r of u.requires || []) st.facts.push(this.tr(r.expr, st, [], { spec: true })); + // The run checks a DECREASES measure on every call, the first one too: it has to be 0 or more for every call the + // REQUIREs allow (library/pseudo/prover/decreases-entry.pseudo). + if (u.sub?.decreases) this.obligation('decreases-bound', u.sub.decreases, `${sourceOf(u.sub.decreases.expr)} is at least 0 on every call`, st, [], cmp('>=', this.tr(u.sub.decreases.expr, st, [], { spec: true }), num(0))); + const ends = this.run(u.body, [st], { breaks: [], continues: [] }); + if (u.kind === 'Procedure') for (const e of ends) this.ensure({ kind: 'End', line: u.node!.endLine, col: 1 }, e, null); + } +} + +function conjuncts(e: PNode | null | undefined): PNode[] { + if (!e) return []; + if (e.kind === 'Binary' && e.op === 'AND') return [...conjuncts(e.left), ...conjuncts(e.right)]; + return [e]; +} + +function fieldRead(r: Term, field: string, s: Sort): Term { + if (r.k === 'app' && r.f.startsWith('FSTORE.')) return r.f === `FSTORE.${up(field)}` ? r.args[1] : fieldRead(r.args[0], field, s); + return app(`FIELD.${field}`, [r], s); +} + +function columns(x: Term): Term { + if (x.k === 'app' && x.f === 'NEWARRAY') return x.args[2] || num(0); + if (x.k === 'app' && x.f === 'STORE2') return columns(x.args[0]); + return app('COLUMNS', [x], 'int'); +} + +/* ------------------------------------------------------------------ */ +/* Grouping and proving */ +/* ------------------------------------------------------------------ */ + +/** The symbols of each fact, kept: one fact is among the hypotheses of every obligation after it. */ +const SYMS = new WeakMap(); +const symKeys = (t: Term): string[] => { let ks = SYMS.get(t); if (!ks) { ks = [...symsOf(t).keys()]; SYMS.set(t, ks); } return ks; }; +/** The hypotheses looked through for the facts shown with an obligation: the ones nearest it. */ +const SHOWN_FROM = 400; + +/** The facts that share a symbol with the goal, directly or through another fact. */ +function relevantFacts(all: Term[], goal: Term, limit = 14): Term[] { + const hyps = all.length > SHOWN_FROM ? all.slice(all.length - SHOWN_FROM) : all; + const want = new Set(symKeys(goal)); + const chosen = new Set(); + for (let round = 0; round < 3; round++) { + let grew = false; + hyps.forEach((h, i) => { + if (chosen.has(i)) return; + const ks = symKeys(h); + if (ks.some((k) => want.has(k))) { chosen.add(i); for (const k of ks) if (!want.has(k)) { want.add(k); grew = true; } } + }); + if (!grew) break; + } + return [...chosen].sort((a, b) => a - b).slice(0, limit).map((i) => hyps[i]); +} + +/** Display names: the newest version of I prints as I, older ones as I0, I1, ... */ +function namesFor(terms: Term[]): Map { + const all = new Map(); + for (const t of terms) symsOf(t, all); + const byName = new Map>(); + for (const s of all.values()) { let l = byName.get(s.name); if (!l) { l = []; byName.set(s.name, l); } l.push(s); } + const names = new Map(); + for (const [n, list] of byName) { + list.sort((a, b) => a.id - b.id); + list.forEach((s, i) => names.set(key(s), i === list.length - 1 ? n : `${n}${'₀₁₂₃₄₅₆₇₈₉'[i] || i}`)); + } + return names; +} + +/** + * The work (logic.ts `work`) the checker may do on one subroutine: following its paths and finding its loops' + * invariants (UNIT_WORK), then proving what it found (PROOF_WORK); and on a whole program (PROGRAM_WORK). Each is + * counted, so it runs out at the same point on every machine, and each is far past what any program of the corpus + * or of prove-fuzz takes (tests/prover-limits.test.mjs). Past the first, the subroutine is partial: nothing in it + * is called proved, and a note says why. Past the second, the obligations not yet proved are open, too large to + * prove here. The wall-clock budget (`budgetMs`) stays as a last resort. + */ +export const UNIT_WORK = 40_000_000; +/** The deepest value the checker follows (Exec.shallow); the corpus's deepest is far below (tests/prover-limits.test.mjs). */ +export const MAX_DEPTH = 500; +export const PROOF_WORK = 20_000_000; +export const PROGRAM_WORK = 60_000_000; +/** + * The characters of facts and goals one unit's report writes out. Past them an obligation shows its goal alone, cut + * at CLIP characters: each fact can be a thousand parts long (logic.ts `show`), and a long subroutine has an + * obligation on almost every line. + */ +const SHOWN_CHARS = 1_000_000; +const CLIP = 400; +const clip = (s: string): string => (s.length > CLIP ? `${s.slice(0, CLIP)}…` : s); +/** Work or size past what the checker can do: its own budget, or the engine's stack or a string or array too long. */ +const tooLarge = (err: unknown): boolean => err instanceof OutOfWork || err instanceof RangeError; +const TOO_LARGE_NOTE = 'This subroutine is too large to prove here: the checker stopped before the end of it, so nothing in it is marked proved. Every run still checks it.'; + +export function verifyUnit(program: Program, unit: Unit, options: VerifyOptions = {}): UnitReport { + const ex = new Exec(program, unit, options); + ex.deadline = Date.now() + (options.budgetMs ?? 4000); + try { + metered(UNIT_WORK, () => ex.runUnit(), () => { throw new OutOfWork(); }); + } catch (err) { + ex.partial = true; + ex.notes.push({ line: unit.node?.line ?? 1, ...(tooLarge(err) ? { kind: 'too-large', text: TOO_LARGE_NOTE } : { text: `The checker stopped early: ${(err as Error).message}` }) }); + } + // The number model behind "proved": INTEGER is a whole number of any size. + // Neither the lab's limit (a run stops past 2^53 - 1) nor Mojo's 64-bit Int + // (which wraps past 2^63 - 1, probes/int-overflow-int) is part of the proof. + if (ex.floatArith) ex.notes.push({ line: unit.node?.line ?? 1, kind: 'float-model', text: 'The proofs here treat a FLOAT as a finite number and assume nothing about a rounded result except that the same calculation gives it again. A run in this lab stops when a FLOAT passes about 1.8 × 10^308; in the Mojo made from this program, a Float64 division by zero gives inf or nan and the run goes on (probes/float-special).' }); + if (ex.intArith) ex.notes.push({ line: unit.node?.line ?? 1, kind: 'int-model', text: 'The proofs here treat INTEGER as a whole number of any size. A run in this lab stops when a whole number passes 9,007,199,254,740,991, and the Mojo made from this program uses a 64-bit Int, which wraps around past 9,223,372,036,854,775,807. Neither limit is part of the proof.' }); + type Group = { kind: string; line: number | null; col: number; label: string; instances: RawOb[]; target?: number; measure?: string; needsMeasure?: boolean; recursive?: 'self' | 'mutual' }; + const groups = new Map(); + for (const o of ex.raw) { + const k = `${o.kind}@${o.line}:${o.col}@${o.label}`; + if (!groups.has(k)) groups.set(k, { kind: o.kind, line: o.line, col: o.col, label: o.label, instances: [], target: o.target, measure: o.measure, needsMeasure: o.needsMeasure, recursive: o.recursive }); + groups.get(k)!.instances.push(o); + } + const obligations: Obligation[] = []; + let shownLeft = SHOWN_CHARS; + // once the unit's proving budget is spent, every instance not yet proved is open + let spent = false; + const decided = new Map(); + metered(PROOF_WORK, () => { + for (const gr of groups.values()) { + let status: 'proved' | 'unknown' = 'proved'; + let failing: RawOb | null = null; + if (gr.needsMeasure) { status = 'unknown'; failing = gr.instances[0]; } + else { + for (const inst of gr.instances) { + if (spent) { status = 'unknown'; failing = inst; inst.reason = TOO_LARGE; break; } + if (Date.now() > ex.deadline + 2000) { status = 'unknown'; failing = inst; inst.reason = 'the checker ran out of time'; break; } + let r: import('./solver.js').ProveResult; + try { r = inst.goal ? prove(inst.hyps, inst.goal) : { status: 'unknown', reason: 'there is no goal to prove' }; } catch (err) { + if (err instanceof OutOfWork) spent = true; + r = { status: 'unknown', reason: tooLarge(err) ? TOO_LARGE : (err as Error).message }; + } + inst.status = r.status; + if (r.status !== 'proved') { status = 'unknown'; failing = inst; inst.reason = r.reason; inst.witness = r.witness; break; } + } + } + decided.set(gr, { status, failing }); + } + }, () => undefined); + for (const gr of groups.values()) { + const { status, failing } = decided.get(gr)!; + const shownInst = failing || gr.instances[0]; + let shown: Obligation['shown'] = null; + // what is shown is written out within its own limits (show, symsOf, SHOWN_CHARS), whatever the proof had left + if (shownInst && shownInst.goal) shown = unmetered(() => { try { + const goal = shownInst.goal!; + if (shownLeft <= 0) return { facts: [], goal: clip(show(goal, namesFor([goal]))), trail: shownInst.trail, reason: shownInst.reason }; + const facts = relevantFacts(shownInst.hyps, goal); + const names = namesFor([...facts, goal]); + const out: NonNullable = { facts: facts.map((f) => show(f, names)), goal: show(goal, names), trail: shownInst.trail, reason: shownInst.reason }; + if (options.debug && shownInst.witness) out.witness = shownInst.witness.map((l) => show(l, namesFor([...shownInst.hyps, goal]))); + shownLeft -= out.goal.length + out.facts.reduce((n, f) => n + f.length, 0); + return out; + } catch (err) { + if (!tooLarge(err)) throw err; + return { facts: [], goal: '…', trail: shownInst.trail, reason: shownInst.reason }; + } }); + const { instances, ...place } = gr; + obligations.push({ ...place, status, paths: instances.length, shown, instances: undefined } as Obligation); + } + obligations.sort((a, b) => (a.line ?? 0) - (b.line ?? 0) || KIND_ORDER.indexOf(a.kind) - KIND_ORDER.indexOf(b.kind)); + return { name: unit.name, kind: unit.kind, line: unit.node?.line ?? 1, obligations, inferred: ex.inferred, notes: ex.notes, partial: ex.partial }; +} + +const KIND_ORDER = ['require', 'index', 'key', 'slice', 'divide', 'overflow', 'argument', 'invariant-init', 'invariant-keep', 'decreases-bound', 'decreases-step', 'terminates', 'assert', 'ensure']; + +export function verify(program: Program, options: VerifyOptions = {}): Report { + const units: UnitReport[] = []; + // one budget for the whole program: a unit after it is spent is partial, and says so (verifyUnit catches it) + metered(PROGRAM_WORK, () => { + for (const sub of program.subs.values()) { + units.push(verifyUnit(program, { name: sub.name, kind: sub.kind, node: sub.node, body: sub.node.body, params: sub.params, symbols: sub.symbols || [], requires: sub.requires, ensures: sub.ensures, sub }, options)); + } + if (program.main.body.length) units.push(verifyUnit(program, { name: 'main program', kind: 'main', node: null, body: program.main.body, params: [], symbols: program.main.symbols, requires: [], ensures: [] }, options)); + }, () => undefined); + // USE IEEE lets NaN and infinities into the run, and the prover's reasoning about numbers (X == X; X < Y or + // X >= Y) does not hold for them, so nothing in such a program is called proved. + if ((program as any).ieee) { + for (const u of units) { + u.notes.push({ line: 1, kind: 'ieee', text: 'This program says USE IEEE, so a FLOAT may be NaN or infinite, and the prover\'s reasoning about numbers does not cover those values. Nothing here is marked proved; the obligations are listed so you can see what each line relies on.' }); + for (const o of u.obligations) if (o.status === 'proved') (o as any).status = 'unknown'; + } + } + const all = units.flatMap((u) => u.obligations); + return { units, counts: { proved: all.filter((o) => o.status === 'proved').length, unknown: all.filter((o) => o.status !== 'proved').length, total: all.length } }; +} + +export const OBLIGATION_WORDS: Record = { + index: 'Position exists', + key: 'Key present', + slice: 'Slice in range', + divide: 'No division by zero', + overflow: 'Fits its type', + argument: 'Argument allowed', + require: 'Precondition met', + ensure: 'Postcondition holds', + 'invariant-init': 'Invariant on entry', + 'invariant-keep': 'Invariant kept', + 'decreases-bound': 'Measure stays at 0 or above', + 'decreases-step': 'Measure shrinks', + terminates: 'Loop or recursion ends', + assert: 'Assertion holds', +}; diff --git a/src/engines/pseudo/views-analysis.js b/src/engines/pseudo/views-analysis.js new file mode 100644 index 0000000..749cbfc --- /dev/null +++ b/src/engines/pseudo/views-analysis.js @@ -0,0 +1,803 @@ +/** + * pseudo/views-analysis.js + * + * Views about a program as a whole rather than one moment of one run: the + * coach's findings, how values flow, the control-flow chart, the four jobs, + * what the checker can prove, test results, generated checks, how the work + * grows, and the Mojo translation. Each is a factory (ctx, options) -> + * { node, update(frame) }, like the views in views.js; most draw once and + * use `update` only to highlight whatever the current step touches. + */ + +import { el, sv, add, clear, panel, plural, commas, legend, guideLink, button } from './dom.js'; +import { lineHtml, esc } from './highlight.js'; +import { sourceOf } from './parse.js'; +import { formatValue, typeName } from './lang.js'; +import { verify, OBLIGATION_WORDS } from './vcgen.js'; +import { parseTests, runTests, checkProperties, measureGrowth } from './testing.js'; +import { translate } from './translate.js'; +import { jobsReport, PATTERN_NAMES } from './plan.js'; +import { walkStatements, childBlocks } from './analyze.js'; + +const up = (s) => String(s).toUpperCase(); + +/** The analysis unit a view should show: the one being called, or the first. */ +function unitFor(ctx, name) { + const units = ctx.analysis?.units || []; + if (name) return units.find((u) => up(u.name) === up(name)) || units[0]; + if (ctx.call) return units.find((u) => up(u.name) === up(ctx.call.name)) || units[0]; + return units.find((u) => u.kind === 'main') || units[0]; +} + +/* ------------------------------------------------------------------ */ +/* Findings */ +/* ------------------------------------------------------------------ */ + +const SEVERITY_WORD = { error: 'Error', warning: 'Warning', note: 'Note' }; + +/** + * Every finding, errors first, each with the line it is about, the rule in + * a sentence, and a link to the guide's paragraph on that rule. + * + * options: { notes: true, onSelect(line), max } + */ +export function diagnosticsView(ctx, options = {}) { + const opts = { notes: true, max: 60, ...options }; + const body = el('div', { class: 'pl-diags' }); + const diags = ctx.study.diags.filter((d) => opts.notes || d.severity !== 'note'); + const counts = { error: 0, warning: 0, note: 0 }; + for (const d of diags) counts[d.severity]++; + if (!diags.length) add(body, el('p', { class: 'pl-ok-line', text: 'Nothing to report: the program is well formed, and the coach found none of the mistakes it looks for.' })); + const RANK = { error: 0, warning: 1, note: 2 }; + const sorted = [...diags].sort((a, b) => RANK[a.severity] - RANK[b.severity] || a.line - b.line); + for (const d of sorted.slice(0, opts.max)) { + const item = el('div', { class: `pl-diag is-${d.severity}` }, [ + el('span', { class: 'pl-diag-icon', text: d.severity === 'error' ? '✖' : d.severity === 'warning' ? '▲' : '•', 'aria-hidden': 'true' }), + el('div', { class: 'pl-diag-body' }, [ + el('div', { class: 'pl-diag-head' }, [el('strong', { text: `${SEVERITY_WORD[d.severity]}, line ${d.line}` }), d.fix ? el('span', { class: 'pl-tag', text: 'can be fixed automatically' }) : null]), + el('p', { text: d.message }), + d.hint ? el('p', { class: 'pl-muted', text: d.hint }) : null, + guideLink(d.guide, 'What the guide says'), + ]), + ]); + const select = opts.onSelect || (ctx.widget ? (line) => ctx.widget.gotoLine(line) : null); + if (select) { item.classList.add('is-clickable'); item.addEventListener('click', (e) => { if (e.target.tagName !== 'A') select(d.line, d); }); } + body.appendChild(item); + } + if (sorted.length > opts.max) add(body, el('p', { class: 'pl-muted', text: `… and ${sorted.length - opts.max} more.` })); + const summary = [counts.error && plural(counts.error, 'error'), counts.warning && plural(counts.warning, 'warning'), counts.note && plural(counts.note, 'note')].filter(Boolean).join(', '); + const node = panel(options.title ?? `Coach${summary ? `: ${summary}` : ''}`, body, 'pl-diags-panel'); + return { node, update() { } }; +} + +/* ------------------------------------------------------------------ */ +/* How values flow */ +/* ------------------------------------------------------------------ */ + +/** + * A graph of which values feed which. Inputs sit on the left, what the + * procedure hands back on the right. A solid arrow carries a value (Total + * is computed from Total); a dashed arrow decides whether a line runs at + * all (T[I] == B decides whether Total changes). A loop on a node marks an + * accumulator, a name that is computed from its own old value. + * + * options: { unit: 'COUNT', controls: true, width } + */ +export function flowView(ctx, options = {}) { + const opts = { controls: true, ...options }; + const unit = unitFor(ctx, opts.unit); + const wrap = el('div', { class: 'pl-flow' }); + if (!unit) return { node: panel('How values flow', el('p', { class: 'pl-muted', text: 'Nothing to draw.' })), update() { } }; + const g = unit.graph; + const nodes = g.nodes.filter((n) => n.role !== 'constant' || g.edges.some((e) => e.from === n.id)); + const edges = g.edges.filter((e) => (opts.controls || e.kind === 'data') && nodes.some((n) => n.id === e.from) && nodes.some((n) => n.id === e.to)); + // layers: longest path from the sources, ignoring edges that close a cycle + const out = new Map(nodes.map((n) => [n.id, []])); + for (const e of edges) if (e.from !== e.to) out.get(e.from).push(e.to); + const order = []; const state = new Map(); const back = new Set(); + const dfs = (v) => { state.set(v, 1); for (const w of out.get(v) || []) { if (state.get(w) === 1) back.add(`${v}>${w}`); else if (!state.get(w)) dfs(w); } state.set(v, 2); order.push(v); }; + const roots = nodes.filter((n) => n.role === 'input' || n.role === 'source' || n.role === 'constant'); + for (const n of [...roots, ...nodes]) if (!state.get(n.id)) dfs(n.id); + order.reverse(); + const layer = new Map(nodes.map((n) => [n.id, 0])); + for (const v of order) for (const w of out.get(v) || []) if (!back.has(`${v}>${w}`)) layer.set(w, Math.max(layer.get(w), layer.get(v) + 1)); + const maxLayer = Math.max(0, ...layer.values()); + for (const n of nodes) if (n.role === 'output') layer.set(n.id, Math.max(maxLayer, layer.get(n.id))); + const layers = []; + for (const n of nodes) { const l = layer.get(n.id); (layers[l] ||= []).push(n); } + // order within a layer by the average position of what feeds it + const pos = new Map(); + layers.forEach((L, i) => { + if (i > 0) L.sort((a, b) => avgIn(a) - avgIn(b)); + L.forEach((n, k) => pos.set(n.id, k)); + function avgIn(n) { const ins = edges.filter((e) => e.to === n.id && e.from !== n.id && layer.get(e.from) < i).map((e) => pos.get(e.from) ?? 0); return ins.length ? ins.reduce((a, b) => a + b, 0) / ins.length : 99; } + }); + const colW = 150; const rowH = 62; const nodeW = 104; const nodeH = 34; const padX = 18; const padY = 30; + const rows = Math.max(1, ...layers.map((L) => (L ? L.length : 0))); + const W = padX * 2 + (layers.length - 1) * colW + nodeW; + const H = padY * 2 + (rows - 1) * rowH + nodeH; + const xy = new Map(); + layers.forEach((L, i) => (L || []).forEach((n, k) => { + const off = ((rows - L.length) * rowH) / 2; + xy.set(n.id, { x: padX + i * colW, y: padY + off + k * rowH }); + })); + const svg = sv('svg', { viewBox: `0 0 ${W} ${H}`, width: W, height: H, class: 'pl-flow-svg', role: 'img', 'aria-label': `How values flow in ${unit.name}` }); + add(svg, sv('defs', {}, [ + sv('marker', { id: `plArrow${unit.name}`, viewBox: '0 0 10 10', refX: '9', refY: '5', markerWidth: '7', markerHeight: '7', orient: 'auto-start-reverse' }, [sv('path', { d: 'M0,0 L10,5 L0,10 z', class: 'pl-flow-head' })]), + ])); + const edgeEls = []; + for (const e of edges) { + const a = xy.get(e.from); const b = xy.get(e.to); + if (!a || !b) continue; + let d; + if (e.from === e.to) { + const cx = a.x + nodeW / 2; + d = `M${cx - 14},${a.y} C${cx - 30},${a.y - 30} ${cx + 30},${a.y - 30} ${cx + 14},${a.y}`; + } else if (b.x > a.x) { + const x1 = a.x + nodeW; const y1 = a.y + nodeH / 2; const x2 = b.x; const y2 = b.y + nodeH / 2; + const mx = (x1 + x2) / 2; + d = `M${x1},${y1} C${mx},${y1} ${mx},${y2} ${x2 - 2},${y2}`; + } else { + const x1 = a.x + nodeW / 2; const y1 = a.y + nodeH; const x2 = b.x + nodeW / 2; const y2 = b.y + nodeH; + const dip = Math.max(y1, y2) + 26; + d = `M${x1},${y1} C${x1},${dip} ${x2},${dip} ${x2},${y2 + 2}`; + } + const p = sv('path', { d, class: `pl-flow-edge is-${e.kind}`, 'marker-end': `url(#plArrow${unit.name})` }); + add(p, sv('title', { text: `${labelOf(e.from)} ${e.kind === 'data' ? 'is used to compute' : 'decides whether this changes:'} ${labelOf(e.to)} (line${e.lines.length > 1 ? 's' : ''} ${e.lines.join(', ')})` })); + svg.appendChild(p); + edgeEls.push({ e, p }); + } + function labelOf(id) { return nodes.find((n) => n.id === id)?.label || id; } + const nodeEls = new Map(); + for (const n of nodes) { + const p = xy.get(n.id); + const gEl = sv('g', { class: `pl-flow-node is-${n.role}${n.accumulator ? ' is-acc' : ''}`, transform: `translate(${p.x},${p.y})` }); + add(gEl, sv('rect', { width: nodeW, height: nodeH, rx: 7 })); + add(gEl, sv('text', { x: nodeW / 2, y: nodeH / 2 - 1, 'text-anchor': 'middle', class: 'pl-flow-label', text: n.label })); + add(gEl, sv('text', { x: nodeW / 2, y: nodeH / 2 + 11, 'text-anchor': 'middle', class: 'pl-flow-sub', text: n.role === 'input' ? 'input' : n.role === 'output' ? 'the answer' : n.role === 'counter' ? 'loop counter' : n.accumulator ? 'accumulator' : n.role === 'source' ? 'typed in' : n.role === 'constant' ? 'constant' : '' })); + add(gEl, sv('title', { text: `${n.label}${n.type ? `: ${typeName(n.type)}` : ''}${n.lines?.length ? `, set on line${n.lines.length > 1 ? 's' : ''} ${n.lines.join(', ')}` : ''}` })); + svg.appendChild(gEl); + nodeEls.set(n.id, gEl); + } + add(wrap, el('div', { class: 'pl-flow-scroll' }, [svg])); + add(wrap, legend([['pl-sw-data', 'a value used to compute another'], ['pl-sw-control', 'a test that decides whether it changes'], ['pl-sw-acc', 'an accumulator: computed from its own old value']])); + if (!edges.length) add(wrap, el('p', { class: 'pl-muted', text: 'No value feeds another yet.' })); + function update(frame) { + if (!frame) return; + const reads = new Set((frame.last?.reads || []).map((r) => up(r.name))); + const writes = new Set((frame.last?.writes || []).map((w) => up(w.name))); + if (frame.last?.kind === 'return') writes.add('RETURN'); + if (frame.last?.kind === 'output') writes.add('OUTPUT'); + for (const [id, gEl] of nodeEls) { + gEl.classList.toggle('is-read', reads.has(id) && !writes.has(id)); + gEl.classList.toggle('is-written', writes.has(id)); + } + const line = frame.last?.line; + for (const { e, p } of edgeEls) p.classList.toggle('is-live', line != null && e.lines.includes(line) && writes.has(e.to)); + } + const node = panel(options.title ?? `How values flow in ${unit.kind === 'main' ? 'the main program' : unit.name}`, wrap, 'pl-flow-panel pl-span-2'); + return { node, update }; +} + +/** + * Where each value of one variable comes from. Every line that sets the + * variable gets a filled dot, every line that reads it a hollow one, and an + * arc joins each read to every assignment whose value can reach it. An arc + * that climbs back up the listing is a value carried round a loop. + * + * options: { unit, variable } + */ +export function defUseView(ctx, options = {}) { + const unit = unitFor(ctx, options.unit); + const body = el('div', { class: 'pl-defuse' }); + if (!unit) return { node: panel('Where values come from', body), update() { } }; + const vars = [...new Set(unit.defs.filter((d) => d.kind !== 'unset' || true).map((d) => d.var))]; + const nameOf = (k) => unit.defs.find((d) => d.var === k)?.name || k; + const select = el('select', { class: 'pl-select', 'aria-label': 'variable' }, vars.map((v) => el('option', { value: v, text: nameOf(v) }))); + let chosen = up(options.variable || unit.accumulators?.[0]?.name || vars.find((v) => !unit.params?.some((p) => up(p.name) === v)) || vars[0] || ''); + select.value = chosen; + const src = ctx.study.source.replace(/\r\n?/g, '\n').split('\n'); + const first = unit.node ? unit.node.line : 1; + const last = unit.node ? (unit.node.endLine || src.length) : src.length; + const listing = el('div', { class: 'pl-defuse-listing' }); + const lineH = 21; + function draw() { + clear(listing); + const defs = unit.defs.filter((d) => d.var === chosen && d.line != null); + const uses = unit.uses.filter((u) => u.var === chosen && u.line != null && u.kind !== 'spec'); + const setLines = new Set(defs.filter((d) => d.kind !== 'unset' && d.kind !== 'param').map((d) => d.line)); + const readLines = new Set(uses.map((u) => u.line)); + const unsetLines = new Set(defs.filter((d) => d.kind === 'unset').map((d) => d.line)); + const arcs = []; + for (const u of uses) for (const d of u.defs) if (d.line != null && d.kind !== 'unset' && d.kind !== 'param') arcs.push({ from: d.line, to: u.line, back: d.line >= u.line }); + const rowsN = last - first + 1; + const gutter = 60; + const svg = sv('svg', { class: 'pl-defuse-svg', width: gutter, height: rowsN * lineH, viewBox: `0 0 ${gutter} ${rowsN * lineH}` }); + const y = (l) => (l - first) * lineH + lineH / 2; + const seen = new Set(); + arcs.forEach((a, i) => { + const k = `${a.from}>${a.to}`; + if (seen.has(k)) return; + seen.add(k); + const depth = 8 + (Math.abs(a.to - a.from) % 5) * 7 + (a.back ? 14 : 0); + const x = gutter - 8; + const p = sv('path', { d: `M${x},${y(a.from)} C${x - depth * 2},${y(a.from)} ${x - depth * 2},${y(a.to)} ${x},${y(a.to)}`, class: `pl-defuse-arc${a.back ? ' is-back' : ''}`, 'marker-end': 'url(#plDuArrow)' }); + add(p, sv('title', { text: a.back ? `The value set on line ${a.from} is read again on line ${a.to}, on a later pass of the loop.` : `The value set on line ${a.from} can reach the read on line ${a.to}.` })); + svg.appendChild(p); + }); + add(svg, sv('defs', {}, [sv('marker', { id: 'plDuArrow', viewBox: '0 0 10 10', refX: '8', refY: '5', markerWidth: '6', markerHeight: '6', orient: 'auto' }, [sv('path', { d: 'M0,0 L10,5 L0,10 z', class: 'pl-flow-head' })])])); + const lines = el('div', { class: 'pl-defuse-lines' }); + for (let l = first; l <= last; l++) { + const mark = setLines.has(l) && readLines.has(l) ? '◑' : setLines.has(l) ? '●' : readLines.has(l) ? '○' : unsetLines.has(l) ? '◌' : ''; + const row = el('div', { class: `pl-defuse-row${setLines.has(l) ? ' is-set' : ''}${readLines.has(l) ? ' is-read' : ''}`, style: { height: `${lineH}px` } }, [ + el('span', { class: 'pl-defuse-mark', text: mark }), + el('span', { class: 'pl-code-num', text: String(l) }), + el('code', { class: 'pl-code-text', html: lineHtml(src[l - 1] || '') || ' ' }), + ]); + lines.appendChild(row); + } + add(listing, [svg, lines]); + } + select.addEventListener('change', () => { chosen = select.value; draw(); }); + draw(); + add(body, [el('div', { class: 'pl-row' }, [el('label', { class: 'pl-label', text: 'Follow the variable ' }), select]), listing, + legend([['pl-sw-set', 'sets it'], ['pl-sw-read', 'reads it'], ['pl-sw-back', 'an arc going up: a value carried to the next pass']])]); + return { node: panel(options.title ?? 'Where values come from', body, 'pl-defuse-panel'), update() { } }; +} + +/* ------------------------------------------------------------------ */ +/* The shape of the program */ +/* ------------------------------------------------------------------ */ + +/** + * A flowchart, laid out from the program's own structure: a box per + * statement, a diamond per decision, and a loop's arrow back to its test. + * The box for the line about to run is marked. + * + * options: { unit } + */ +export function cfgView(ctx, options = {}) { + const unit = unitFor(ctx, options.unit); + const body = el('div', { class: 'pl-cfg' }); + if (!unit) return { node: panel('Flowchart', body), update() { } }; + const BOX_H = 26; const GAP = 18; const CH = 6.6; const MAXC = 34; + const clip = (s) => (s.length > MAXC ? `${s.slice(0, MAXC - 1)}…` : s); + const textOf = (s) => { + switch (s.kind) { + case 'Assign': return `${sourceOf(s.target)} <- ${sourceOf(s.value)}`; + case 'Declare': return `DECLARE ${s.names.map((n) => n.name).join(', ')}`; + case 'Output': return `OUTPUT ${s.values.map(sourceOf).join(', ')}`; + case 'Input': return `INPUT ${sourceOf(s.target)}`; + case 'Return': return s.value ? `RETURN ${sourceOf(s.value)}` : 'RETURN'; + case 'Call': return `CALL ${s.name}(${s.args.map(sourceOf).join(', ')})`; + case 'BuiltinCall': return `${s.name}(${s.args.map(sourceOf).join(', ')})`; + case 'Break': return 'BREAK'; + case 'Continue': return 'CONTINUE'; + case 'Assert': return `ASSERT ${sourceOf(s.expr)}`; + default: return s.kind; + } + }; + const boxes = []; + // layout(stmts) returns { w, h, place(x, y) } where x is the center line + const layoutSeq = (stmts) => { + const parts = stmts.filter((s) => s.kind !== 'Declare' || options.declares).map(layoutStmt); + const w = Math.max(120, ...parts.map((p) => p.w)); + const h = parts.reduce((a, p) => a + p.h, 0) + Math.max(0, parts.length - 1) * GAP; + return { + w, h, + place(x, y, g) { + let cy = y; + parts.forEach((p, i) => { + p.place(x, cy, g); + if (i < parts.length - 1) add(g, sv('line', { x1: x, y1: cy + p.h, x2: x, y2: cy + p.h + GAP, class: 'pl-cfg-link', 'marker-end': 'url(#plCfgArrow)' })); + cy += p.h + GAP; + }); + }, + }; + }; + const rectFor = (text, line, cls = '') => { + const w = Math.min(MAXC, text.length) * CH + 20; + return { + w, h: BOX_H, + place(x, y, g) { + const r = sv('g', { class: `pl-cfg-box ${cls}`, 'data-line': line }); + add(r, sv('rect', { x: x - w / 2, y, width: w, height: BOX_H, rx: 5 })); + add(r, sv('text', { x, y: y + BOX_H / 2 + 4, 'text-anchor': 'middle', text: clip(text) })); + add(r, sv('title', { text: `line ${line}: ${text}` })); + g.appendChild(r); + boxes.push({ line, el: r }); + }, + }; + }; + const diamond = (text, line, cls = '') => { + const w = Math.min(MAXC, text.length) * CH + 44; const h = 38; + return { + w, h, + place(x, y, g) { + const r = sv('g', { class: `pl-cfg-box pl-cfg-diamond ${cls}`, 'data-line': line }); + add(r, sv('polygon', { points: `${x},${y} ${x + w / 2},${y + h / 2} ${x},${y + h} ${x - w / 2},${y + h / 2}` })); + add(r, sv('text', { x, y: y + h / 2 + 4, 'text-anchor': 'middle', text: clip(text) })); + add(r, sv('title', { text: `line ${line}: ${text}` })); + g.appendChild(r); + boxes.push({ line, el: r }); + }, + }; + }; + const label = (g, x, y, t, anchor = 'start') => add(g, sv('text', { x, y, class: 'pl-cfg-yes', 'text-anchor': anchor, text: t })); + const layoutStmt = (s) => { + switch (s.kind) { + case 'If': { + const d = diamond(`${sourceOf(s.branches[0].cond)}?`, s.line); + const yes = layoutSeq(s.branches[0].body); + const rest = s.branches.length > 1 ? layoutStmt({ ...s, branches: s.branches.slice(1), line: s.branches[1].line }) : s.elseBody ? layoutSeq(s.elseBody) : null; + const w = yes.w + (rest ? rest.w : 60) + 40; + const h = d.h + GAP + Math.max(yes.h, rest ? rest.h : 0) + GAP; + return { + w: Math.max(w, d.w), h, + place(x, y, g) { + d.place(x, y, g); + const lx = x - w / 2 + yes.w / 2; const rx = x + w / 2 - (rest ? rest.w / 2 : 30); + const by = y + d.h + GAP; + add(g, sv('path', { d: `M${x - d.w / 2},${y + d.h / 2} H${lx} V${by}`, class: 'pl-cfg-link', 'marker-end': 'url(#plCfgArrow)' })); + label(g, lx + 4, y + d.h / 2 - 5, 'yes'); + add(g, sv('path', { d: `M${x + d.w / 2},${y + d.h / 2} H${rx} V${rest ? by : y + h - GAP / 2}`, class: 'pl-cfg-link', 'marker-end': rest ? 'url(#plCfgArrow)' : null })); + label(g, rx - 4, y + d.h / 2 - 5, 'no', 'end'); + yes.place(lx, by, g); + if (rest) rest.place(rx, by, g); + const bottom = y + h - GAP / 2; + add(g, sv('path', { d: `M${lx},${by + yes.h} V${bottom} H${rx}${rest ? ` M${rx},${by + rest.h} V${bottom}` : ''}`, class: 'pl-cfg-link' })); + add(g, sv('line', { x1: x, y1: bottom, x2: x, y2: y + h, class: 'pl-cfg-link' })); + }, + }; + } + case 'Case': { + const d = diamond(`CASE OF ${sourceOf(s.subject)}`, s.line); + const arms = [...s.arms.map((a) => ({ lab: a.labels.map((l) => (l.value ? sourceOf(l.value) : `${sourceOf(l.low)} TO ${sourceOf(l.high)}`)).join(', '), seq: layoutSeq(a.body) })), ...(s.otherwise ? [{ lab: 'OTHERWISE', seq: layoutSeq(s.otherwise) }] : [])]; + const w = arms.reduce((a, x) => a + x.seq.w + 20, 0); + const h = d.h + GAP * 2 + Math.max(...arms.map((a) => a.seq.h)) + GAP; + return { + w: Math.max(w, d.w), h, + place(x, y, g) { + d.place(x, y, g); + let cx = x - w / 2; + const by = y + d.h + GAP * 2; + const bottom = y + h - GAP / 2; + for (const a of arms) { + const mx = cx + a.seq.w / 2 + 10; + add(g, sv('path', { d: `M${x},${y + d.h} V${y + d.h + GAP / 2} H${mx} V${by}`, class: 'pl-cfg-link', 'marker-end': 'url(#plCfgArrow)' })); + label(g, mx + 4, by - 4, a.lab); + a.seq.place(mx, by, g); + add(g, sv('path', { d: `M${mx},${by + a.seq.h} V${bottom} H${x}`, class: 'pl-cfg-link' })); + cx += a.seq.w + 20; + } + add(g, sv('line', { x1: x, y1: bottom, x2: x, y2: y + h, class: 'pl-cfg-link' })); + }, + }; + } + case 'For': case 'ForEach': case 'While': case 'Repeat': { + const head = s.kind === 'For' ? `${s.var} < ${sourceOf(s.stop)}?` : s.kind === 'ForEach' ? `another value of ${sourceOf(s.iterable)}?` : s.kind === 'While' ? `${sourceOf(s.cond)}?` : `UNTIL ${sourceOf(s.cond)}?`; + const init = s.kind === 'For' ? rectFor(`${s.var} <- ${sourceOf(s.start)}`, s.line, 'is-loop') : null; + const d = diamond(head, s.kind === 'Repeat' ? s.endLine : s.line, 'is-loop'); + const b = layoutSeq(s.body); + const nx = s.kind === 'For' ? rectFor(`${s.var} <- ${s.var} + ${s.step ? sourceOf(s.step) : 1}`, s.endLine, 'is-loop') : null; + const inner = b.h + (nx ? nx.h + GAP : 0); + const w = Math.max(d.w, b.w, init?.w || 0) + 70; + const h = (init ? init.h + GAP : 0) + d.h + GAP + inner + GAP; + return { + w, h, + place(x, y, g) { + let cy = y; + if (init) { init.place(x, cy, g); add(g, sv('line', { x1: x, y1: cy + init.h, x2: x, y2: cy + init.h + GAP, class: 'pl-cfg-link', 'marker-end': 'url(#plCfgArrow)' })); cy += init.h + GAP; } + if (s.kind === 'Repeat') { + b.place(x, cy, g); + const dy = cy + b.h + GAP; + add(g, sv('line', { x1: x, y1: cy + b.h, x2: x, y2: dy, class: 'pl-cfg-link', 'marker-end': 'url(#plCfgArrow)' })); + d.place(x, dy, g); + const lx = x - w / 2 + 10; + add(g, sv('path', { d: `M${x - d.w / 2},${dy + d.h / 2} H${lx} V${cy - 6} H${x - 4}`, class: 'pl-cfg-link is-back', 'marker-end': 'url(#plCfgArrow)' })); + label(g, lx + 4, dy + d.h / 2 - 5, 'no'); + add(g, sv('line', { x1: x, y1: dy + d.h, x2: x, y2: y + h, class: 'pl-cfg-link' })); + label(g, x + 5, dy + d.h + 12, 'yes'); + return; + } + d.place(x, cy, g); + const by = cy + d.h + GAP; + add(g, sv('line', { x1: x, y1: cy + d.h, x2: x, y2: by, class: 'pl-cfg-link', 'marker-end': 'url(#plCfgArrow)' })); + label(g, x + 5, cy + d.h + 12, 'yes'); + b.place(x, by, g); + let end = by + b.h; + if (nx) { add(g, sv('line', { x1: x, y1: end, x2: x, y2: end + GAP, class: 'pl-cfg-link', 'marker-end': 'url(#plCfgArrow)' })); nx.place(x, end + GAP, g); end += GAP + nx.h; } + const lx = x - w / 2 + 10; + add(g, sv('path', { d: `M${x},${end} V${end + GAP / 2} H${lx} V${cy + d.h / 2} H${x - d.w / 2 - 2}`, class: 'pl-cfg-link is-back', 'marker-end': 'url(#plCfgArrow)' })); + const rx = x + w / 2 - 10; + add(g, sv('path', { d: `M${x + d.w / 2},${cy + d.h / 2} H${rx} V${y + h} H${x}`, class: 'pl-cfg-link' })); + label(g, x + d.w / 2 + 4, cy + d.h / 2 - 5, 'no'); + }, + }; + } + default: return rectFor(textOf(s), s.line, s.kind === 'Return' ? 'is-return' : s.kind === 'Break' ? 'is-break' : ''); + } + }; + const top = layoutSeq(unit.body); + const W = top.w + 40; const H = top.h + 60; + const svg = sv('svg', { viewBox: `0 0 ${W} ${H}`, class: 'pl-cfg-svg', role: 'img', 'aria-label': `Flowchart of ${unit.name}` }); + add(svg, sv('defs', {}, [sv('marker', { id: 'plCfgArrow', viewBox: '0 0 10 10', refX: '9', refY: '5', markerWidth: '6', markerHeight: '6', orient: 'auto' }, [sv('path', { d: 'M0,0 L10,5 L0,10 z', class: 'pl-flow-head' })])])); + const gEl = sv('g'); + add(gEl, sv('text', { x: W / 2, y: 16, 'text-anchor': 'middle', class: 'pl-cfg-title', text: unit.kind === 'main' ? 'start' : `${unit.name}(${(unit.params || []).map((p) => p.name).join(', ')})` })); + top.place(W / 2, 28, gEl); + add(svg, gEl); + add(body, el('div', { class: 'pl-cfg-scroll' }, [svg])); + function update(frame) { + if (!frame) return; + const l = frame.next?.line ?? frame.last?.line; + for (const b of boxes) b.el.classList.toggle('is-current', b.line === l); + } + return { node: panel(options.title ?? 'Flowchart', body, 'pl-cfg-panel'), update }; +} + +/* ------------------------------------------------------------------ */ +/* The four jobs */ +/* ------------------------------------------------------------------ */ + +const JOB_TITLE = { setup: 'Set up', visit: 'Visit', test: 'Test', update: 'Update', report: 'Report' }; + +/** + * The four jobs of a scan, found in this program: which lines do each one, + * which are missing, and what the coach found wrong with each. + * + * options: { unit, expect: 'count' } expect: the pattern an exercise asks for + */ +export function jobsView(ctx, options = {}) { + const unit = unitFor(ctx, options.unit); + const body = el('div', { class: 'pl-jobs' }); + if (!unit) return { node: panel('The four jobs', body), update() { } }; + const report = jobsReport(unit, ctx.study.diags); + const pattern = unit.pattern ? PATTERN_NAMES[unit.pattern] : null; + add(body, el('p', { class: 'pl-jobs-pattern' }, pattern + ? ['This reads as the ', el('strong', { text: pattern }), ` pattern: ${patternSentence(unit)}`] + : ['No single scan pattern stands out here.'])); + if (options.expect && unit.pattern && options.expect !== unit.pattern) { + add(body, el('p', { class: 'pl-warn-line', text: `The task asks for ${PATTERN_NAMES[options.expect] || options.expect}, and this program reads as ${pattern}.` })); + } + const cards = el('div', { class: 'pl-job-cards' }); + const cardEls = []; + for (const j of report) { + const card = el('div', { class: `pl-job role-${j.job} is-${j.status}` }, [ + el('div', { class: 'pl-job-head' }, [ + el('span', { class: 'pl-job-status', text: j.status === 'ok' ? '✓' : j.status === 'missing' ? '–' : '!', 'aria-hidden': 'true' }), + el('strong', { text: JOB_TITLE[j.job] }), + el('span', { class: 'pl-muted', text: j.lines.length ? ` lines ${compactLines(j.lines)}` : '' }), + ]), + el('p', { text: j.status === 'ok' ? j.message.replace(/^[^:]+: /, '') : j.message }), + ...j.issues.slice(0, 3).map((d) => el('p', { class: `pl-job-issue is-${d.severity}`, text: `Line ${d.line}: ${d.message}` })), + ]); + cards.appendChild(card); + cardEls.push({ j, card }); + } + add(body, cards); + function update(frame) { + if (!frame) return; + const l = frame.next?.line; + const role = unit.roles.get(l)?.role; + for (const { j, card } of cardEls) card.classList.toggle('is-current', j.job === role); + } + return { node: panel(options.title ?? 'The four jobs', body, 'pl-jobs-panel'), update }; +} + +function patternSentence(unit) { + const acc = unit.accumulators.find((a) => a.how !== 'flag'); + const words = { + count: acc ? `${acc.name} starts at 0 and grows by one each time the test passes.` : '', + sum: acc ? `${acc.name} adds up a value at every position.` : '', + 'find-all': acc ? `every position that passes is appended to ${acc.name}.` : '', + filter: acc ? `the values that pass are appended to ${acc.name}.` : '', + tally: acc ? `${acc.name} counts how often each value occurs.` : '', + 'find-best': acc ? `${acc.name} holds the best seen so far.` : '', + 'find-first': 'the loop stops at the first position that passes.', + check: 'a flag records whether the test ever passed, or ever failed.', + build: acc ? `${acc.name} grows by one piece per position.` : '', + }; + return words[unit.pattern] || ''; +} + +function compactLines(ls) { + const out = []; let i = 0; + while (i < ls.length) { let j = i; while (j + 1 < ls.length && ls[j + 1] === ls[j] + 1) j++; out.push(j > i ? `${ls[i]}–${ls[j]}` : String(ls[i])); i = j + 1; } + return out.join(', '); +} + +/* ------------------------------------------------------------------ */ +/* Proof */ +/* ------------------------------------------------------------------ */ + +const verifyCache = new WeakMap(); +export function verificationFor(ctx) { + if (!verifyCache.has(ctx.study)) verifyCache.set(ctx.study, verify(ctx.program)); + return verifyCache.get(ctx.study); +} + +/** + * What the checker can prove, place by place: every position read exists, + * every division is safe, every invariant holds on entry and survives a + * pass, every loop ends, every ENSURE holds. Each place is proved, broken + * by a run (with the input that breaks it), or open: not proved, and not + * broken by any input tried. + * + * options: { runs: 300, unit, open: false } + */ +export function proofView(ctx, options = {}) { + const opts = { runs: 300, ...options }; + const body = el('div', { class: 'pl-proof' }); + const v = verificationFor(ctx); + // runs that break something, keyed by line + const broken = new Map(); + if (opts.runs) { + for (const sub of ctx.program.subs.values()) { + const r = checkProperties(ctx.program, sub.name, { count: opts.runs, maxFailures: 6 }); + for (const f of r.failures) { + const kinds = { index: ['index', 'slice'], divide: ['divide'], key: ['key'], invariant: [f.phase === 'init' ? 'invariant-init' : 'invariant-keep'], ensure: ['ensure'], decreases: f.phase ? [f.phase === 'bound' ? 'decreases-bound' : 'decreases-step'] : ['decreases-bound', 'decreases-step'], assert: ['assert'], require: ['require'], value: ['argument'], limit: ['terminates'] }[f.kind] || []; + for (const k of kinds) broken.set(`${k}@${f.line}`, f); + } + } + } + let proved = 0; let open = 0; let brokenN = 0; + const units = opts.unit ? v.units.filter((u) => up(u.name) === up(opts.unit)) : v.units; + for (const u of units) { + if (!u.obligations.length && !u.inferred.length) continue; + const sec = el('div', { class: 'pl-proof-unit' }); + add(sec, el('h5', { text: u.kind === 'main' ? 'Main program' : `${u.kind === 'Function' ? 'FUNCTION' : 'PROCEDURE'} ${u.name}` })); + const sub = ctx.program.subs.get(up(u.name)); + if (sub?.requires?.length) add(sec, el('p', { class: 'pl-muted', text: `Assumed, because the caller promises it: ${sub.requires.map((r) => sourceOf(r.expr)).join('; ')}.` })); + if (u.inferred.length) { + add(sec, el('div', { class: 'pl-proof-inferred' }, [el('span', { text: 'Invariants the checker found for itself: ' }), ...u.inferred.map((x) => el('span', { class: 'pl-chip', title: `the loop on line ${x.line}`, text: `line ${x.line}: ${x.facts.join(', ')}` }))])); + } + for (const o of u.obligations) { + const f = broken.get(`${o.kind}@${o.line}`) || (o.target ? broken.get(`${o.kind}@${o.target}`) : null); + const status = o.status === 'proved' ? 'proved' : f ? 'broken' : 'open'; + if (status === 'proved') proved++; else if (status === 'broken') brokenN++; else open++; + const details = el('div', { class: 'pl-proof-details', hidden: true }); + if (o.shown) { + if (o.shown.facts.length) add(details, [el('p', { class: 'pl-muted', text: 'What is known at this point:' }), el('ul', { class: 'pl-proof-facts' }, o.shown.facts.map((x) => el('li', {}, [el('code', { html: lineHtml(x) })])))]); + add(details, [el('p', { class: 'pl-muted', text: status === 'proved' ? 'What follows from it:' : 'What has to follow from it:' }), el('p', {}, [el('code', { class: 'pl-proof-goal', html: lineHtml(o.shown.goal) })])]); + if (o.shown.trail?.length) add(details, el('p', { class: 'pl-muted', text: `Reached by: ${o.shown.trail.join('; ')}.` })); + } + if (o.recursive === 'self') add(details, el('p', { text: 'Nothing shows these calls stop calling themselves. Add a DECREASES line directly under the FUNCTION or PROCEDURE line: a whole number, worked out from the parameters, that is at least 0 on every call and smaller on every call it makes to itself. Until then, every other claim here holds only for calls that return.' })); + else if (o.recursive === 'mutual') add(details, el('p', { text: 'These subroutines call each other in a cycle, and the checker follows a DECREASES measure only through a subroutine that calls itself directly. Every other claim here holds only for calls that return.' })); + else if (o.needsMeasure) add(details, el('p', { text: 'Nothing shows this loop ends. Add a DECREASES line at the top of its body: a whole number that is at least 0 while the loop runs and gets smaller on every pass.' })); + if (o.measure) add(details, el('p', { text: `Every pass makes ${o.measure} smaller, and it cannot go below 0 while the loop continues.` })); + if (status === 'broken') add(details, el('p', { class: 'pl-error-inline', text: `${f.call}: ${f.message}` })); + if (status === 'open') add(details, el('p', { class: 'pl-muted', text: `Not proved: the checker could not rule out every case, and none of the ${opts.runs} generated inputs broke it either. Either a fact is missing (an INVARIANT or REQUIRE would supply it), or the reasoning needs more than the checker knows, such as multiplication of two unknowns.` })); + const row = el('div', { class: `pl-ob is-${status}` }, [ + el('button', { type: 'button', class: 'pl-ob-head', 'aria-expanded': 'false' }, [ + el('span', { class: 'pl-ob-icon', text: status === 'proved' ? '✓' : status === 'broken' ? '✖' : '?', 'aria-hidden': 'true' }), + el('span', { class: 'pl-ob-kind', text: OBLIGATION_WORDS[o.kind] || o.kind }), + el('span', { class: 'pl-ob-label', text: `line ${o.line}: ${o.label}` }), + el('span', { class: 'pl-ob-status', text: status === 'proved' ? 'proved' : status === 'broken' ? 'broken by a run' : 'open' }), + ]), + details, + ]); + const head = row.firstChild; + head.addEventListener('click', () => { details.hidden = !details.hidden; head.setAttribute('aria-expanded', String(!details.hidden)); if (ctx.widget && o.line) ctx.widget.gotoLine(o.line); options.onSelect?.(o.line); }); + if (opts.open && status !== 'proved') { details.hidden = false; head.setAttribute('aria-expanded', 'true'); } + sec.appendChild(row); + } + for (const n of u.notes) add(sec, el('p', { class: 'pl-muted', text: n.text })); + if (u.partial) add(sec, el('p', { class: 'pl-warn-line', text: 'The checker looked at only part of this subroutine: it has more paths than the checker follows. Every run still checks it.' })); + body.appendChild(sec); + } + const total = proved + open + brokenN; + const summary = total ? `${proved} of ${total} proved${brokenN ? `, ${brokenN} broken by a run` : ''}${open ? `, ${open} open` : ''}` : 'nothing to prove'; + body.prepend(el('p', { class: 'pl-proof-summary' }, [el('strong', { text: summary }), total ? '. Select a line to see what is known there and what has to follow.' : '. Add an ENSURE, an INVARIANT, or an index into a sequence, and there will be.'])); + return { node: panel(options.title ?? 'What can be proved', body, 'pl-proof-panel'), update() { }, counts: { proved, open, broken: brokenN } }; +} + +/* ------------------------------------------------------------------ */ +/* Tests */ +/* ------------------------------------------------------------------ */ + +/** + * The spec's test lines, run: each call, what it should give, what it gave. + * A test written with ? in place of an answer is a prediction to check. + * + * options: { tests: 'COUNT(\'A\', "GATTACA") -> 3', onRun(test) } + */ +export function testsView(ctx, options = {}) { + const text = options.tests ?? ctx.spec.tests ?? ''; + const body = el('div', { class: 'pl-tests' }); + const tests = parseTests(String(text).replace(/->\s*\?\s*$/gm, '-> ERROR open')); + const results = runTests(ctx.program, tests, { record: true }); + if (!results.length) add(body, el('p', { class: 'pl-muted', text: 'No tests yet. A test is a call, an arrow, and the answer: COUNT(\'A\', "GATTACA") -> 3.' })); + let pass = 0; + const table = el('table', { class: 'pl-test-table' }); + add(table, el('thead', {}, [el('tr', {}, ['', 'call', 'should give', 'gave', ''].map((h) => el('th', { text: h })))])); + const tbody = el('tbody'); + for (const r of results) { + const open = r.expectError === 'open'; + const status = open ? 'open' : r.status; + if (status === 'pass') pass++; + const sub = r.call ? ctx.program.subs.get(up(r.call.name)) : null; + const gave = r.error ? `stopped: ${r.error.message}` : r.kind === 'io' ? (r.got || []).join(' | ') : r.got !== undefined ? formatValue(r.got, sub?.returns) : ''; + const should = open ? '? (predict it)' : r.expectError ? 'an error' : r.kind === 'io' ? r.output.join(' | ') : r.expected !== undefined ? formatValue(r.expected, sub?.returns) : ''; + const tr = el('tr', { class: `is-${status}` }, [ + el('td', { class: 'pl-test-icon', text: status === 'pass' ? '✓' : status === 'open' ? '?' : status === 'invalid' ? '!' : '✖', title: status }), + el('td', {}, [el('code', { text: r.kind === 'io' ? `INPUT ${r.input.map((x) => JSON.stringify(x)).join(', ')}` : r.source.split('->')[0].trim() })]), + el('td', {}, [el('code', { text: should })]), + el('td', { class: status === 'fail' || status === 'error' ? 'pl-bad' : '' }, [open && !r.error ? el('code', { text: formatValue(r.got, sub?.returns) }) : status === 'invalid' ? el('span', { text: r.message }) : el('code', { text: gave })]), + el('td', {}, [options.onRun && r.call ? button('Step through', () => options.onRun(r)) : null]), + ]); + tbody.appendChild(tr); + } + add(table, tbody); + if (results.length) add(body, [el('p', { class: 'pl-tests-summary' }, [el('strong', { text: `${pass} of ${results.filter((r) => r.expectError !== 'open').length} pass` }), results.some((r) => r.expectError === 'open') ? '. Rows marked ? were left for you to predict; compare your prediction with what it gave.' : '.']), table]); + return { node: panel(options.title ?? 'Tests', body, 'pl-tests-panel pl-span-2'), update() { }, results }; +} + +/** + * Hundreds of generated inputs: every DNA string up to length 4, then random + * ones. Each run checks every REQUIRE, ENSURE, INVARIANT, and ASSERT, and, + * given a reference program, compares answers. A failure is shrunk to the + * smallest input that still fails the same way. + * + * options: { name, reference (a program or a function), count: 400 } + */ +export function checksView(ctx, options = {}) { + const body = el('div', { class: 'pl-checks' }); + const names = options.name ? [options.name] : [...ctx.program.subs.values()].filter((s) => s.kind === 'Function' || s.ensures.length).map((s) => s.name); + if (!names.length) add(body, el('p', { class: 'pl-muted', text: 'Generated checks call a FUNCTION with made-up inputs; this program has none.' })); + const results = []; + for (const n of names) { + const r = checkProperties(ctx.program, n, { count: options.count ?? 400, reference: options.reference, generators: options.generators }); + results.push(r); + const sub = ctx.program.subs.get(up(n)); + const what = [sub.ensures.length && 'its ENSURE', options.reference && 'the model answer', 'every contract and position'].filter(Boolean).join(', '); + if (r.problem) { add(body, el('p', { class: 'pl-muted', text: r.problem })); continue; } + add(body, el('div', { class: `pl-check ${r.failures.length ? 'is-fail' : 'is-pass'}` }, [ + el('p', {}, [el('strong', { text: `${sub.name}: ` }), r.failures.length ? `${plural(r.failures.length, 'problem')} found in ${commas(r.checked)} runs.` : `${commas(r.checked)} generated inputs, all consistent with ${what}.`, r.skipped ? ` ${commas(r.skipped)} inputs broke a REQUIRE and were skipped.` : '']), + ...r.failures.map((f) => el('div', { class: 'pl-check-fail' }, [ + el('code', { text: f.call }), + el('p', { text: f.kind === 'mismatch' ? `${f.call.split('(')[0]} ${f.message}.` : f.message }), + f.original !== f.call ? el('p', { class: 'pl-muted', text: `The first failing input was ${f.original}; this is the smallest version of it that still fails.` }) : null, + options.onRun ? button('Step through this input', () => options.onRun(f)) : null, + ])), + ])); + } + return { node: panel(options.title ?? 'Generated checks', body, 'pl-checks-panel'), update() { }, results }; +} + +/* ------------------------------------------------------------------ */ +/* How the work grows */ +/* ------------------------------------------------------------------ */ + +/** + * Run a subroutine at growing sizes and plot its work against the size. + * The dashed curve is the growth model that fits best: constant, log n, n, + * n log n, n squared, n cubed, or doubling. + * + * options: { name, sizes, args(n, random), measure: 'steps' | 'letterCompares', xLabel } + */ +export function growthView(ctx, options = {}) { + const name = options.name || ctx.call?.name || [...ctx.program.subs.keys()][0]; + const body = el('div', { class: 'pl-growth' }); + const measureSel = el('select', { class: 'pl-select', 'aria-label': 'what to count' }, [ + el('option', { value: 'steps', text: 'steps' }), el('option', { value: 'letterCompares', text: 'letter comparisons' }), el('option', { value: 'compares', text: 'all comparisons' }), el('option', { value: 'reads', text: 'values read' }), + ]); + measureSel.value = options.measure || 'steps'; + const chart = el('div', { class: 'pl-growth-chart' }); + const table = el('details', { class: 'pl-growth-table' }); + const fitLine = el('p', { class: 'pl-growth-fit' }); + function draw() { + clear(chart); clear(table); + const g = measureGrowth(ctx.program, name, { sizes: options.sizes, args: options.args, measure: measureSel.value }); + const pts = g.points.filter((p) => p.work != null); + if (g.problem || !pts.length) { add(chart, el('p', { class: 'pl-muted', text: g.problem || (g.points[0]?.error ? `The run failed: ${g.points[0].error}` : 'Nothing to measure.') })); fitLine.textContent = ''; return; } + const W = 520; const H = 250; const L = 64; const R = 110; const T = 14; const B = 40; + const xmax = Math.max(...pts.map((p) => p.n)); const ymax = Math.max(...pts.map((p) => p.work)) * 1.08 || 1; + const X = (n) => L + (n / xmax) * (W - L - R); const Y = (w) => H - B - (w / ymax) * (H - T - B); + const svg = sv('svg', { viewBox: `0 0 ${W} ${H}`, class: 'pl-growth-svg', role: 'img', 'aria-label': `Work against input size for ${name}` }); + for (let i = 0; i <= 4; i++) { + const v = (ymax * i) / 4; + add(svg, [sv('line', { x1: L, x2: W - R, y1: Y(v), y2: Y(v), class: 'pl-grid-line' }), sv('text', { x: L - 8, y: Y(v) + 4, 'text-anchor': 'end', class: 'pl-axis-text', text: shortNum(v) })]); + } + for (const p of pts) add(svg, sv('text', { x: X(p.n), y: H - B + 16, 'text-anchor': 'middle', class: 'pl-axis-text', text: String(p.n) })); + add(svg, sv('text', { x: (L + W - R) / 2, y: H - 6, 'text-anchor': 'middle', class: 'pl-axis-label', text: options.xLabel || 'input size n' })); + if (g.fit) { + const model = { '1': () => 1, 'log n': (n) => Math.log2(Math.max(2, n)), n: (n) => n, 'n log n': (n) => n * Math.log2(Math.max(2, n)), 'n^2': (n) => n * n, 'n^3': (n) => n ** 3, '2^n': (n) => 2 ** Math.min(n, 60) }[g.fit.id]; + const xs = Array.from({ length: 60 }, (_, i) => pts[0].n + ((xmax - pts[0].n) * i) / 59); + add(svg, sv('path', { d: xs.map((n, i) => `${i ? 'L' : 'M'}${X(n).toFixed(1)},${Y(g.fit.constant * model(n)).toFixed(1)}`).join(' '), class: 'pl-growth-model' })); + add(svg, sv('text', { x: X(xmax) + 8, y: Y(g.fit.constant * model(xmax)) + 4, class: 'pl-growth-label', text: modelWords(g.fit) })); + } + add(svg, sv('path', { d: pts.map((p, i) => `${i ? 'L' : 'M'}${X(p.n)},${Y(p.work)}`).join(' '), class: 'pl-growth-line' })); + for (const p of pts) { + const c = sv('circle', { cx: X(p.n), cy: Y(p.work), r: 4.5, class: 'pl-growth-dot', tabindex: '0' }); + add(c, sv('title', { text: `n = ${p.n}: ${commas(Math.round(p.work))} ${measureWord(measureSel.value)}` })); + svg.appendChild(c); + const hit = sv('circle', { cx: X(p.n), cy: Y(p.work), r: 12, class: 'pl-hit' }); + add(hit, sv('title', { text: `n = ${p.n}: ${commas(Math.round(p.work))} ${measureWord(measureSel.value)}` })); + svg.appendChild(hit); + } + add(chart, svg); + add(chart, legend([['pl-sw-measured', `${measureWord(measureSel.value)}, measured`], ['pl-sw-model', 'the model that fits best']])); + add(table, el('summary', { text: 'The numbers' })); + const t = el('table', { class: 'pl-map' }); + add(t, el('tr', {}, [el('th', { text: 'n' }), el('th', { text: measureWord(measureSel.value) }), el('th', { text: 'per n' })])); + for (const p of pts) add(t, el('tr', {}, [el('td', { text: String(p.n) }), el('td', { text: commas(Math.round(p.work)) }), el('td', { text: (p.work / p.n).toFixed(2) })])); + add(table, t); + fitLine.textContent = g.fit ? `The work ${g.fit.label}: about ${fmtConst(g.fit.constant)} × ${g.fit.id === '1' ? '1' : g.fit.id} ${measureWord(measureSel.value)}. Double the input and the work ${doubling(g.fit.id)}.` : ''; + } + measureSel.addEventListener('change', draw); + add(body, [el('div', { class: 'pl-row' }, [el('label', { class: 'pl-label', text: `Counting, for ${name}: ` }), measureSel]), chart, fitLine, table]); + draw(); + return { node: panel(options.title ?? 'How the work grows', body, 'pl-growth-panel'), update() { } }; +} +const measureWord = (m) => ({ steps: 'steps', letterCompares: 'letter comparisons', compares: 'comparisons', reads: 'values read' }[m] || m); +const shortNum = (v) => (v >= 1e6 ? `${(v / 1e6).toFixed(1)}M` : v >= 1e4 ? `${Math.round(v / 1e3)}k` : String(Math.round(v))); +const fmtConst = (c) => (c >= 10 ? String(Math.round(c)) : c.toFixed(c >= 1 ? 1 : 2)); +const doubling = (id) => ({ '1': 'stays the same', 'log n': 'grows by a fixed amount', n: 'doubles', 'n log n': 'a little more than doubles', 'n^2': 'quadruples', 'n^3': 'grows eightfold', '2^n': 'squares' }[id]); +function modelWords(f) { return f.id === '1' ? 'constant' : f.id.replace('^2', '²').replace('^3', '³').replace('2^n', '2ⁿ'); } + +/* ------------------------------------------------------------------ */ +/* Translation */ +/* ------------------------------------------------------------------ */ + +/** + * The same program in Mojo (or Python), side by side, with the line about + * to run marked on both sides. + * + * options: { target: 'mojo' | 'python', switcher: true } + */ +export function translateView(ctx, options = {}) { + const body = el('div', { class: 'pl-translate' }); + let target = options.target || 'mojo'; + const pick = el('select', { class: 'pl-select', 'aria-label': 'language' }, [el('option', { value: 'mojo', text: 'Mojo' }), el('option', { value: 'python', text: 'Python' })]); + pick.value = target; + const left = el('div', { class: 'pl-tr-side' }); + const right = el('div', { class: 'pl-tr-side' }); + const notes = el('ul', { class: 'pl-tr-notes' }); + let rightRows = []; + const src = ctx.study.source.replace(/\r\n?/g, '\n').split('\n'); + const leftRows = src.map((text, i) => el('div', { class: 'pl-tr-row', 'data-line': String(i + 1) }, [el('span', { class: 'pl-code-num', text: String(i + 1) }), el('code', { html: lineHtml(text) || ' ' })])); + add(left, leftRows); + function draw() { + clear(right); clear(notes); + const r = translate(ctx.program, target); + rightRows = r.lines.map((l) => { + const row = el('div', { class: 'pl-tr-row', 'data-from': l.from ?? '' }, [el('code', { text: l.text || ' ' })]); + right.appendChild(row); + return { row, from: l.from }; + }); + for (const n of r.notes) notes.appendChild(el('li', { text: n })); + } + pick.addEventListener('change', () => { target = pick.value; draw(); update(lastFrame); }); + draw(); + let lastFrame = null; + function update(frame) { + lastFrame = frame; + const l = frame?.next?.line ?? frame?.last?.line; + leftRows.forEach((r, i) => r.classList.toggle('is-current', i + 1 === l)); + for (const r of rightRows) r.row.classList.toggle('is-current', r.from === l); + const cur = rightRows.find((r) => r.from === l); + if (cur && right.scrollHeight > right.clientHeight) { const top = cur.row.offsetTop - right.offsetTop; if (top < right.scrollTop || top > right.scrollTop + right.clientHeight - 20) right.scrollTop = Math.max(0, top - right.clientHeight / 3); } + } + add(body, [ + options.switcher !== false ? el('div', { class: 'pl-row' }, [el('label', { class: 'pl-label', text: 'Translate into ' }), pick]) : null, + el('div', { class: 'pl-tr-grid' }, [left, right]), + notes, + el('p', { class: 'pl-muted', text: 'The translation is literal, line for line, so the two can be read side by side. It is a starting point for code, not a replacement for reading the course\'s Mojo reference.' }), + ]); + return { node: panel(options.title ?? 'The same program in code', body, 'pl-translate-panel pl-span-2'), update }; +} + +export const ANALYSIS_VIEWS = { + diagnostics: diagnosticsView, + flow: flowView, + defuse: defUseView, + cfg: cfgView, + jobs: jobsView, + proof: proofView, + tests: testsView, + checks: checksView, + growth: growthView, + translate: translateView, +}; + +export { walkStatements, childBlocks, esc }; diff --git a/src/engines/pseudo/views.js b/src/engines/pseudo/views.js new file mode 100644 index 0000000..a3f0d21 --- /dev/null +++ b/src/engines/pseudo/views.js @@ -0,0 +1,653 @@ +/** + * pseudo/views.js + * + * Views over one run of a pseudocode program, and the composer that puts + * them together into a widget. Every view is a factory taking (ctx, options) + * and returning { node, update(frame) }; ctx = { study, trace, spec, program, + * analysis, widget }. No view knows about a particular program: each draws + * whatever the trace recorded. + * + * widget(spec) -> node, for an {ojs} cell + * mountWidget(target, spec) -> controller { node, trace, goto, step, play, pause, ... } + * + * spec = { + * code: '...pseudocode...', + * call: 'COUNT(\'A\', "GATTACA")', a subroutine to call, with arguments + * input: ['7', 'GATTACA'], what INPUT reads, for a main program + * views: ['code', 'variables', 'trace', 'explain'], + * layout: 'two-column' | 'one-column', + * title, blurb, start, speed, maxSteps, contracts, seed, + * viewOptions: { code: {...}, trace: {...} } + * } + */ + +import { el, sv, add, clear, panel, plural, commas, legend, guideLink } from './dom.js'; +import { lineHtml, esc } from './highlight.js'; +import { study } from './study.js'; +import { run, evaluateConstant, valueFits } from './interp.js'; +import { parseExpression, sourceOf } from './parse.js'; +import { formatValue, UNSET, PArray, PRecord, chars, typeName } from './lang.js'; +import { ANALYSIS_VIEWS } from './views-analysis.js'; +import { walkStatements, statementExprs, walkExpr } from './analyze.js'; + +const up = (s) => String(s).toUpperCase(); + +/* ------------------------------------------------------------------ */ +/* Sessions */ +/* ------------------------------------------------------------------ */ + +/** Turn "COUNT('A', \"GATTACA\")" into a name and argument values for a program. */ +export function resolveCall(program, call) { + if (!call) return null; + if (typeof call === 'object') return call; + const p = parseExpression(call); + if (p.diags.some((d) => d.severity === 'error') || p.expr.kind !== 'CallExpr') throw new Error(`"${call}" is not a call. Write the subroutine's name and its arguments, as in COUNT('A', "GATTACA").`); + const sub = program.subs.get(up(p.expr.name)); + if (!sub) throw new Error(`The program has no FUNCTION or PROCEDURE called ${p.expr.name}.`); + if (p.expr.args.length !== sub.params.length) throw new Error(`${sub.name} takes ${plural(sub.params.length, 'value')}, and the call gives ${p.expr.args.length}.`); + const args = p.expr.args.map((a, i) => { + let v = evaluateConstant(program, a); + const t = sub.params[i].type; + if (t.k === 'ARRAY' && Array.isArray(v)) { const arr = new PArray([v.length]); arr.data = v; v = arr; } + if (t.k === 'SET' && Array.isArray(v)) v = new Set(v); + if (!valueFits(t, v)) throw new Error(`Parameter ${sub.params[i].name} of ${sub.name} is ${typeName(t)}, and the call gives ${sourceOf(a)}.`); + return v; + }); + return { name: sub.name, args, text: call }; +} + +/** The first subroutine, and a plausible call to it, for a program with no main body. */ +export function defaultCall(program) { + const sub = [...program.subs.values()].find((s) => s.kind === 'Function') || [...program.subs.values()][0]; + if (!sub) return null; + const sample = (t) => { + switch (t?.k) { + case 'STRING': return '"GATTACA"'; + case 'CHAR': return "'A'"; + case 'INTEGER': return '3'; + case 'FLOAT': return '0.5'; + case 'BOOL': return 'TRUE'; + case 'LIST': return t.of?.k === 'STRING' ? '["GATTACA", "GGC"]' : t.of?.k === 'FLOAT' ? '[0.5, 1.5]' : '[12, 30, 25, 8]'; + default: return '[]'; + } + }; + return `${sub.name}(${sub.params.map((p) => sample(p.type)).join(', ')})`; +} + +export function session(spec) { + const s = spec.study || study(spec.code, { coach: spec.coach !== false }); + const ctx = { study: s, program: s.program, analysis: s.analysis, spec, trace: null, error: null }; + if (!s.ok) return ctx; + let call = null; + try { + const text = spec.call ?? (s.program.main.body.length ? null : defaultCall(s.program)); + call = text ? resolveCall(s.program, text) : null; + } catch (err) { ctx.error = err; return ctx; } + ctx.call = call; + ctx.trace = run(s.program, { + call, input: spec.input || [], maxSteps: spec.maxSteps ?? 20000, contracts: spec.contracts !== false, seed: spec.seed ?? 1, shuffle: spec.shuffle, + }); + return ctx; +} + +/** A frame is everything a view needs to draw one moment. */ +export function frameAt(trace, t) { + const c = Math.max(0, Math.min(t, trace.count)); + return { t: c, trace, state: trace.states[c], last: c > 0 ? trace.steps[c - 1] : null, next: c < trace.count ? trace.steps[c] : null }; +} + +/* ------------------------------------------------------------------ */ +/* 1. Code */ +/* ------------------------------------------------------------------ */ + +const ROLE_LABEL = { setup: 'Set up', visit: 'Visit', test: 'Test', update: 'Update', report: 'Report', spec: 'Specification', helper: '' }; + +/** + * The program, with a marker on the line about to run, a tint on the one + * that just ran, how many times each line has run, and a mark beside every + * line the checker or the coach has something to say about. + * + * options: { counts: true, heat: true, roles: false, diags: true, maxHeight, title } + */ +export function codeView(ctx, options = {}) { + const opts = { counts: true, heat: true, roles: false, diags: true, ...options }; + const src = ctx.study.source.replace(/\r\n?/g, '\n').split('\n'); + while (src.length > 1 && !src[src.length - 1].trim()) src.pop(); + const body = el('div', { class: 'pl-code' }); + const byLine = new Map(); + for (const d of ctx.study.diags) { + if (!opts.diags) break; + if (!byLine.has(d.line)) byLine.set(d.line, []); + byLine.get(d.line).push(d); + } + const roles = new Map(); + if (opts.roles && ctx.analysis) for (const u of ctx.analysis.units) for (const [l, r] of u.roles) if (r.role !== 'helper') roles.set(l, r); + const rows = src.map((text, i) => { + const n = i + 1; + const ds = byLine.get(n) || []; + const worst = ds.find((d) => d.severity === 'error') || ds.find((d) => d.severity === 'warning') || ds[0]; + const role = roles.get(n); + const row = el('div', { class: `pl-code-row${role ? ` role-${role.role}` : ''}`, 'data-line': String(n) }); + const count = el('span', { class: 'pl-code-count' }); + const marker = el('span', { class: 'pl-code-marker' }); + const num = el('span', { class: 'pl-code-num', text: String(n) }); + const flag = el('span', { class: `pl-code-flag${worst ? ` is-${worst.severity}` : ''}`, text: worst ? (worst.severity === 'error' ? '✖' : worst.severity === 'warning' ? '▲' : '•') : '', title: ds.map((d) => d.message).join('\n\n') }); + const code = el('code', { class: 'pl-code-text', html: lineHtml(text) || ' ' }); + add(row, [count, marker, num, flag, code]); + if (role && opts.roles) row.title = `${ROLE_LABEL[role.role]}${role.why ? `: ${role.why}` : ''}`; + if (ctx.widget) row.addEventListener('click', () => ctx.widget.gotoLine(n)); + body.appendChild(row); + return { row, count, marker, n }; + }); + if (opts.maxHeight) body.style.maxHeight = opts.maxHeight; + + // cumulative line counts, checkpointed every 256 steps so scrubbing stays quick + const trace = ctx.trace; + const checkpoints = []; + const countsAt = (t) => { + if (!trace) return new Map(); + const base = Math.floor(t / 256); + let start = 0; let m = new Map(); + for (let b = Math.min(base, checkpoints.length - 1); b >= 0; b--) if (checkpoints[b]) { start = b * 256; m = new Map(checkpoints[b]); break; } + for (let i = start; i < t; i++) { + if (i % 256 === 0 && !checkpoints[i / 256]) checkpoints[i / 256] = new Map(m); + const l = trace.steps[i].line; + if (l != null && trace.steps[i].kind !== 'check') m.set(l, (m.get(l) || 0) + 1); + } + return m; + }; + let maxHits = 1; + if (trace) { const all = countsAt(trace.count); maxHits = Math.max(1, ...all.values()); } + + let lastCurrent = null; + function update(frame) { + const counts = opts.counts ? countsAt(frame.t) : null; + const nextLine = frame.next?.line; const lastLine = frame.last?.line; + let current = null; + for (const r of rows) { + const isNext = r.n === nextLine; const isLast = r.n === lastLine; + r.row.classList.toggle('is-next', isNext); + r.row.classList.toggle('is-last', isLast && !isNext); + r.row.classList.toggle('is-error', Boolean(frame.last?.kind === 'error' && r.n === lastLine)); + r.marker.textContent = isNext ? '▸' : ''; + if (isNext || (!nextLine && isLast)) current = r.row; + if (counts) { + const h = counts.get(r.n) || 0; + r.count.textContent = h ? commas(h) : ''; + if (opts.heat) r.row.style.setProperty('--heat', (h / maxHits).toFixed(3)); + } + } + if (current && current !== lastCurrent && body.scrollHeight > body.clientHeight + 4) { + const top = current.offsetTop - body.offsetTop; + if (top < body.scrollTop || top > body.scrollTop + body.clientHeight - 24) body.scrollTop = Math.max(0, top - body.clientHeight / 3); + } + lastCurrent = current; + } + + const tools = opts.roles ? legend([['role-setup', 'Set up'], ['role-visit', 'Visit'], ['role-test', 'Test'], ['role-update', 'Update'], ['role-report', 'Report']], 'pl-roles-legend') : null; + const node = panel(opts.title ?? (ctx.call ? `Running ${ctx.call.text || ctx.call.name}` : 'Program'), el('div', {}, [body, tools]), 'pl-code-panel'); + // A listing too wide for half the widget takes the whole row, rather than clipping its lines. + if (Math.max(...src.map((l) => l.replace(/\t/g, ' ').length)) > (opts.wideAt ?? 52)) node.classList.add('is-wide'); + return { node, update }; +} + +/* ------------------------------------------------------------------ */ +/* 2. Variables */ +/* ------------------------------------------------------------------ */ + +/** For each sequence, the whole-number names used to index it: T -> [I, J]. */ +function pointerNames(program) { + const out = new Map(); + const onExpr = (e) => walkExpr(e, (x) => { + if (x.kind !== 'Index' || x.target.kind !== 'Name') return; + const k = up(x.target.name); + if (!out.has(k)) out.set(k, new Set()); + walkExpr(x.indices[0], (y) => { if (y.kind === 'Name') out.get(k).add(up(y.name)); }); + }); + walkStatements(program.ast.body, (st) => { + for (const e of statementExprs(st)) onExpr(e); + if (st.kind === 'Assign') onExpr(st.target); + }); + return out; +} + +/** + * Every variable of the running subroutine, drawn by type: a STRING or LIST + * as a row of cells with their positions underneath, the positions a loop + * counter points at marked, a HASHMAP as a table, a SET as a group of chips. + * Blue marks what the last step read; amber, what it changed. + * + * options: { frames: 'top' | 'all', hide: ['N'], pointers: true, maxCells: 64 } + */ +export function variablesView(ctx, options = {}) { + const opts = { frames: 'all', pointers: true, maxCells: 64, hide: [], ...options }; + // hooks for pages that show a program in other words (the plain English steps) + const label = (n) => (opts.label ? opts.label(n) : n); + const body = el('div', { class: 'pl-vars' }); + const pointers = pointerNames(ctx.program); + const hide = new Set(opts.hide.map(up)); + + function cellStrip(name, items, type, reads, writes, frameVars) { + const strip = el('div', { class: 'pl-cells' }); + const n = items.length; + const shown = Math.min(n, opts.maxCells); + const ptrs = new Map(); + if (opts.pointers) { + for (const p of pointers.get(up(name)) || []) { + const v = frameVars[p]?.value; + if (Number.isInteger(v) && v >= 0 && v <= n) { if (!ptrs.has(v)) ptrs.set(v, []); ptrs.get(v).push(label(frameVars[p].name)); } + } + } + for (let i = 0; i < shown; i++) { + const cell = el('span', { class: 'pl-cell' }, [ + el('span', { class: 'pl-cell-v', text: type?.k === 'STRING' ? items[i] : formatValue(items[i], type?.of, { bareChar: true, max: 6 }) }), + el('span', { class: 'pl-cell-i', text: String(i) }), + ptrs.has(i) ? el('span', { class: 'pl-cell-p', text: ptrs.get(i).join(' ') }) : null, + ]); + if (items[i] === UNSET) cell.classList.add('is-unset'); + if (reads.has(i)) cell.classList.add('is-read'); + if (writes.has(i)) cell.classList.add('is-written'); + strip.appendChild(cell); + } + if (n > shown) strip.appendChild(el('span', { class: 'pl-cell pl-cell-more', text: `… ${n - shown} more` })); + if (ptrs.has(n) && n <= shown) strip.appendChild(el('span', { class: 'pl-cell pl-cell-end' }, [el('span', { class: 'pl-cell-v', text: '' }), el('span', { class: 'pl-cell-i', text: String(n) }), el('span', { class: 'pl-cell-p', text: ptrs.get(n).join(' ') })])); + if (!n) strip.appendChild(el('span', { class: 'pl-empty', text: type?.k === 'STRING' ? 'empty: ""' : 'empty' })); + return strip; + } + + function valueNode(v, type, name, reads, writes, frameVars) { + if (v === UNSET) return el('span', { class: 'pl-unset', text: 'no value yet' }); + if (typeof v === 'string' && type?.k === 'STRING') return cellStrip(name, chars(v), type, reads, writes, frameVars); + if (Array.isArray(v)) return cellStrip(name, v, type, reads, writes, frameVars); + if (v instanceof PArray) { + if (v.dims.length === 2) { + const table = el('table', { class: 'pl-grid' }); + const head = el('tr', {}, [el('th'), ...Array.from({ length: v.dims[1] }, (_, c) => el('th', { text: String(c) }))]); + add(table, head); + for (let r = 0; r < v.dims[0]; r++) { + const tr = el('tr', {}, [el('th', { text: String(r) })]); + for (let c = 0; c < v.dims[1]; c++) { + const x = v.data[r * v.dims[1] + c]; + const td = el('td', { text: x === UNSET ? '' : formatValue(x, type?.of, { bareChar: true, max: 4 }) }); + if (reads.has(`${r},${c}`)) td.classList.add('is-read'); + if (writes.has(`${r},${c}`)) td.classList.add('is-written'); + tr.appendChild(td); + } + table.appendChild(tr); + } + return table; + } + return cellStrip(name, v.data, type, reads, writes, frameVars); + } + if (v instanceof Map) { + if (!v.size) return el('span', { class: 'pl-empty', text: 'empty' }); + const table = el('table', { class: 'pl-map' }); + for (const k of [...v.keys()].sort()) { + const tr = el('tr', {}, [el('td', { class: 'pl-map-k', text: formatValue(k, type?.key, { bareChar: true }) }), el('td', { text: formatValue(v.get(k), type?.val, { bareChar: true, max: 8 }) })]); + if (reads.has(k)) tr.classList.add('is-read'); + if (writes.has(k)) tr.classList.add('is-written'); + table.appendChild(tr); + } + return table; + } + if (v instanceof Set) { + if (!v.size) return el('span', { class: 'pl-empty', text: 'empty' }); + return el('span', { class: 'pl-chips' }, [...v].sort().map((x) => el('span', { class: 'pl-chip', text: formatValue(x, type?.of, { bareChar: true }) }))); + } + if (v instanceof PRecord) { + const table = el('table', { class: 'pl-map' }); + for (const [k, x] of v.fields) table.appendChild(el('tr', {}, [el('td', { class: 'pl-map-k', text: v.names.get(k) }), el('td', { text: x === UNSET ? 'no value yet' : formatValue(x, type?.fields?.get(k), { bareChar: true, max: 8 }) })])); + return table; + } + return el('span', { class: 'pl-scalar', text: formatValue(v, type, { bareChar: true }) }); + } + + function update(frame) { + clear(body); + const frames = frame.state.frames; + let list = opts.frames === 'top' ? frames.slice(-1) : frames; + // During a call from outside, the empty main program is noise. + if (list.length > 1 && list[0].name === 'main' && !Object.keys(list[0].vars).length) list = list.slice(1); + const readsBy = new Map(); const writesBy = new Map(); + const key = (f, n) => `${f}:${up(n)}`; + for (const r of frame.last?.reads || []) { const k = key(r.frame, r.name); if (!readsBy.has(k)) readsBy.set(k, new Set()); readsBy.get(k).add(r.path ? (r.path.length === 2 ? `${r.path[0]},${r.path[1]}` : r.path[0]) : '*'); } + for (const w of frame.last?.writes || []) { const k = key(w.frame, w.name); if (!writesBy.has(k)) writesBy.set(k, new Set()); writesBy.get(k).add(w.path ? (w.path.length === 2 ? `${w.path[0]},${w.path[1]}` : w.path[0]) : '*'); } + [...list].reverse().forEach((f, idx) => { + const top = idx === 0; + const box = el('div', { class: `pl-frame${top ? ' is-top' : ''}` }); + add(box, el('div', { class: 'pl-frame-head' }, [ + el('strong', { text: opts.frameTitle ? opts.frameTitle(f.name) : f.name === 'main' ? 'Main program' : f.name }), + f.callLine && f.name !== 'main' && !opts.frameTitle ? el('span', { class: 'pl-muted', text: ` called from line ${f.callLine}` }) : null, + ])); + const vars = Object.values(f.vars).filter((x) => !hide.has(up(x.name)) && x.kind !== 'bound' && x.kind !== 'result'); + if (!vars.length) add(box, el('p', { class: 'pl-muted', text: 'No variables yet.' })); + vars.sort((a, b) => (a.line ?? 0) - (b.line ?? 0)); + const byName = {}; for (const x of Object.values(f.vars)) byName[up(x.name)] = x; + for (const x of vars) { + const reads = readsBy.get(key(f.id, x.name)) || new Set(); + const writes = writesBy.get(key(f.id, x.name)) || new Set(); + const row = el('div', { class: 'pl-var' }, [ + el('div', { class: 'pl-var-name' }, [el('code', { text: label(x.name) }), el('span', { class: 'pl-var-type', text: opts.kindText ? opts.kindText(x) : x.kind === 'counter' ? 'FOR counter' : x.kind === 'each' ? 'FOR EACH value' : x.kind === 'param' ? `${typeName(x.type)} parameter` : typeName(x.type) })]), + el('div', { class: 'pl-var-value' }, [valueNode(x.value, x.type, x.name, reads, writes, byName)]), + ]); + if (writes.size) row.classList.add('is-written'); + else if (reads.size) row.classList.add('is-read'); + box.appendChild(row); + } + body.appendChild(box); + }); + } + const node = panel(options.title ?? 'Variables', el('div', {}, [body, legend([['is-read', 'read by the last step'], ['is-written', 'changed by the last step']])]), 'pl-vars-panel'); + return { node, update }; +} + +/* ------------------------------------------------------------------ */ +/* 3. The trace table */ +/* ------------------------------------------------------------------ */ + +const TRACE_KINDS = new Set(['assign', 'test', 'loop-start', 'loop-test', 'loop-end', 'next', 'output', 'input', 'return', 'call', 'builtin', 'select', 'break', 'continue', 'error', 'unrecorded']); + +/** + * The table the guide builds by hand: one row per step, one column per + * watched variable, each cell the value after that step, changed cells + * marked. Clicking a row goes to that step. + * + * options: { watch: ['I', 'Total'], checks: false, declares: false, window: 300 } + */ +export function traceView(ctx, options = {}) { + const opts = { checks: false, declares: false, window: 300, ...options }; + const trace = ctx.trace; + const keep = (s) => (TRACE_KINDS.has(s.kind) || (opts.checks && s.kind === 'check') || (opts.declares && s.kind === 'declare')); + const idx = []; + trace.steps.forEach((s, i) => { if (keep(s)) idx.push(i); }); + // watch: the named variables, or every scalar the run ever showed, top frame first + let watch = (opts.watch || []).map(up); + if (!watch.length) { + const seen = new Map(); + for (const st of trace.states) { + const f = st.frames[st.frames.length - 1]; + for (const [k, v] of Object.entries(f.vars)) { + if (seen.has(k) || v.kind === 'bound' || v.kind === 'result') continue; + const t = v.type?.k; + if (['INTEGER', 'FLOAT', 'BOOL', 'CHAR'].includes(t) || (t === 'LIST' && v.kind !== 'param')) seen.set(k, v.name); + } + if (seen.size > 7) break; + } + watch = [...seen.keys()].slice(0, 7); + } + const names = new Map(); + for (const st of trace.states) for (const f of st.frames) for (const v of Object.values(f.vars)) if (!names.has(up(v.name))) names.set(up(v.name), v.name); + const table = el('table', { class: 'pl-trace' }); + add(table, el('thead', {}, [el('tr', {}, [ + el('th', { text: 'step' }), el('th', { text: 'line' }), el('th', { class: 'pl-trace-what', text: 'what happened' }), + ...watch.map((w) => el('th', {}, [el('code', { text: names.get(w) || w })])), + ])])); + const tbody = el('tbody'); + add(table, tbody); + const wrap = el('div', { class: 'pl-trace-wrap' }, [table]); + + const valueIn = (state, k) => { + for (let i = state.frames.length - 1; i >= 0; i--) { const v = state.frames[i].vars[k]; if (v) return v; } + return null; + }; + const cellText = (v) => (!v ? '' : v.value === UNSET ? '—' : formatValue(v.value, v.type, { bareChar: true, max: 6 })); + let rendered = [-1, -1]; + const rowByStep = new Map(); + function render(center) { + clear(tbody); + rowByStep.clear(); + const pos = Math.max(0, idx.findIndex((i) => i >= center)); + const lo = Math.max(0, pos - Math.floor(opts.window / 2)); + const hi = Math.min(idx.length, lo + opts.window); + if (lo > 0) tbody.appendChild(el('tr', { class: 'pl-trace-gap' }, [el('td', { colspan: String(3 + watch.length), text: `… ${commas(lo)} earlier rows` })])); + for (let r = lo; r < hi; r++) { + const i = idx[r]; + const s = trace.steps[i]; + const before = trace.states[i]; const after = trace.states[i + 1] || before; + const tr = el('tr', { class: `pl-trace-row k-${s.kind}` }, [ + el('td', { class: 'pl-muted', text: String(i + 1) }), + el('td', { text: s.line == null ? '' : String(s.line) }), + el('td', { class: 'pl-trace-what', text: s.text }), + ...watch.map((w) => { + const a = cellText(valueIn(after, w)); const b = cellText(valueIn(before, w)); + return el('td', { class: a !== b ? 'is-changed' : '', text: a }); + }), + ]); + tr.addEventListener('click', () => ctx.widget?.goto(i + 1)); + tbody.appendChild(tr); + rowByStep.set(i, tr); + } + if (hi < idx.length) tbody.appendChild(el('tr', { class: 'pl-trace-gap' }, [el('td', { colspan: String(3 + watch.length), text: `… ${commas(idx.length - hi)} later rows` })])); + rendered = [idx[lo] ?? 0, idx[hi - 1] ?? 0]; + } + render(0); + let lastRow = null; + function update(frame) { + const cur = frame.t - 1; + if (idx.length > opts.window && (cur < rendered[0] || cur > rendered[1])) render(cur); + if (lastRow) lastRow.classList.remove('is-current'); + let k = cur; + while (k >= 0 && !rowByStep.has(k)) k--; + const row = rowByStep.get(k); + tbody.querySelectorAll('tr.is-future').forEach((r) => r.classList.remove('is-future')); + for (const [i, r] of rowByStep) r.classList.toggle('is-future', i > cur); + if (row) { + row.classList.add('is-current'); + const top = row.offsetTop; + if (top < wrap.scrollTop + 30 || top > wrap.scrollTop + wrap.clientHeight - 30) wrap.scrollTop = Math.max(0, top - wrap.clientHeight / 2); + } + lastRow = row; + } + const node = panel(options.title ?? 'Trace table', wrap, 'pl-trace-panel pl-span-2'); + return { node, update }; +} + +/* ------------------------------------------------------------------ */ +/* 4. Explanation and output */ +/* ------------------------------------------------------------------ */ + +/** What the last step did, in a sentence, and what happens next. */ +export function explainView(ctx, options = {}) { + const body = el('div', { class: 'pl-explain' }); + function update(frame) { + clear(body); + const s = frame.last; + if (!s) { + add(body, el('p', { text: ctx.call ? `Ready to call ${ctx.call.text || ctx.call.name}. Step to begin.` : 'Ready to run the main program. Step to begin.' })); + } else if (s.kind === 'error') { + add(body, el('div', { class: 'pl-error-box' }, [el('strong', { text: `Line ${s.line ?? '?'} stops the run. ` }), s.text])); + } else { + add(body, el('p', {}, [el('span', { class: 'pl-explain-line', text: s.line != null ? `Line ${s.line}` : '' }), ' ', s.text])); + } + if (frame.next && frame.next.kind !== 'error') add(body, el('p', { class: 'pl-muted', text: `Next: line ${frame.next.line}.` })); + else if (!frame.next && frame.t > 0 && frame.last?.kind !== 'error') { + const tr = ctx.trace; + add(body, el('p', { class: 'pl-done', text: tr.call && tr.result !== undefined ? `Finished: ${tr.call.name} returned ${formatValue(tr.result, tr.resultType)}.` : `Finished after ${plural(tr.count, 'step')}.` })); + } + } + return { node: panel(options.title ?? 'What just happened', body, 'pl-explain-panel'), update }; +} + +/** Everything OUTPUT has shown so far. */ +export function outputView(ctx, options = {}) { + const body = el('pre', { class: 'pl-console' }); + const shows = /\bOUTPUT\b/.test(ctx.study.source); + function update(frame) { + if (!shows) return; + const n = frame.state.outputLength; + const lines = ctx.trace.output.slice(0, n); + body.textContent = lines.length ? lines.join('\n') : ''; + body.classList.toggle('is-empty', !lines.length); + if (!lines.length) body.textContent = 'Nothing shown yet.'; + } + const node = panel(options.title ?? 'Output', body, 'pl-output-panel'); + if (!shows && !options.always) node.hidden = true; + return { node, update }; +} + +/* ------------------------------------------------------------------ */ +/* Composer */ +/* ------------------------------------------------------------------ */ + +const VIEW_REGISTRY = { + code: codeView, + variables: variablesView, + trace: traceView, + explain: explainView, + output: outputView, + ...ANALYSIS_VIEWS, +}; + +export function registerViewType(name, factory) { VIEW_REGISTRY[name] = factory; } +export function viewNames() { return Object.keys(VIEW_REGISTRY); } + +/** Views that make sense without a run, when a program cannot run yet. */ +const STATIC_VIEWS = new Set(['diagnostics', 'flow', 'cfg', 'jobs', 'proof', 'translate', 'defuse']); + +/** A widget-shaped box for a program with errors: the listing, marked, and the findings. */ +function failure(ctx, config) { + const root = el('div', { class: `pl pl-${config.layout || 'one-column'}` }); + if (config.title) add(root, el('h3', { class: 'pl-title', text: config.title })); + const errs = ctx.study.errors; + if (errs.length) add(root, el('p', { class: 'pl-error', text: `${plural(errs.length, 'line needs', 'lines need')} attention before this can run.` })); + if (ctx.error) add(root, el('p', { class: 'pl-error', text: ctx.error.message })); + const grid = el('div', { class: 'pl-grid' }); + add(root, grid); + add(grid, codeView(ctx, { counts: false, title: 'Program' }).node); + const d = VIEW_REGISTRY.diagnostics(ctx, {}); + d.update(null); + add(grid, d.node); + for (const entry of config.views || []) { + const [name, o] = Array.isArray(entry) ? entry : [entry, {}]; + if (!STATIC_VIEWS.has(name) || name === 'diagnostics' || !ctx.study.analysis || ctx.study.errors.some((x) => x.code !== 'blank')) continue; + try { const v = VIEW_REGISTRY[name](ctx, o); v.update(null); add(grid, v.node); } catch { /* skip */ } + } + return root; +} + +export function mountWidget(target, spec, extra = {}) { + const config = { views: ['code', 'variables', 'explain', 'trace'], layout: 'two-column', speed: 450, start: 0, ...spec, ...extra }; + const ctx = session(config); + const noop = () => { }; + if (!ctx.trace) { + const root = failure(ctx, config); + if (target) { clear(target); target.appendChild(root); } + return { node: root, ctx, trace: null, error: ctx.error || ctx.study.errors[0], goto: noop, gotoLine: noop, play: noop, pause: noop, step: noop, reset: noop, t: 0, onChange: noop, fullscreen: noop }; + } + const trace = ctx.trace; + const root = el('div', { class: `pl pl-${config.layout}` }); + if (config.title) add(root, el('h3', { class: 'pl-title', text: config.title })); + if (config.blurb) add(root, el('p', { class: 'pl-blurb', text: config.blurb })); + + let t = Math.max(0, Math.min(config.start === 'end' ? trace.count : config.start, trace.count)); + let timer = null; + const listeners = []; + const slider = el('input', { type: 'range', min: '0', max: String(trace.count), value: String(t), class: 'pl-slider', 'aria-label': 'step' }); + const readout = el('span', { class: 'pl-readout' }); + const playBtn = el('button', { class: 'pl-btn', type: 'button', text: 'Play' }); + const fullBtn = el('button', { class: 'pl-btn pl-btn-full', type: 'button', text: 'Fullscreen', title: 'Open this widget at full window size (Esc to close)', 'aria-pressed': 'false' }); + const bar = el('div', { class: 'pl-controls' }, [ + el('button', { class: 'pl-btn', type: 'button', text: 'Reset', onclick: () => { pause(); goto(0); } }), + el('button', { class: 'pl-btn', type: 'button', text: 'Back', onclick: () => { pause(); goto(t - 1); } }), + playBtn, + el('button', { class: 'pl-btn pl-btn-primary', type: 'button', text: 'Step', onclick: () => { pause(); goto(t + 1); } }), + el('button', { class: 'pl-btn', type: 'button', text: 'End', onclick: () => { pause(); goto(trace.count); } }), + slider, readout, fullBtn, + ]); + slider.addEventListener('input', () => { pause(); goto(Number(slider.value)); }); + playBtn.addEventListener('click', () => (timer ? pause() : play())); + let full = false; + function setFullscreen(on) { + full = Boolean(on); + root.classList.toggle('is-fullscreen', full); + fullBtn.textContent = full ? 'Close' : 'Fullscreen'; + fullBtn.setAttribute('aria-pressed', full ? 'true' : 'false'); + if (typeof document !== 'undefined' && document.body) document.body.style.overflow = full ? 'hidden' : ''; + if (full) root.focus?.(); + } + fullBtn.addEventListener('click', () => setFullscreen(!full)); + add(root, bar); + + const grid = el('div', { class: 'pl-grid' }); + add(root, grid); + const controller = { + node: root, ctx, trace, + goto: (n) => { pause(); goto(n); }, + gotoLine: (line) => { + // the next time that line runs after the current step, or the first time + let k = trace.steps.findIndex((s, i) => i >= t && s.line === line); + if (k < 0) k = trace.steps.findIndex((s) => s.line === line); + if (k >= 0) { pause(); goto(k); } + }, + }; + ctx.widget = controller; + const views = []; + for (const entry of config.views) { + const [name, options] = Array.isArray(entry) ? entry : [entry, {}]; + const factory = VIEW_REGISTRY[name]; + if (!factory) { add(grid, el('p', { class: 'pl-error', text: `No view named "${name}".` })); continue; } + try { + const v = factory(ctx, { ...(config.viewOptions?.[name] || {}), ...options }); + if (options?.span) v.node.classList.add(`pl-span-${options.span}`); + v.name = name; + views.push(v); + add(grid, v.node); + } catch (err) { + add(grid, el('p', { class: 'pl-error', text: `The ${name} view could not be built: ${err.message}` })); + console.error(err); // eslint-disable-line no-console + } + } + if (trace.error && config.showError !== false) { + add(root, el('p', { class: 'pl-error' }, [el('strong', { text: `Stopped at line ${trace.error.line ?? '?'}, after step ${trace.count}. ` }), trace.error.message])); + } + + function draw() { + const frame = frameAt(trace, t); + for (const v of views) { + try { v.update(frame); v.node.classList.remove('has-failed'); } catch (err) { + if (!v.node.classList.contains('has-failed')) { + v.node.classList.add('has-failed'); + add(v.node, el('p', { class: 'pl-error', text: `The ${v.name} view failed at step ${t}: ${err.message}` })); + console.error(err); // eslint-disable-line no-console + } + } + } + slider.value = String(t); + readout.textContent = `step ${t} of ${trace.count}`; + playBtn.textContent = timer ? 'Pause' : 'Play'; + for (const fn of listeners) fn(frame); + } + function goto(n) { t = Math.max(0, Math.min(n, trace.count)); draw(); } + function play() { + if (timer) return; + if (t >= trace.count) t = 0; + timer = setInterval(() => (t >= trace.count ? pause() : goto(t + 1)), config.speed); + draw(); + } + function pause() { if (timer) { clearInterval(timer); timer = null; draw(); } } + + root.tabIndex = 0; + root.addEventListener('keydown', (e) => { + if (/^(INPUT|TEXTAREA|SELECT)$/.test(e.target.tagName) && e.target !== slider) return; + if (e.key === 'Home') { pause(); goto(0); e.preventDefault(); } + if (e.key === 'End') { pause(); goto(trace.count); e.preventDefault(); } + if (e.key === 'ArrowRight') { pause(); goto(t + 1); e.preventDefault(); } + if (e.key === 'ArrowLeft') { pause(); goto(t - 1); e.preventDefault(); } + if (e.key === ' ') { if (timer) pause(); else play(); e.preventDefault(); } + if (e.key === 'Escape' && full) { setFullscreen(false); e.preventDefault(); } + }); + if (target) { clear(target); target.appendChild(root); } + draw(); + Object.assign(controller, { + play, pause, fullscreen: setFullscreen, + step: () => goto(t + 1), reset: () => goto(0), + get t() { return t; }, + onChange: (fn) => { listeners.push(fn); fn(frameAt(trace, t)); }, + }); + return controller; +} + +/** For Observable JS, where a cell returns its node. */ +export function widget(spec, extra = {}) { return mountWidget(null, spec, extra).node; } + +export { study, run }; diff --git a/src/engines/shared/dom.js b/src/engines/shared/dom.js new file mode 100644 index 0000000..f2d2d0e --- /dev/null +++ b/src/engines/shared/dom.js @@ -0,0 +1,66 @@ +/** + * shared/dom.js + * + * The DOM helpers every interactive widget on the site builds on. No + * framework: the nodes are plain elements, so they drop into Observable JS, + * a Quarto page, or a bare HTML file. + * + * Both widget families (pseudo/ and asm86/) re-export this file from their + * own dom.js and add the few helpers that carry their CSS class prefix + * (panel, legend, button). Import from the family's dom.js in widget code; + * import from here only for code that belongs to neither family. + */ + +export const SVGNS = 'http://www.w3.org/2000/svg'; + +/** An HTML element: el('div', { class: 'x', text: 'hi', onclick }, [children]). */ +export function el(tag, attrs = {}, children = []) { + const n = document.createElement(tag); + apply(n, attrs); + add(n, children); + return n; +} + +/** An SVG element, same signature as el(). */ +export function sv(tag, attrs = {}, children = []) { + const n = document.createElementNS(SVGNS, tag); + for (const [k, v] of Object.entries(attrs)) { + if (v == null || v === false) continue; + // `text` is content, not an attribute. Setting it as an attribute leaves + // every element empty, which is silent and hard to spot. + if (k === 'text') { n.textContent = String(v); continue; } + if (k.startsWith('on') && typeof v === 'function') { n.addEventListener(k.slice(2), v); continue; } + n.setAttribute(k, String(v)); + } + add(n, children); + return n; +} + +export function apply(n, attrs) { + for (const [k, v] of Object.entries(attrs)) { + if (v == null || v === false) continue; + if (k === 'class') n.className = v; + else if (k === 'text') n.textContent = String(v); + else if (k === 'html') n.innerHTML = v; + else if (k === 'style' && typeof v === 'object') Object.assign(n.style, v); + else if (k.startsWith('on') && typeof v === 'function') n.addEventListener(k.slice(2), v); + else if (k === 'dataset') Object.assign(n.dataset, v); + else n.setAttribute(k, String(v)); + } +} + +export function add(n, children) { + const list = Array.isArray(children) ? children : [children]; + for (const c of list) { + if (c == null || c === false) continue; + n.appendChild(typeof c === 'string' || typeof c === 'number' ? document.createTextNode(String(c)) : c); + } +} + +export function clear(n) { while (n.firstChild) n.removeChild(n.firstChild); } + +/** 1,234,567 */ +export function commas(n) { return Number(n).toLocaleString('en-US'); } + +/** plural(3, 'cycle') -> '3 cycles'; plural(1, 'line needs', 'lines need') -> '1 line needs' */ +export function plural(n, word, many = `${word}s`) { return `${commas(n)} ${n === 1 ? word : many}`; } diff --git a/src/engines/shared/share.js b/src/engines/shared/share.js new file mode 100644 index 0000000..466654d --- /dev/null +++ b/src/engines/shared/share.js @@ -0,0 +1,19 @@ +/** + * shared/share.js + * + * Put a widget's state in a URL fragment and take it back out, so a lab + * configuration can be shared as a link. The state is JSON, UTF-8 encoded, + * then base64url (no padding), which is safe in a fragment. + */ + +export function encodeState(state) { + const bytes = new TextEncoder().encode(JSON.stringify(state)); + let bin = ''; + for (const b of bytes) bin += String.fromCharCode(b); + return btoa(bin).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} + +export function decodeState(s) { + const bin = atob(s.replace(/-/g, '+').replace(/_/g, '/')); + return JSON.parse(new TextDecoder().decode(Uint8Array.from(bin, (c) => c.charCodeAt(0)))); +} diff --git a/src/engines/shared/text.js b/src/engines/shared/text.js new file mode 100644 index 0000000..8a5f407 --- /dev/null +++ b/src/engines/shared/text.js @@ -0,0 +1,27 @@ +/** + * shared/text.js + * + * Small string helpers used by more than one widget family. + */ + +/** + * How many single-letter edits (insert, delete, substitute) turn one string + * into another. + * + * options: + * fold: compare ignoring case + * swaps: count two adjacent letters swapped as one edit, not two + * (a slip of the fingers, not a different word) + */ +export function editDistance(a, b, { fold = false, swaps = false } = {}) { + if (fold) { a = a.toLowerCase(); b = b.toLowerCase(); } + const d = Array.from({ length: a.length + 1 }, (_, i) => [i, ...new Array(b.length).fill(0)]); + for (let j = 1; j <= b.length; j++) d[0][j] = j; + for (let i = 1; i <= a.length; i++) { + for (let j = 1; j <= b.length; j++) { + d[i][j] = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1)); + if (swaps && i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) d[i][j] = Math.min(d[i][j], d[i - 2][j - 2] + 1); + } + } + return d[a.length][b.length]; +} diff --git a/src/engines/shared/viz.js b/src/engines/shared/viz.js new file mode 100644 index 0000000..180e36d --- /dev/null +++ b/src/engines/shared/viz.js @@ -0,0 +1,714 @@ +/** + * shared/viz.js + * + * Small, dependency-free charts for course pages: line, bar, scatter and + * heatmap, plus the pieces (axes, legend, scales) to build a custom figure. + * Everything is plain SVG built with dom.js, so the nodes drop into Observable + * JS, a Quarto page, or a bare HTML file. No d3, no Observable Plot, no CDN. + * + * Needs base/tokens.css on the page: colors are CSS custom properties + * (--viz-1 .. --viz-8, --viz-ink, --viz-muted, --viz-grid, --viz-bg, ...) set + * through `style` attributes, so the charts follow the light/dark toggle with + * no JavaScript. The math (scales, ticks, number formatting, color strings) + * is pure and works in Node without a DOM; only the chart builders need one. + * + * Every chart REQUIRES `alt`, a sentence or two saying what the figure shows. + * It becomes the SVG's ; a chart without it throws. + * + * const viz = await import("/static/js/shared/viz.js"); + * const chart = viz.lineChart({ + * title: 'Bacterial growth', + * alt: 'Cell counts rise about tenfold per hour, then level off near one million.', + * xLabel: 'Time (h)', yLabel: 'Cells per mL', yScale: 'log', + * series: [{ name: 'Wild type', x: hours, y: wild }, + * { name: 'Mutant', x: hours, y: mutant, dash: '6 4' }], + * }); + * display(viz.figure(chart, { caption: 'Growth curves.', source: 'Lab 3 data' })); + * + * Scales are functions with plain properties: scale.domain, scale.range, + * scale.type ('linear' | 'log' | 'band'), scale.invert (linear and log), + * scale.bandwidth and scale.step (band). Colors in the palette are strings + * such as 'var(--viz-1)', not hex values. + */ +import { sv, el } from './dom.js'; + +/* ------------------------------------------------------------------ */ +/* Pure math */ +/* ------------------------------------------------------------------ */ + +const isNum = (v) => typeof v === 'number' && Number.isFinite(v); +const clamp = (v, lo, hi) => Math.min(hi, Math.max(lo, v)); +const r2 = (n) => Math.round(n * 100) / 100; + +/** [min, max] of the finite numbers in a list (no spread: safe for long data), or null. */ +function extent(values) { + let lo = Infinity, hi = -Infinity; + for (const v of values) if (isNum(v)) { if (v < lo) lo = v; if (v > hi) hi = v; } + return lo <= hi ? [lo, hi] : null; +} + +/** Round a raw step to 1, 2 or 5 times a power of ten. */ +function niceStep(raw) { + const exp = Math.floor(Math.log10(raw)); + const f = raw / 10 ** exp; + const m = f >= 7.07 ? 10 : f >= 3.16 ? 5 : f >= 1.41 ? 2 : 1; + return Number(`${m}e${exp}`); +} + +/** + * "Nice" tick values (steps of 1, 2 or 5 times a power of ten) that cover + * [min, max]: the first tick is <= min and the last is >= max. `count` is a + * target, not a promise; expect between count-1 and about 2*count ticks. + * min > max is swapped; min == max is widened by one order of magnitude; a + * non-finite bound gives []. + */ +export function niceTicks(min, max, count = 5) { + if (!Number.isFinite(min) || !Number.isFinite(max)) return []; + if (min > max) [min, max] = [max, min]; + count = Number.isFinite(count) ? Math.max(2, Math.floor(count)) : 5; + if (min === max) { + const pad = min === 0 ? 1 : 10 ** Math.floor(Math.log10(Math.abs(min))); + min -= pad; max += pad; + } + const step = niceStep((max - min) / (count - 1)); + if (!Number.isFinite(step) || step === 0) return [min, max]; + const k0 = Math.floor(min / step + 1e-9); + const k1 = Math.ceil(max / step - 1e-9); + const out = []; + for (let k = k0; k <= k1; k++) { + // k * step carries binary rounding noise (0.30000000000000004). k has few + // digits and step has one, so rounding to digits(k) + 2 removes the noise + // without touching real values, even for a range of 1e-13 around 1. + const sig = clamp(String(Math.abs(k)).length + 2, 3, 17); + const v = Number((k * step).toPrecision(sig)) || 0; // || 0 turns -0 into 0 + if (out.length === 0 || v !== out[out.length - 1]) out.push(v); + } + return out; +} + +/** + * Tick values for a log axis: powers of ten covering [min, max] (both > 0). + * When the range is two decades or less it adds 2x and 5x ticks, so a narrow + * log axis is not left with one label. Very wide ranges skip decades to stay + * near eight ticks. Only values inside [min, max] are returned. + */ +export function logTicks(min, max) { + if (!(min > 0 && max > 0)) throw new Error(`logTicks: range must be positive, got [${min}, ${max}]`); + if (min > max) [min, max] = [max, min]; + const a = Math.floor(Math.log10(min) + 1e-9), b = Math.ceil(Math.log10(max) - 1e-9); + const mults = b - a <= 2 ? [1, 2, 5] : [1]; + const stride = Math.ceil((b - a + 1) / 8); + const out = []; + for (let e = a; e <= b; e += stride) { + for (const m of mults) { + const v = Number(`${m}e${e}`); + if (v >= min * (1 - 1e-9) && v <= max * (1 + 1e-9)) out.push(v); + } + } + return out; +} + +function checkPair(fn, pair, what) { + if (!Array.isArray(pair) || pair.length !== 2 || !pair.every(isNum)) { + throw new TypeError(`${fn}: ${what} must be two finite numbers, got ${JSON.stringify(pair)}`); + } +} + +/** + * Linear scale: maps domain [d0, d1] onto range [r0, r1]. Ranges may run + * backwards (an SVG y axis maps small values to large pixel numbers). With + * `clamp` the output never leaves the range. If d0 == d1 every input maps to + * the middle of the range. Properties: .invert(pixel), .domain, .range, + * .type = 'linear', .ticks(count). + */ +export function linearScale(domain, range, { clamp: clampOut = false } = {}) { + checkPair('linearScale', domain, 'domain'); + checkPair('linearScale', range, 'range'); + const [d0, d1] = domain, [r0, r1] = range; + const flat = d0 === d1; + const f = (x) => { + if (flat) return (r0 + r1) / 2; + let t = (x - d0) / (d1 - d0); + if (clampOut) t = clamp(t, 0, 1); + return r0 + t * (r1 - r0); + }; + f.invert = (y) => { + if (r0 === r1) return d0; + let t = (y - r0) / (r1 - r0); + if (clampOut) t = clamp(t, 0, 1); + return d0 + t * (d1 - d0); + }; + f.domain = [d0, d1]; + f.range = [r0, r1]; + f.type = 'linear'; + f.ticks = (count = 5) => niceTicks(d0, d1, count); + return f; +} + +/** + * Log10 scale. Same shape as linearScale, but the domain must be strictly + * positive (a log axis cannot show 0 or negative values) or an Error is + * thrown. Inputs <= 0 map to NaN. Properties: .invert, .domain, .range, + * .type = 'log', .ticks(). + */ +export function logScale(domain, range) { + checkPair('logScale', domain, 'domain'); + checkPair('logScale', range, 'range'); + const [d0, d1] = domain, [r0, r1] = range; + if (!(d0 > 0 && d1 > 0)) { + throw new Error(`logScale: domain must be strictly positive, got [${d0}, ${d1}]. A log axis cannot show zero or negative values; use a linear scale or drop those points.`); + } + const l0 = Math.log10(d0), l1 = Math.log10(d1); + const f = (x) => { + if (!(x > 0)) return NaN; + if (l0 === l1) return (r0 + r1) / 2; + return r0 + ((Math.log10(x) - l0) / (l1 - l0)) * (r1 - r0); + }; + f.invert = (y) => (r0 === r1 ? d0 : 10 ** (l0 + ((y - r0) / (r1 - r0)) * (l1 - l0))); + f.domain = [d0, d1]; + f.range = [r0, r1]; + f.type = 'log'; + f.ticks = () => logTicks(d0, d1); + return f; +} + +/** + * Band scale for categories: label -> pixel where that label's band starts + * (undefined for an unknown label). `padding` (0 <= p < 1) is the gap between + * bands and at both ends, as a fraction of the step. Labels must be unique. + * A backwards range lists the labels from the far end. Properties: + * .bandwidth (width of one band), .step, .domain (the labels), .range, + * .type = 'band', .ticks() (the labels). + */ +export function bandScale(labels, range, padding = 0.2) { + checkPair('bandScale', range, 'range'); + if (!(padding >= 0 && padding < 1)) throw new RangeError(`bandScale: padding must be in [0, 1), got ${padding}`); + const n = labels.length; + const index = new Map(); + labels.forEach((label, i) => { + if (index.has(label)) throw new Error(`bandScale: duplicate label ${JSON.stringify(label)}; labels must be unique`); + index.set(label, i); + }); + const lo = Math.min(...range), hi = Math.max(...range); + const reversed = range[1] < range[0]; + const step = n ? (hi - lo) / (n + padding) : 0; + const f = (label) => { + const i = index.get(label); + if (i === undefined) return undefined; + return lo + step * padding + step * (reversed ? n - 1 - i : i); + }; + f.bandwidth = step * (1 - padding); + f.step = step; + f.domain = labels.slice(); + f.range = [range[0], range[1]]; + f.type = 'band'; + f.ticks = () => f.domain; + return f; +} + +const SUPERSCRIPT = { '-': '⁻', 0: '⁰', 1: '¹', 2: '²', 3: '³', 4: '⁴', 5: '⁵', 6: '⁶', 7: '⁷', 8: '⁸', 9: '⁹' }; + +/** + * A number as compact text, locale-independent, no abbreviations like "1.2k". + * `digits` is the number of significant digits (default 3), trailing zeros + * are trimmed, and values of at least 10^digits keep all their integer digits. + * Inside [1e-3, 1e6) the result is plain (0.00123, 42, 123456). Outside it + * uses 1.23×10⁶ (`sci: true`) or 1.23e6 (`sci: false`). 0 -> '0', NaN -> 'NaN', + * Infinity -> '∞'. + */ +export function formatNumber(x, { digits = 3, sci = true } = {}) { + x = Number(x); + if (Number.isNaN(x)) return 'NaN'; + if (!Number.isFinite(x)) return x < 0 ? '-∞' : '∞'; + if (x === 0) return '0'; + digits = clamp(Math.floor(digits) || 3, 1, 15); + const a = Math.abs(x); + if (a >= 1e-3 && a < 1e6) { + return a >= 10 ** digits ? String(Math.round(x)) : String(Number(x.toPrecision(digits))); + } + const [mantissa, exponent] = x.toExponential(digits - 1).split('e'); + const m = Number(mantissa), e = Number(exponent); + if (!sci) return `${m}e${e}`; + const power = `10${String(e).replace(/./g, (c) => SUPERSCRIPT[c])}`; + return Math.abs(m) === 1 ? `${m < 0 ? '-' : ''}${power}` : `${m}×${power}`; +} + +/** n categorical colors as CSS strings 'var(--viz-1)' ... 'var(--viz-8)', cycling after 8. */ +export function palette(n) { + return Array.from({ length: Math.max(0, Math.floor(n) || 0) }, (_, i) => `var(--viz-${(i % 8) + 1})`); +} + +const unit = (t) => (Number.isFinite(t) ? clamp(t, 0, 1) : 0); +const percent = (t) => Number((unit(t) * 100).toFixed(1)); + +/** Single-hue ramp: t in [0, 1] runs from the page background (0) to --viz-ramp (1). */ +export function rampColor(t) { + return `color-mix(in srgb, var(--viz-ramp) ${percent(t)}%, var(--viz-bg))`; +} + +/** Diverging color: t in [-1, 1]; -1 is --viz-neg, 0 is the page background, +1 is --viz-pos. */ +export function divergingColor(t) { + t = Number.isFinite(t) ? clamp(t, -1, 1) : 0; + return `color-mix(in srgb, var(${t < 0 ? '--viz-neg' : '--viz-pos'}) ${percent(Math.abs(t))}%, var(--viz-bg))`; +} + +// Reference colors (the token defaults) used only to choose black or white +// text on a heatmap cell. The CSS cannot be read back, so the choice is made +// from t and these numbers. +const REF = { + light: { bg: [255, 255, 255], ramp: [0, 114, 178], neg: [0, 114, 178], pos: [213, 94, 0] }, + dark: { bg: [30, 30, 30], ramp: [86, 180, 233], neg: [86, 180, 233], pos: [242, 142, 75] }, +}; + +function luminance(rgb) { + const [r, g, b] = rgb.map((c) => { c /= 255; return c <= 0.03928 ? c / 12.92 : ((c + 0.055) / 1.055) ** 2.4; }); + return 0.2126 * r + 0.7152 * g + 0.0722 * b; +} + +/** + * '#000' or '#fff', whichever reads better on the heatmap cell for value t + * (t in [0,1] for a ramp, [-1,1] when `diverging`). `dark` selects the + * dark-theme token values. + */ +export function textColorFor(t, { diverging = false, dark = false } = {}) { + const ref = dark ? REF.dark : REF.light; + const a = Number.isFinite(t) ? clamp(Math.abs(t), 0, 1) : 0; + const base = diverging ? (t < 0 ? ref.neg : ref.pos) : ref.ramp; + const L = luminance(base.map((c, i) => c * a + ref.bg[i] * (1 - a))); + return 1.05 / (L + 0.05) > (L + 0.05) / 0.05 ? '#fff' : '#000'; +} + +/* ------------------------------------------------------------------ */ +/* Drawing helpers (need a DOM) */ +/* ------------------------------------------------------------------ */ + +let uid = 0; // unique ids for , <desc>, clipPaths and gradients +const INK = 'var(--viz-ink)', MUTED = 'var(--viz-muted)'; +const textWidth = (s, size = 11) => String(s).length * size * 0.6; // rough (wide system fonts included): enough to size margins +const clip = (s, n = 16) => (String(s).length > n ? `${String(s).slice(0, n - 1)}…` : String(s)); + +/** An SVG <text> in the chart's type scale. Options: size, anchor, fill, bold, rotate (degrees about x,y), dy, cls. */ +function label(x, y, str, { size = 11, anchor = 'start', fill = MUTED, bold = false, rotate = 0, dy = null, full = null, cls = null } = {}) { + const t = sv('text', { + class: cls, x: r2(x), y: r2(y), text: str, dy, 'text-anchor': anchor, + transform: rotate ? `rotate(${rotate} ${r2(x)} ${r2(y)})` : null, + style: `fill:${fill};font-family:inherit;font-size:${size}px;font-weight:${bold ? 700 : 400}`, + }); + if (full) t.append(sv('title', { text: full })); // clipped text keeps its full form as a tooltip + return t; +} + +/** + * Legend as an SVG <g>. items: [{ label, color, dash?, mark? }]; mark is + * 'line' (default when `dash` is given), 'box' or 'dot'. `horizontal` lays + * items in rows that wrap at `maxWidth`; otherwise one item per row. The + * group's measured size is in g.vizSize = { width, height }. + */ +export function legend(items, { horizontal = true, maxWidth = Infinity } = {}) { + const g = sv('g', { class: 'viz-legend' }); + const rowH = 18, gap = 18; + let x = 0, y = 0, width = 0; + for (const it of items) { + const mark = it.mark ?? (it.dash != null ? 'line' : 'box'); + const sw = mark === 'line' ? 22 : 12; + const w = sw + 6 + textWidth(it.label); + if (horizontal && x > 0 && x + w > maxWidth) { x = 0; y += rowH; } + const cy = y + rowH / 2; + if (mark === 'line') { + g.append(sv('line', { class: 'viz-legend-swatch', x1: x, x2: x + sw, y1: cy, y2: cy, 'stroke-dasharray': it.dash === true ? '6 4' : it.dash, style: `stroke:${it.color};stroke-width:2.5` })); + } else if (mark === 'dot') { + g.append(sv('circle', { class: 'viz-legend-swatch', cx: x + 6, cy, r: 5, style: `fill:${it.color}` })); + } else { + g.append(sv('rect', { class: 'viz-legend-swatch', x, y: cy - 6, width: 12, height: 12, rx: 2, style: `fill:${it.color}` })); + } + g.append(label(x + sw + 6, cy, it.label, { fill: INK, dy: '0.32em' })); + width = Math.max(width, x + w); + if (horizontal) x += w + gap; else y += rowH; + } + g.vizSize = { width, height: items.length ? y + (horizontal ? rowH : 0) : 0 }; + return g; +} + +/** Tick positions and text for a scale; `values` overrides the scale's own ticks. */ +function tickList(scale, values, format) { + const band = scale.type === 'band'; + return (values || scale.ticks()).map((v) => { + const p = scale(v); + return { pos: band ? p + scale.bandwidth / 2 : p, text: format ? format(v) : band ? String(v) : formatNumber(v), value: v }; + }).filter((t) => Number.isFinite(t.pos)); +} + +/** Band labels are turned 45 degrees when they would collide horizontally. */ +const bandCrowded = (scale) => scale.type === 'band' && Math.max(0, ...scale.domain.map((l) => textWidth(clip(l)))) > scale.step - 6; + +/** + * Axes as an SVG <g>: gridlines, axis lines, ticks, tick labels and axis + * titles (classes viz-axes, viz-grid, viz-axis, viz-zero, viz-tick, + * viz-tick-label, viz-axis-label). xScale and yScale are scales from this + * file whose ranges already match the plot area, that is [left, width-right] + * for x and [height-bottom, top] for y. xTicks/yTicks override the tick + * values; xFormat/yFormat turn a tick value into text. margin defaults to + * { top: 12, right: 16, bottom: 44, left: 56 }. Add this group to the SVG + * before the marks so gridlines sit underneath them. + */ +export function axes({ xScale, yScale, xLabel, yLabel, xTicks, yTicks, xFormat, yFormat, width, height, margin = {} }) { + const m = { top: 12, right: 16, bottom: 44, left: 56, ...margin }; + const L = m.left, R = width - m.right, T = m.top, B = height - m.bottom; + const xt = tickList(xScale, xTicks, xFormat), yt = tickList(yScale, yTicks, yFormat); + const g = sv('g', { class: 'viz-axes' }); + const line = (cls, x1, y1, x2, y2, style) => sv('line', { class: cls, x1: r2(x1), y1: r2(y1), x2: r2(x2), y2: r2(y2), style }); + const gridStyle = 'stroke:var(--viz-grid);stroke-width:1', axisStyle = `stroke:${MUTED};stroke-width:1`; + + if (xScale.type !== 'band') for (const t of xt) g.append(line('viz-grid viz-grid-x', t.pos, T, t.pos, B, gridStyle)); + if (yScale.type !== 'band') for (const t of yt) g.append(line('viz-grid viz-grid-y', L, t.pos, R, t.pos, gridStyle)); + g.append(line('viz-axis viz-axis-x', L, B, R, B, axisStyle), line('viz-axis viz-axis-y', L, T, L, B, axisStyle)); + // A heavier line at zero, so bars and curves that go negative have a visible baseline. + const [xd0, xd1] = xScale.domain, [yd0, yd1] = yScale.domain; + if (xScale.type === 'linear' && xd0 * xd1 < 0) g.append(line('viz-zero viz-zero-x', xScale(0), T, xScale(0), B, `stroke:${INK};stroke-width:1`)); + if (yScale.type === 'linear' && yd0 * yd1 < 0) g.append(line('viz-zero viz-zero-y', L, yScale(0), R, yScale(0), `stroke:${INK};stroke-width:1`)); + + const turned = bandCrowded(xScale); + for (const t of xt) { + g.append(line('viz-tick', t.pos, B, t.pos, B + 4, axisStyle)); + const opts = turned ? { anchor: 'end', rotate: -45 } : { anchor: 'middle' }; + g.append(label(t.pos, B + 16, clip(t.text), { ...opts, cls: 'viz-tick-label', full: clip(t.text) === t.text ? null : t.text })); + } + for (const t of yt) { + g.append(line('viz-tick', L - 4, t.pos, L, t.pos, axisStyle)); + g.append(label(L - 8, t.pos, clip(t.text, 24), { anchor: 'end', dy: '0.32em', cls: 'viz-tick-label', full: clip(t.text, 24) === t.text ? null : t.text })); + } + if (xLabel) g.append(label((L + R) / 2, height - 8, xLabel, { size: 12, anchor: 'middle', fill: INK, cls: 'viz-axis-label' })); + if (yLabel) g.append(label(14, (T + B) / 2, yLabel, { size: 12, anchor: 'middle', fill: INK, rotate: -90, dy: '0.32em', cls: 'viz-axis-label' })); + return g; +} + +/** + * An HTML <figure class="viz-figure"> around a chart (or any node) with an + * optional <figcaption> and a <p class="viz-source"> ("Source: " is added if + * the text does not already start with it). For hand-built widgets; Quarto's + * own figure markup does not need it. + */ +export function figure(node, { caption, source } = {}) { + return el('figure', { class: 'viz-figure', style: { margin: '1.5rem 0' } }, [ + node, + caption && el('figcaption', { class: 'viz-caption', text: caption, style: { marginTop: '0.5rem', fontSize: '0.9rem' } }), + source && el('p', { class: 'viz-source', text: /^sources?\b/i.test(source) ? source : `Source: ${source}`, style: { margin: '0.25rem 0 0', fontSize: '0.8rem', color: 'var(--viz-muted)' } }), + ]); +} + +/* ------------------------------------------------------------------ */ +/* Chart plumbing */ +/* ------------------------------------------------------------------ */ + +function need(o, fn) { + if (!o || typeof o.alt !== 'string' || !o.alt.trim()) { + throw new Error(`${fn}: pass \`alt\`, a sentence or two describing what the figure shows. Every chart needs one for screen-reader users.`); + } +} + +/** The <svg>: viewBox-sized (responsive), role="img", <title>/<desc> wired with aria-labelledby/-describedby. */ +function makeSvg(o, W, H, kids) { + const id = ++uid, tid = `viz-title-${id}`, did = `viz-desc-${id}`; + return sv('svg', { + class: 'viz-chart', viewBox: `0 0 ${W} ${H}`, role: 'img', + 'aria-labelledby': o.title ? tid : did, 'aria-describedby': o.title ? did : null, + style: `display:block;width:100%;height:auto;max-width:${W}px;font-family:inherit`, + }, [ + o.title && sv('title', { id: tid, text: o.title }), + sv('desc', { id: did, text: o.alt }), + o.title && label(12, 20, o.title, { size: 14, bold: true, fill: INK }), + ...kids, + ]); +} + +/** Space around the plot for the title, legend, axis titles and tick labels. */ +function plotMargin(o, W, { legendH = 0, yTexts = [], bandX = null }) { + const m = { + top: (o.title ? 30 : 10) + (legendH ? legendH + 8 : 0), right: 16, bottom: 30 + (o.xLabel ? 20 : 0), + left: (o.yLabel ? 20 : 0) + Math.max(16, ...yTexts.map((t) => textWidth(clip(t, 24)))) + 14, + }; + if (bandX) { + const step = (W - m.left - m.right) / (bandX.length + 0.2); + const wide = Math.max(0, ...bandX.map((l) => textWidth(clip(l)))); + if (wide > step - 6) { + m.bottom += Math.ceil(wide * 0.71) + 2; + m.left = Math.max(m.left, Math.ceil(wide * 0.71 - step / 2) + 8); + } + } + return m; +} + +/** A legend group placed under the title, or null when it is off or redundant (one series). */ +function chartLegend(o, W, items) { + if (o.legend === false || items.length < 2) return null; + return legend(items, { maxWidth: W - 32 }); +} + +/** Domain and ticks for a numeric axis. `explicit` fixes the domain; log axes snap to whole decades. */ +function numericAxis(values, type, explicit, count) { + if (type === 'log') { + const pos = values.filter((v) => isNum(v) && v > 0); + let [lo, hi] = extent(pos) || [1, 10]; + if (lo === hi) { lo /= 10; hi *= 10; } + const a = Math.floor(Math.log10(lo) + 1e-9); + let b = Math.ceil(Math.log10(hi) - 1e-9); + if (b === a) b++; + const domain = explicit || [Number(`1e${a}`), Number(`1e${b}`)]; + return { domain, ticks: logTicks(Math.min(...domain), Math.max(...domain)) }; + } + const [lo, hi] = extent(values) || [0, 1]; + const t = niceTicks(...(explicit || [lo, hi]), count); + const domain = explicit || [t[0], t[t.length - 1]]; + const dl = Math.min(...domain), dh = Math.max(...domain), eps = (dh - dl) * 1e-9; + return { domain, ticks: t.filter((v) => v >= dl - eps && v <= dh + eps) }; +} + +const makeScale = (type, domain, range) => (type === 'log' ? logScale(domain, range) : linearScale(domain, range)); +const tickCounts = (W, H) => ({ x: clamp(Math.round((W - 100) / 80), 2, 10), y: clamp(Math.round((H - 90) / 45) + 1, 2, 8) }); + +function clipPath(L, T, R, B) { + const id = `viz-clip-${++uid}`, pad = 8; // room for a marker sitting on the axis + return { id, node: sv('clipPath', { id }, [sv('rect', { x: L - pad, y: T - pad, width: R - L + 2 * pad, height: B - T + 2 * pad })]) }; +} + +const xyValues = (series, key) => series.flatMap((s) => s[key]); + +/** Shared setup for line and scatter charts: axes, margins, legend and scales from x/y value lists. */ +function cartesian(fn, o, series, xs, ys, legendItems) { + need(o, fn); + const W = o.width ?? 640, H = o.height ?? 360; + const xType = o.xScale ?? 'linear', yType = o.yScale ?? 'linear'; + const n = tickCounts(W, H); + const xa = numericAxis(xs, xType, o.xDomain, n.x), ya = numericAxis(ys, yType, o.yDomain, n.y); + const lg = chartLegend(o, W, legendItems); + const m = plotMargin(o, W, { legendH: lg ? lg.vizSize.height : 0, yTexts: ya.ticks.map((v) => formatNumber(v)) }); + const L = m.left, R = W - m.right, T = m.top, B = H - m.bottom; + const sx = makeScale(xType, xa.domain, [L, R]), sy = makeScale(yType, ya.domain, [B, T]); + const ax = axes({ xScale: sx, yScale: sy, xLabel: o.xLabel, yLabel: o.yLabel, xTicks: xa.ticks, yTicks: ya.ticks, width: W, height: H, margin: m }); + const cp = clipPath(L, T, R, B); + if (lg) lg.setAttribute('transform', `translate(${L} ${o.title ? 28 : 8})`); + return { W, H, sx, sy, cp, lg, ax }; +} + +function checkSeries(fn, series) { + if (!Array.isArray(series) || series.length === 0) throw new Error(`${fn}: pass \`series\`, a non-empty array`); +} + +/* ------------------------------------------------------------------ */ +/* Charts */ +/* ------------------------------------------------------------------ */ + +/** + * Line chart. Options: series [{ name, x, y, dash? }] (x defaults to 0, 1, + * 2 ...; `dash` is a stroke-dasharray such as '6 4', or true), xLabel, yLabel, + * title, alt (required), width (640, also the maximum display width), height + * (360), xScale / yScale ('linear' | 'log'), xDomain / yDomain ([lo, hi]; + * default: nice bounds of the data), legend (true; shown for 2+ series), + * points (false; draw a dot at every sample). Non-numeric values, and + * values <= 0 on a log axis, leave a gap in the line. + */ +export function lineChart(o) { + need(o, 'lineChart'); + checkSeries('lineChart', o.series); + const series = o.series.map((s, i) => { + const y = s.y ?? [], x = s.x ?? y.map((_, k) => k); + if (x.length !== y.length) throw new Error(`lineChart: series "${s.name ?? i}" has ${x.length} x values but ${y.length} y values`); + return { ...s, x, y }; + }); + const cols = palette(series.length); + const items = series.map((s, i) => ({ label: s.name ?? `Series ${i + 1}`, color: cols[i], dash: s.dash ?? null, mark: 'line' })); + const c = cartesian('lineChart', o, series, xyValues(series, 'x'), xyValues(series, 'y'), items); + const marks = series.map((s, i) => { + let d = '', pen = false; + const pts = []; + s.x.forEach((xv, k) => { + const px = isNum(xv) && isNum(s.y[k]) ? c.sx(xv) : NaN, py = isNum(s.y[k]) ? c.sy(s.y[k]) : NaN; + if (!Number.isFinite(px) || !Number.isFinite(py)) { pen = false; return; } + d += `${pen ? 'L' : 'M'}${r2(px)} ${r2(py)}`; + pen = true; + pts.push([px, py, xv, s.y[k]]); + }); + const style = `fill:none;stroke:${cols[i]};stroke-width:2;stroke-linejoin:round;stroke-linecap:round`; + return sv('g', { class: 'viz-series', 'data-series': items[i].label, 'clip-path': `url(#${c.cp.id})` }, [ + sv('path', { class: 'viz-line', d, 'stroke-dasharray': s.dash === true ? '6 4' : s.dash, style }, [sv('title', { text: items[i].label })]), + ...(o.points ? pts.map(([px, py, xv, yv]) => sv('circle', { + class: 'viz-point', cx: r2(px), cy: r2(py), r: 3.5, style: `fill:${cols[i]};stroke:var(--viz-bg);stroke-width:1`, + }, [sv('title', { text: `${items[i].label}: (${formatNumber(xv, { digits: 4 })}, ${formatNumber(yv, { digits: 4 })})` })])) : []), + ]); + }); + return makeSvg(o, c.W, c.H, [c.cp.node, c.ax, ...marks, c.lg]); +} + +/** + * Scatter chart. Options: series [{ name, points: [{ x, y, label? }] }], plus + * the same common options as lineChart. Each point is a circle with a + * <title> tooltip ("label: (x, y)"). Points that cannot be drawn (non-numeric, + * or <= 0 on a log axis) are left out. + */ +export function scatterChart(o) { + need(o, 'scatterChart'); + checkSeries('scatterChart', o.series); + const series = o.series.map((s) => ({ ...s, points: s.points ?? [] })); + const cols = palette(series.length); + const items = series.map((s, i) => ({ label: s.name ?? `Series ${i + 1}`, color: cols[i], mark: 'dot' })); + const pts = series.map((s) => s.points); + const c = cartesian('scatterChart', o, series, pts.flat().map((p) => p.x), pts.flat().map((p) => p.y), items); + const marks = series.map((s, i) => sv('g', { class: 'viz-series', 'data-series': items[i].label, 'clip-path': `url(#${c.cp.id})` }, + s.points.flatMap((p) => { + const px = isNum(p.x) ? c.sx(p.x) : NaN, py = isNum(p.y) ? c.sy(p.y) : NaN; + if (!Number.isFinite(px) || !Number.isFinite(py)) return []; + const tip = `${p.label ?? items[i].label}: (${formatNumber(p.x, { digits: 4 })}, ${formatNumber(p.y, { digits: 4 })})`; + return [sv('circle', { + class: 'viz-point', cx: r2(px), cy: r2(py), r: 4.5, + style: `fill:${cols[i]};fill-opacity:0.8;stroke:var(--viz-bg);stroke-width:1`, + }, [sv('title', { text: tip })])]; + }))); + return makeSvg(o, c.W, c.H, [c.cp.node, c.ax, ...marks, c.lg]); +} + +/** + * Bar chart. Options: labels (category names, unique), either series [{ name, + * values }] or values (a single unnamed series), horizontal (false), stacked + * (false; otherwise several series are grouped side by side), xLabel, yLabel, + * title, alt (required), width, height, legend, and yDomain (xDomain when + * horizontal) for the value axis. The value axis always includes 0 and bars + * grow from it, so negative values point the other way. Bars need a linear + * value axis; asking for 'log' throws. + */ +export function barChart(o) { + need(o, 'barChart'); + const labels = (o.labels ?? []).map(String); + const series = o.series ?? [{ name: '', values: o.values ?? [] }]; + series.forEach((s, i) => { + if (!Array.isArray(s.values) || s.values.length !== labels.length) { + throw new Error(`barChart: series "${s.name ?? i}" needs one value per label (${labels.length}), got ${s.values ? s.values.length : 'none'}`); + } + }); + const horizontal = !!o.horizontal, stacked = !!o.stacked; + if ((horizontal ? o.xScale : o.yScale) === 'log') throw new Error('barChart: bars need a linear value axis (a log axis has no zero to grow from)'); + const W = o.width ?? 640, H = o.height ?? 360; + // Each bar is the interval [from, to] on the value axis. + const up = labels.map(() => 0), down = labels.map(() => 0); + const bars = series.map((s) => s.values.map((v, i) => { + v = isNum(v) ? v : 0; + if (!stacked) return [0, v]; + const from = v >= 0 ? up[i] : down[i]; + if (v >= 0) up[i] += v; else down[i] += v; + return [from, from + v]; + })); + const all = bars.flat(2).concat(0); + const n = tickCounts(W, H); + const va = numericAxis(all, 'linear', horizontal ? o.xDomain : o.yDomain, horizontal ? n.x : n.y); + const cols = palette(series.length); + const lg = chartLegend(o, W, series.map((s, i) => ({ label: s.name || `Series ${i + 1}`, color: cols[i], mark: 'box' }))); + const m = plotMargin(o, W, { + legendH: lg ? lg.vizSize.height : 0, + yTexts: horizontal ? labels : va.ticks.map((v) => formatNumber(v)), + bandX: horizontal ? null : labels, + }); + const L = m.left, R = W - m.right, T = m.top, B = H - m.bottom; + const cat = horizontal ? bandScale(labels, [T, B]) : bandScale(labels, [L, R]); + const val = horizontal ? linearScale(va.domain, [L, R]) : linearScale(va.domain, [B, T]); + const ax = axes({ + xScale: horizontal ? val : cat, yScale: horizontal ? cat : val, xLabel: o.xLabel, yLabel: o.yLabel, + xTicks: horizontal ? va.ticks : undefined, yTicks: horizontal ? undefined : va.ticks, width: W, height: H, margin: m, + }); + const sub = bandScale(series.map((_, k) => k), [0, cat.bandwidth], series.length > 1 && !stacked ? 0.1 : 0); + const marks = series.map((s, k) => sv('g', { class: 'viz-series', 'data-series': s.name ?? '' }, s.values.map((v, i) => { + const [from, to] = bars[k][i]; + const a = val(from), b = val(to); + const start = cat(labels[i]) + (stacked ? 0 : sub(k)), thick = stacked ? cat.bandwidth : sub.bandwidth; + const box = horizontal + ? { x: Math.min(a, b), y: start, width: Math.abs(b - a), height: thick } + : { x: start, y: Math.min(a, b), width: thick, height: Math.abs(b - a) }; + const tip = `${labels[i]}${s.name ? ` (${s.name})` : ''}: ${formatNumber(v, { digits: 4 })}`; + return sv('rect', { + class: 'viz-bar', x: r2(box.x), y: r2(box.y), width: r2(box.width), height: r2(box.height), + style: `fill:${cols[k]}${stacked ? ';stroke:var(--viz-bg);stroke-width:1' : ''}`, + }, [sv('title', { text: tip })]); + }))); + if (lg) lg.setAttribute('transform', `translate(${L} ${o.title ? 28 : 8})`); + return makeSvg(o, W, H, [ax, ...marks, lg]); +} + +/** Re-run `update` when the page switches between light and dark (body.quarto-dark). */ +function watchTheme(svg, update) { + if (typeof MutationObserver === 'undefined' || !document.body) return; + const ref = new WeakRef(svg); + const obs = new MutationObserver(() => { const s = ref.deref(); if (s) update(s); else obs.disconnect(); }); + obs.observe(document.body, { attributes: true, attributeFilter: ['class'] }); +} + +const isDark = () => typeof document !== 'undefined' && !!document.body && document.body.classList.contains('quarto-dark'); + +/** + * Heatmap of a matrix. Options: matrix (array of rows), rowLabels, colLabels + * (unique; default 1, 2, 3 ...), colors ('ramp' for magnitudes, 'diverging' + * for values either side of a midpoint), domain ([lo, hi]; default the data + * range, or symmetric about 0 for 'diverging'), showValues (false; print the + * value in each cell, black or white for contrast), digits (2; significant + * digits for those values), and the common title / alt (required) / width / + * height / xLabel / yLabel. A color key is drawn on the right. Cells that are + * not numbers are drawn empty. Each cell has a <title> tooltip. + */ +export function heatmap(o) { + need(o, 'heatmap'); + const matrix = o.matrix ?? []; + const rows = matrix.length, cols = rows ? matrix[0].length : 0; + if (matrix.some((r) => r.length !== cols)) throw new Error('heatmap: every row of `matrix` must have the same length'); + const rowLabels = (o.rowLabels ?? matrix.map((_, i) => i + 1)).map(String); + const colLabels = (o.colLabels ?? Array.from({ length: cols }, (_, i) => i + 1)).map(String); + if (rowLabels.length !== rows || colLabels.length !== cols) throw new Error(`heatmap: expected ${rows} rowLabels and ${cols} colLabels`); + const diverging = o.colors === 'diverging'; + const [lo, hi] = extent(matrix.flat()) || [0, 1]; + const big = Math.max(Math.abs(lo), Math.abs(hi)); + const [d0, d1] = o.domain ?? (diverging ? [-big, big] : [lo, hi]); + const tOf = (v) => { + if (diverging) return d1 === d0 ? 0 : clamp((v - (d0 + d1) / 2) / ((d1 - d0) / 2), -1, 1); + return d1 === d0 ? 0.5 : clamp((v - d0) / (d1 - d0), 0, 1); + }; + const colorOf = diverging ? divergingColor : rampColor; + + const W = o.width ?? 640, H = o.height ?? 360; + const keyW = 78; + const m = plotMargin(o, W, { yTexts: rowLabels, bandX: colLabels }); + m.right = keyW + 16; + const L = m.left, R = W - m.right, T = m.top, B = H - m.bottom; + const xs = bandScale(colLabels, [L, R], 0), ys = bandScale(rowLabels, [T, B], 0); + const cw = (R - L) / Math.max(cols, 1), ch = (B - T) / Math.max(rows, 1); + const ax = axes({ xScale: xs, yScale: ys, xLabel: o.xLabel, yLabel: o.yLabel, width: W, height: H, margin: m }); + + const cells = [], texts = []; + matrix.forEach((row, i) => row.forEach((v, j) => { + const x = L + j * cw, y = T + i * ch, has = isNum(v); + const tip = `${rowLabels[i]}, ${colLabels[j]}: ${has ? formatNumber(v, { digits: 4 }) : 'no data'}`; + cells.push(sv('rect', { + class: 'viz-cell', x: r2(x), y: r2(y), width: r2(cw), height: r2(ch), 'data-t': has ? r2(tOf(v)) : null, + style: `fill:${has ? colorOf(tOf(v)) : 'none'};stroke:var(--viz-bg);stroke-width:1`, + }, [sv('title', { text: tip })])); + if (o.showValues && has) { + const text = formatNumber(v, { digits: o.digits ?? 2 }); + const size = clamp(Math.floor((cw - 4) / (text.length * 0.6)), 7, 11); + const t = label(x + cw / 2, y + ch / 2, text, { size, anchor: 'middle', dy: '0.32em', cls: 'viz-cell-text', fill: textColorFor(tOf(v), { diverging, dark: isDark() }) }); + t.dataset.t = String(tOf(v)); + texts.push(t); + } + })); + + // Color key: the gradient the cells are drawn from, with its end and middle values. + const gid = `viz-key-${++uid}`, kx = R + 16; + const stops = diverging ? [[0, -1], [0.5, 0], [1, 1]] : [[0, 0], [1, 1]]; + const key = sv('g', { class: 'viz-key' }, [ + sv('defs', {}, [sv('linearGradient', { id: gid, x1: 0, y1: 1, x2: 0, y2: 0 }, + stops.map(([off, t]) => sv('stop', { offset: off, style: `stop-color:${colorOf(t)}` })))]), + sv('rect', { x: kx, y: T, width: 12, height: B - T, style: `fill:url(#${gid});stroke:var(--viz-grid);stroke-width:1` }), + ...[[d0, B], [(d0 + d1) / 2, (B + T) / 2], [d1, T]].map(([v, y]) => label(kx + 18, y, formatNumber(v), { dy: '0.32em' })), + ]); + const svg = makeSvg(o, W, H, [ax, ...cells, ...texts, key]); + if (texts.length) { + watchTheme(svg, (s) => s.querySelectorAll('.viz-cell-text').forEach((t) => { + t.style.fill = textColorFor(Number(t.dataset.t), { diverging, dark: isDark() }); + })); + } + return svg; +} diff --git a/src/exercises/_helpers.ts b/src/exercises/_helpers.ts new file mode 100644 index 0000000..26947e4 --- /dev/null +++ b/src/exercises/_helpers.ts @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** Shared helpers for exercise files. */ +import { SHARPEN } from '../engines/pseudo/specs.js'; + +/** The English steps the course wrote for one of its sharpen ladders, if any. */ +export const stepsOf = (key: string): string | undefined => (SHARPEN as any)[key]?.steps || undefined; + +/** The setup block (registers, memory, stdin) of a course assembly widget, in the sandbox's syntax. */ +export function setupFromSpec(spec: any): string { + const lines: string[] = []; + for (const [r, v] of Object.entries(spec.registers || {})) { + const b = BigInt(v as any); + lines.push(`${r} = ${b > 255n ? `0x${b.toString(16)}` : b.toString()}`); + } + for (const m of spec.memory || []) { + if (m.text != null) lines.push(`memory 0x${Number(m.addr).toString(16)} ${JSON.stringify(String(m.text))}`); + else if (m.bytes) lines.push(`memory 0x${Number(m.addr).toString(16)} bytes ${Array.from(m.bytes as number[]).join(' ')}`); + else lines.push(`memory 0x${Number(m.addr).toString(16)} zero ${m.size || 0}`); + } + if (spec.stdin) lines.push(`stdin ${JSON.stringify(spec.stdin)}`); + return lines.join('\n'); +} diff --git a/src/exercises/_template.ts b/src/exercises/_template.ts new file mode 100644 index 0000000..0c7d029 --- /dev/null +++ b/src/exercises/_template.ts @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Copy this file to `<id>.ts`, fill it in, then import it in `index.ts` and + * add it to the EXERCISES list. Delete the fields you do not need. Run + * `npm test`: it parses the pseudocode, runs the call, and compares the + * answer with `expect`. + */ +import { exercise } from './types.js'; + +export default exercise({ + id: 'my-exercise', // the URL: #/bridge/p/my-exercise + title: 'My exercise', + group: 'Your own', // the home-page section; a new name makes a new section + blurb: 'One or two sentences: what the student builds and what they will notice.', + meta: { lecture: '2.1', level: 'core', tags: ['scan'], concepts: ['accumulator'], minutes: 20, page: '/programmingplus/...' }, + sources: { + // Any subset. Pseudo is derived from lingo (or matho, written in LaTeX) when absent. + lingo: `To count the G bases in T: + Start with Count at 0. + For each letter b of T: + If b is 'G', add 1 to Count. + The answer is Count. +`, + pseudo: `FUNCTION COUNT_G(T : STRING) RETURNS INTEGER + DECLARE Count : INTEGER + Count <- 0 + FOR I <- 0 TO LENGTH(T) + IF T[I] == 'G' THEN + Count <- Count + 1 + ENDIF + NEXT I + RETURN Count +ENDFUNCTION +`, + speco: `SPEC COUNT_G(T) + ENSURE result = |{ K ∈ [0, |T|) : T[K] = 'G' }| + LOOP FOR I + INVARIANT Count = |{ K ∈ [0, I) : T[K] = 'G' }| +`, + // asm: `...`, with `setup:` and `machine:` below + }, + call: 'COUNT_G("GATTACA")', + expect: '1', + // setup: 'rdi = 0x1000\nrsi = 7\nmemory 0x1000 "GATTACA"', + // machine: 'teaching', + lang: 'lingo', // which pane opens first +}); diff --git a/src/exercises/asm-branchless.ts b/src/exercises/asm-branchless.ts new file mode 100644 index 0000000..4f29304 --- /dev/null +++ b/src/exercises/asm-branchless.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { specBranchless } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'asm-branchless', + title: 'Counting without a branch', + group: 'The machine', + blurb: 'setcc turns a condition into a number; the branch predictor has nothing left to guess.', + meta: { level: 'core' }, + sources: { + asm: specBranchless.code, + }, + setup: setupFromSpec(specBranchless), + machine: 'realistic', +}); diff --git a/src/exercises/asm-cmov.ts b/src/exercises/asm-cmov.ts new file mode 100644 index 0000000..fbe4f94 --- /dev/null +++ b/src/exercises/asm-cmov.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { specConditionalMove } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'asm-cmov', + title: 'Choosing with cmov', + group: 'The machine', + blurb: 'Both values are computed; one is kept.', + meta: { level: 'core' }, + sources: { + asm: specConditionalMove.code, + }, + setup: setupFromSpec(specConditionalMove), + machine: 'realistic', +}); diff --git a/src/exercises/asm-first-instructions.ts b/src/exercises/asm-first-instructions.ts new file mode 100644 index 0000000..7ce006e --- /dev/null +++ b/src/exercises/asm-first-instructions.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { specFirstInstructions } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'asm-first-instructions', + title: 'Eight instructions, one at a time', + group: 'The machine', + blurb: 'Read the next line, say what it will change, then step.', + meta: { level: 'core' }, + sources: { + asm: specFirstInstructions.code, + }, + setup: setupFromSpec(specFirstInstructions), + machine: 'teaching', +}); diff --git a/src/exercises/asm-pack-bits.ts b/src/exercises/asm-pack-bits.ts new file mode 100644 index 0000000..34e5b99 --- /dev/null +++ b/src/exercises/asm-pack-bits.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { specPackBits } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'asm-pack-bits', + title: 'Packing bases into bits', + group: 'The machine', + blurb: 'Two bits per base: shifts and ORs.', + meta: { level: 'core' }, + sources: { + asm: specPackBits.code, + }, + setup: setupFromSpec(specPackBits), +}); diff --git a/src/exercises/asm-register-widths.ts b/src/exercises/asm-register-widths.ts new file mode 100644 index 0000000..313c773 --- /dev/null +++ b/src/exercises/asm-register-widths.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { specRegisterWidths } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'asm-register-widths', + title: 'One register, four names', + group: 'The machine', + blurb: 'A four-byte write clears the top half; a one-byte write leaves it alone.', + meta: { level: 'core' }, + sources: { + asm: specRegisterWidths.code, + }, + setup: setupFromSpec(specRegisterWidths), +}); diff --git a/src/exercises/asm-two-pointers.ts b/src/exercises/asm-two-pointers.ts new file mode 100644 index 0000000..6ac3713 --- /dev/null +++ b/src/exercises/asm-two-pointers.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { specTwoPointers } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'asm-two-pointers', + title: 'Two pointers', + group: 'The machine', + blurb: 'Reverse in place: two addresses walking toward each other.', + meta: { level: 'core' }, + sources: { + asm: specTwoPointers.code, + }, + setup: setupFromSpec(specTwoPointers), +}); diff --git a/src/exercises/asm-vector.ts b/src/exercises/asm-vector.ts new file mode 100644 index 0000000..000ada0 --- /dev/null +++ b/src/exercises/asm-vector.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { specVectorCount } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'asm-vector', + title: 'Sixteen letters at a time', + group: 'The machine', + blurb: 'pcmpeqb and pmovmskb: the SIMD count.', + meta: { level: 'core' }, + sources: { + asm: specVectorCount.code, + }, + setup: setupFromSpec(specVectorCount), + machine: 'realistic', +}); diff --git a/src/exercises/best-position.ts b/src/exercises/best-position.ts new file mode 100644 index 0000000..e822774 --- /dev/null +++ b/src/exercises/best-position.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; + +export default exercise({ + id: 'best-position', + mojoProgram: 'best-position', + title: 'The best quality score, proved', + group: 'Loop forms', + blurb: 'Find the best: the set-up takes the first value as the current best rather than inventing one.', + meta: { level: 'core', tags: ['find the best', 'proof'], minutes: 15 }, + sources: { + pseudo: EX.best.code, + }, + call: EX.best.call, + expect: "1", +}); diff --git a/src/exercises/binary-search.ts b/src/exercises/binary-search.ts new file mode 100644 index 0000000..05c44f9 --- /dev/null +++ b/src/exercises/binary-search.ts @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; +import { specBinarySearch } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'binary-search', + mojoProgram: 'binary-search', + title: 'Binary search', + group: 'Beyond the scan', + blurb: 'A WHILE loop whose DECREASES measure is the width of the range still to search.', + meta: { level: 'advanced', tags: ['DECREASES', 'WHILE'], minutes: 30 }, + sources: { + pseudo: EX.binary.code, + asm: specBinarySearch.code, + }, + call: EX.binary.call, + expect: "TRUE", + setup: setupFromSpec(specBinarySearch), +}); diff --git a/src/exercises/count-base-proved.ts b/src/exercises/count-base-proved.ts new file mode 100644 index 0000000..ddeca60 --- /dev/null +++ b/src/exercises/count-base-proved.ts @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; +import { specCountBase } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'count-base-proved', + mojoProgram: 'count-base-proved', + title: 'Count one base, proved', + group: 'The linear scan', + blurb: 'The same scan with its specification and loop invariant. The speco pane holds the claims; the proof view shows each one proved.', + meta: { lecture: '1.1', level: 'core', tags: ['scan', 'invariant', 'proof'], minutes: 30, page: '/programmingplus/reference/pseudocode/proofs/' }, + sources: { + pseudo: EX.countProved.code, + asm: specCountBase.code, + }, + call: EX.countProved.call, + expect: "3", + setup: setupFromSpec(specCountBase), + lang: 'speco', +}); diff --git a/src/exercises/count-base.ts b/src/exercises/count-base.ts new file mode 100644 index 0000000..46fd820 --- /dev/null +++ b/src/exercises/count-base.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; +import { specCountBase } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'count-base', + title: 'Count one base', + group: 'The linear scan', + blurb: 'The counting lesson: a scan that accumulates one number. Every layer the course has, from English steps to the bytes the machine runs.', + meta: { lecture: '1.1', level: 'intro', tags: ['scan', 'accumulator'], concepts: ['half-open range', 'accumulator'], minutes: 20, page: '/programmingplus/linear-scan/counting-bases/' }, + sources: { + lingo: `To count the B bases in T: + Start with Count at 0. + For each letter c of T: + If c is B, add 1 to Count. + The answer is Count. +`, + pseudo: EX.count.code, + asm: specCountBase.code, + }, + mojoProgram: 'count-base', + call: EX.count.call, + expect: "3", + setup: setupFromSpec(specCountBase), + machine: 'teaching', +}); diff --git a/src/exercises/demo.ts b/src/exercises/demo.ts new file mode 100644 index 0000000..eac25e5 --- /dev/null +++ b/src/exercises/demo.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; + +export default exercise({ + id: 'demo', + // The Mojo is library/mojo/local/demo.mojo, and the x86-64 pane starts from the compiler's listing for it. + // Edit the .mojo file, then run `pixi run play demo` (docs/mojo-corpus.md, "The fast loop"). + mojoProgram: 'demo', + title: 'Demo: aligning a sequencing read', + group: 'Your own', + blurb: 'A read that lost two letters is aligned to the reference with Smith-Waterman and affine gaps: three tables filled cell by cell, then a traceback that switches tables where a gap opened. Scored with linear gaps instead, the same read gets a different story. Step through the fill and the traceback, watch the three tables freed the moment the traceback is done, and run the recurrence itself on the model machine.', + meta: { level: 'core' }, + sources: { + pseudo: '', + lingo: '', + speco: '', + }, + lang: 'mojo', + view: 'trace', +}); diff --git a/src/exercises/edit-distance.ts b/src/exercises/edit-distance.ts new file mode 100644 index 0000000..d3aed8d --- /dev/null +++ b/src/exercises/edit-distance.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; + +export default exercise({ + id: 'edit-distance', + mojoProgram: 'edit-distance', + title: 'Edit distance', + group: 'Beyond the scan', + blurb: 'A table filled cell by cell: the shape of every alignment algorithm.', + meta: { level: 'advanced', tags: ['table', 'dynamic programming'], minutes: 40 }, + sources: { + pseudo: EX.edit.code, + }, + call: EX.edit.call, + expect: "3", +}); diff --git a/src/exercises/first-unknown.ts b/src/exercises/first-unknown.ts new file mode 100644 index 0000000..49557aa --- /dev/null +++ b/src/exercises/first-unknown.ts @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; + +export default exercise({ + id: 'first-unknown', + mojoProgram: 'first-unknown', + title: 'The first unknown base', + group: 'Loop forms', + blurb: 'A WHILE scan with a guard on the left of AND. Swap the two sides and watch the checker lose the proof.', + meta: { level: 'core', tags: ['WHILE', 'guard order'], minutes: 15 }, + sources: { + lingo: `To find the first unknown base in T: + Start with i at 0. + Repeat while i is less than the length of T and the letter of T at i is not 'N': + Add 1 to i. + The answer is i. +`, + pseudo: EX.firstUnknown.code, + }, + call: EX.firstUnknown.call, + expect: "3", +}); diff --git a/src/exercises/five-bugs.ts b/src/exercises/five-bugs.ts new file mode 100644 index 0000000..93bf2e0 --- /dev/null +++ b/src/exercises/five-bugs.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; + +export default exercise({ + id: 'five-bugs', + mojoProgram: 'five-bugs', + title: 'Five mistakes in one function', + group: 'Bugs to find', + blurb: 'The coach finds each one and links it to the guide.', + meta: { level: 'intro', tags: ['bug hunt'], minutes: 15 }, + sources: { + pseudo: EX.fiveBugs.code, + }, + call: EX.fiveBugs.call, +}); diff --git a/src/exercises/gc-fraction.ts b/src/exercises/gc-fraction.ts new file mode 100644 index 0000000..7ce796f --- /dev/null +++ b/src/exercises/gc-fraction.ts @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; +import { stepsOf } from './_helpers.js'; + +export default exercise({ + id: 'gc-fraction', + mojoProgram: 'gc-fraction', + title: 'GC fraction', + group: 'Loop forms', + blurb: 'FLOAT division and a REQUIRE on the empty sequence.', + meta: { level: 'intro', tags: ['FLOAT', 'REQUIRE'], minutes: 10 }, + sources: { + lingo: stepsOf('countGC') as string | undefined, + pseudo: EX.gc.code, + }, + call: EX.gc.call, +}); diff --git a/src/exercises/hamming.ts b/src/exercises/hamming.ts new file mode 100644 index 0000000..ee635c4 --- /dev/null +++ b/src/exercises/hamming.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; + +export default exercise({ + id: 'hamming', + mojoProgram: 'hamming', + title: 'Hamming distance, proved', + group: 'Loop forms', + blurb: 'Two sequences walked in step, with a REQUIRE that they have the same length. Delete it and see which claim opens.', + meta: { level: 'core', tags: ['two sequences', 'proof'], minutes: 15 }, + sources: { + pseudo: EX.hamming.code, + }, + call: EX.hamming.call, + expect: "2", +}); diff --git a/src/exercises/index.ts b/src/exercises/index.ts new file mode 100644 index 0000000..60f7b49 --- /dev/null +++ b/src/exercises/index.ts @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Every exercise, in the order the home page lists them. To add one: copy + * `_template.ts` to `<id>.ts`, fill it in, import it here, and add it to the + * list. `npm test` validates all of them. + */ +import type { Exercise } from './types.js'; +import countBase from './count-base.js'; +import countBaseProved from './count-base-proved.js'; +import match from './match.js'; +import matchProved from './match-proved.js'; +import matchOffByOne from './match-off-by-one.js'; +import matchReset from './match-reset.js'; +import fiveBugs from './five-bugs.js'; +import firstUnknown from './first-unknown.js'; +import gcFraction from './gc-fraction.js'; +import hamming from './hamming.js'; +import bestPosition from './best-position.js'; +import tally from './tally.js'; +import kmers from './kmers.js'; +import revcomp from './revcomp.js'; +import binarySearch from './binary-search.js'; +import editDistance from './edit-distance.js'; +import translateCodons from './translate-codons.js'; +import asmFirstInstructions from './asm-first-instructions.js'; +import asmRegisterWidths from './asm-register-widths.js'; +import asmBranchless from './asm-branchless.js'; +import asmCmov from './asm-cmov.js'; +import asmVector from './asm-vector.js'; +import asmTwoPointers from './asm-two-pointers.js'; +import asmPackBits from './asm-pack-bits.js'; +import demo from './demo.js'; + +export const EXERCISES: Exercise[] = [ + countBase, + countBaseProved, + match, + matchProved, + matchOffByOne, + matchReset, + fiveBugs, + firstUnknown, + gcFraction, + hamming, + bestPosition, + tally, + kmers, + revcomp, + binarySearch, + editDistance, + translateCodons, + asmFirstInstructions, + asmRegisterWidths, + asmBranchless, + asmCmov, + asmVector, + asmTwoPointers, + asmPackBits, + demo, +]; + +export type { Exercise } from './types.js'; diff --git a/src/exercises/kmers.ts b/src/exercises/kmers.ts new file mode 100644 index 0000000..f150a9e --- /dev/null +++ b/src/exercises/kmers.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; + +export default exercise({ + id: 'kmers', + mojoProgram: 'kmers', + title: 'Count every k-mer', + group: 'Collections', + blurb: 'SLICE and a HASHMAP, the shape of every k-mer counter in the course.', + meta: { level: 'core', tags: ['HASHMAP', 'SLICE'], minutes: 20 }, + sources: { + pseudo: EX.kmers.code, + }, + call: EX.kmers.call, +}); diff --git a/src/exercises/match-off-by-one.ts b/src/exercises/match-off-by-one.ts new file mode 100644 index 0000000..0b14486 --- /dev/null +++ b/src/exercises/match-off-by-one.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; + +export default exercise({ + id: 'match-off-by-one', + mojoProgram: 'match-off-by-one', + title: 'One bound is wrong', + group: 'Bugs to find', + blurb: 'MATCH with its outer bound moved. Run it, watch where it reads past the end, then prove why.', + meta: { level: 'intro', tags: ['bug', 'off-by-one'], minutes: 10 }, + sources: { + pseudo: EX.matchBound.code, + }, + call: EX.matchBound.call, +}); diff --git a/src/exercises/match-proved.ts b/src/exercises/match-proved.ts new file mode 100644 index 0000000..68a200b --- /dev/null +++ b/src/exercises/match-proved.ts @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; +import { specNaiveSearch } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'match-proved', + mojoProgram: 'match-proved', + title: 'Find a pattern, proved', + group: 'The linear scan', + blurb: 'MATCH with two invariants, one per loop, proved against the specification from "Specifying the problem".', + meta: { lecture: '1.3', level: 'advanced', tags: ['invariant', 'proof'], minutes: 40 }, + sources: { + pseudo: EX.matchProved.code, + asm: specNaiveSearch.code, + }, + call: EX.matchProved.call, + expect: "[1]", + setup: setupFromSpec(specNaiveSearch), + lang: 'speco', +}); diff --git a/src/exercises/match-reset.ts b/src/exercises/match-reset.ts new file mode 100644 index 0000000..4fcb678 --- /dev/null +++ b/src/exercises/match-reset.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; + +export default exercise({ + id: 'match-reset', + mojoProgram: 'match-reset', + title: 'A flag reset in the wrong loop', + group: 'Bugs to find', + blurb: 'The accumulator-placement mistake from the guide. The trace table shows the first answer right and every later one wrong.', + meta: { level: 'intro', tags: ['bug', 'accumulator'], minutes: 10 }, + sources: { + pseudo: EX.matchReset.code, + }, + call: EX.matchReset.call, + expect: "[]", +}); diff --git a/src/exercises/match.ts b/src/exercises/match.ts new file mode 100644 index 0000000..e307cae --- /dev/null +++ b/src/exercises/match.ts @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; +import { STEPS_EXAMPLES } from '../engines/pseudo/specs.js'; +import { specNaiveSearch } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'match', + mojoProgram: 'match', + title: 'Find a pattern', + group: 'The linear scan', + blurb: 'Every start of P in T. Two nested loops, an early exit, and the bound N - M + 1 that every student gets wrong once.', + meta: { lecture: '1.3', level: 'intro', tags: ['nested loop', 'early exit'], concepts: ['N - M + 1'], minutes: 30, page: '/programmingplus/linear-scan/finding-a-pattern/' }, + sources: { + lingo: (STEPS_EXAMPLES as any).match.steps, + pseudo: EX.match.code, + asm: specNaiveSearch.code, + }, + call: EX.match.call, + expect: "[1]", + setup: setupFromSpec(specNaiveSearch), +}); diff --git a/src/exercises/revcomp.ts b/src/exercises/revcomp.ts new file mode 100644 index 0000000..07aa8ae --- /dev/null +++ b/src/exercises/revcomp.ts @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; +import { stepsOf } from './_helpers.js'; +import { specTableLookup } from '../engines/asm86/specs.js'; +import { setupFromSpec } from './_helpers.js'; + +export default exercise({ + id: 'revcomp', + mojoProgram: 'revcomp', + title: 'Reverse complement', + group: 'Collections', + blurb: 'A CASE table and a loop that counts down.', + meta: { level: 'core', tags: ['CASE', 'countdown'], minutes: 20 }, + sources: { + lingo: stepsOf('revcomp') as string | undefined, + pseudo: EX.revcomp.code, + asm: specTableLookup.code, + }, + call: EX.revcomp.call, + expect: '"TGTAATC"', + setup: setupFromSpec(specTableLookup), +}); diff --git a/src/exercises/tally.ts b/src/exercises/tally.ts new file mode 100644 index 0000000..91d3c93 --- /dev/null +++ b/src/exercises/tally.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; + +export default exercise({ + id: 'tally', + mojoProgram: 'tally', + title: 'Tally the bases', + group: 'Collections', + blurb: 'The HASHMAP counting pattern.', + meta: { level: 'core', tags: ['HASHMAP'], minutes: 15 }, + sources: { + pseudo: EX.tally.code, + }, + call: EX.tally.call, +}); diff --git a/src/exercises/translate-codons.ts b/src/exercises/translate-codons.ts new file mode 100644 index 0000000..4e97e56 --- /dev/null +++ b/src/exercises/translate-codons.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import { exercise } from './types.js'; +import { EXAMPLES } from '../engines/pseudo/specs.js'; +const EX: any = EXAMPLES; + +export default exercise({ + id: 'translate-codons', + mojoProgram: 'translate-codons', + title: 'Translate codons', + group: 'Beyond the scan', + blurb: 'STEP 3, a SET of stop codons, and a HASHMAP codon table.', + meta: { level: 'core', tags: ['SET', 'HASHMAP', 'STEP'], minutes: 20 }, + sources: { + pseudo: EX.translate.code, + }, + call: EX.translate.call, + expect: '"MAK"', +}); diff --git a/src/exercises/types.ts b/src/exercises/types.ts new file mode 100644 index 0000000..3861da2 --- /dev/null +++ b/src/exercises/types.ts @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * One exercise = one file in this folder. Add a file, export an `Exercise`, + * and list it in `index.ts`; the catalog, the home page, the problem menu, + * the embeds, and the tests pick it up. `npm test` checks every exercise: + * its pseudocode parses, its call runs, and its expected answer matches. + */ +import type { LangId, ViewId } from '../common/types.js'; +import type { AreaId } from '../common/types.js'; + +export interface Exercise { + /** URL id: lowercase, hyphens. */ + id: string; + title: string; + /** The section of the home page it files under. */ + group: string; + blurb: string; + /** Free-form metadata, shown as chips and available to the lessons. */ + meta?: { + lecture?: string; + level?: 'intro' | 'core' | 'advanced'; + tags?: string[]; + concepts?: string[]; + /** Minutes a student should expect to spend. */ + minutes?: number; + /** Where this exercise is taught; a URL or a page path on the course site. */ + page?: string; + }; + /** Whichever layers the exercise provides. Missing ones are derived (pseudo from lingo or matho, speco from the pseudo's contracts) or hidden. */ + sources: Partial<Record<LangId, string>>; + /** The call to run, such as COUNT_BASE('A', "GATTACA"); the default call is used when absent. */ + call?: string | null; + /** What the call should produce, in the notation: used by the tests. */ + expect?: string; + /** Assembly: the setup block and the machine preset. */ + setup?: string; + machine?: string; + /** Which pane opens first. */ + lang?: LangId; + /** The area it opens in (bridge, dev, hw). Absent: Hardware for asm-* problems, Developer for Mojo-only ones, Bridge for the rest. */ + area?: AreaId; + /** Which tab beside the editor opens first (trace, table, flow, data, life, proof, tests); trace when absent. */ + view?: ViewId; + /** + * The exercise's Mojo, by name: a program in library/mojo/local/<name>.mojo, which + * `corpus.py verify` checks on the pinned compiler. The Mojo pane shows it, + * badged as verified only if godbolt's answer passed its own expectation. + */ + mojoProgram?: string; +} + +/** A small helper so an exercise file reads as data. */ +export function exercise(e: Exercise): Exercise { return e; } diff --git a/src/library/generated.ts b/src/library/generated.ts new file mode 100644 index 0000000..39d2f97 --- /dev/null +++ b/src/library/generated.ts @@ -0,0 +1,1803 @@ +// Generated by scripts/library.mjs emit from library/ (pplus-library). Do not edit; run pixi run library-emit. +import type { LibraryModule } from './types.js'; + +export const LIBRARY_MODULES: LibraryModule[] = [ + { + "name": "align", + "title": "Sequence alignment", + "summary": "Scores for aligning two sequences: gap models, letter scores, and global alignment by dynamic programming.", + "routines": [ + { + "id": "align.AffineGapPenalty", + "module": "align", + "name": "AffineGapPenalty", + "dir": "modules/align/affine-gap", + "summary": "The score of a gap of Length positions when opening a gap scores Open and each further position scores Extend.", + "tags": [ + "alignment", + "gap model" + ], + "sources": [ + { + "title": "Gotoh, An improved algorithm for matching biological sequences, J. Mol. Biol. 162 (1982)", + "url": "https://doi.org/10.1016/0022-2836(82)90398-9" + } + ], + "source": "// AffineGapPenalty: the score of a gap of Length positions under an affine\n// gap model (Gotoh, 1982): Open for the first position and Extend for each\n// one after it. When Open is more negative than Extend, one long gap scores\n// better than several short gaps of the same total length. With\n// Open == Extend it is the linear model.\nFUNCTION AffineGapPenalty(Length : INTEGER, Open : INTEGER, Extend : INTEGER) RETURNS INTEGER\n REQUIRE Length >= 1\n ENSURE RESULT == Open + (Length - 1) * Extend\n RETURN Open + (Length - 1) * Extend\nENDFUNCTION\n", + "cases": [ + { + "call": "AffineGapPenalty(1, -10, -1)", + "result": "-10" + }, + { + "call": "AffineGapPenalty(5, -10, -1)", + "result": "-14" + }, + { + "call": "AffineGapPenalty(3, -2, -2)", + "result": "-6" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same formula.", + "source": "def affine_gap_penalty(length, open_, extend):\n return open_ + (length - 1) * extend\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same formula in Mojo.", + "source": "def affine_gap_penalty(length: Int, open_: Int, extend: Int) -> Int:\n return open_ + (length - 1) * extend\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 3 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 3 case(s) (exactly)" + } + } + } + ], + "proof": { + "lang": "dafny", + "file": "proof/affine_gap.dfy", + "note": "The formula equals the score built position by position, and with Open == Extend it equals the linear model.", + "source": "method AffineGapPenalty(length: int, open: int, extend: int) returns (result: int)\n requires length >= 1\n ensures result == open + (length - 1) * extend\n{\n result := open + (length - 1) * extend;\n}\n\n// Position by position: the first position opens the gap, every later one extends it.\nmethod AffineByPositions(length: int, open: int, extend: int) returns (score: int)\n requires length >= 1\n ensures score == open + (length - 1) * extend\n{\n score := open;\n var i := 1;\n while i < length\n invariant 1 <= i <= length\n invariant score == open + (i - 1) * extend\n {\n score := score + extend;\n i := i + 1;\n }\n}\n\n// With the same score for opening and extending, the affine model is the linear one.\nlemma AffineIsLinearWhenEqual(length: int, gap: int)\n requires length >= 1\n ensures gap + (length - 1) * gap == length * gap\n{\n}\n", + "check": { + "status": "pass", + "detail": "Dafny 4.11.0: 3 verified, 0 errors" + } + }, + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION AffineGapPenalty" + }, + "pseudo": { + "status": "pass", + "detail": "3 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 1 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 201 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + }, + { + "id": "align.LinearGapPenalty", + "module": "align", + "name": "LinearGapPenalty", + "dir": "modules/align/linear-gap", + "summary": "The score of a gap of Length positions when every position of a gap scores Gap.", + "tags": [ + "alignment", + "gap model" + ], + "sources": [ + { + "title": "Durbin, Eddy, Krogh and Mitchison, Biological Sequence Analysis (Cambridge, 1998), chapter 2, Pairwise alignment", + "url": "https://doi.org/10.1017/CBO9780511790492.003" + } + ], + "source": "// LinearGapPenalty: the score of a gap of Length positions under a linear gap\n// model, where every position of a gap scores Gap (usually negative).\nFUNCTION LinearGapPenalty(Length : INTEGER, Gap : INTEGER) RETURNS INTEGER\n REQUIRE Length >= 0\n ENSURE RESULT == Length * Gap\n RETURN Length * Gap\nENDFUNCTION\n", + "cases": [ + { + "call": "LinearGapPenalty(3, -2)", + "result": "-6" + }, + { + "call": "LinearGapPenalty(0, -2)", + "result": "0" + }, + { + "call": "LinearGapPenalty(10, -1)", + "result": "-10" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same product.", + "source": "def linear_gap_penalty(length, gap):\n return length * gap\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same product in Mojo.", + "source": "def linear_gap_penalty(length: Int, gap: Int) -> Int:\n return length * gap\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 3 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 3 case(s) (exactly)" + } + } + } + ], + "proof": { + "lang": "dafny", + "file": "proof/linear_gap.dfy", + "note": "The product equals the score built position by position: each of the Length positions adds Gap.", + "source": "// A gap of `length` positions under a linear model: each position scores `gap`.\nmethod LinearGapPenalty(length: int, gap: int) returns (result: int)\n requires length >= 0\n ensures result == length * gap\n{\n result := length * gap;\n}\n\n// The same score, built one position at a time: the product is the model, not a shortcut.\nmethod LinearGapByPositions(length: int, gap: int) returns (score: int)\n requires length >= 0\n ensures score == length * gap\n{\n score := 0;\n var i := 0;\n while i < length\n invariant 0 <= i <= length\n invariant score == i * gap\n {\n score := score + gap;\n i := i + 1;\n }\n}\n", + "check": { + "status": "pass", + "detail": "Dafny 4.11.0: 2 verified, 0 errors" + } + }, + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION LinearGapPenalty" + }, + "pseudo": { + "status": "pass", + "detail": "3 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 1 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 123 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + }, + { + "id": "align.MatchScore", + "module": "align", + "name": "MatchScore", + "dir": "modules/align/match-score", + "summary": "The score of aligning letter A with letter B: Match when they are the same, Mismatch otherwise.", + "tags": [ + "alignment", + "scoring" + ], + "sources": [ + { + "title": "Durbin, Eddy, Krogh and Mitchison, Biological Sequence Analysis (Cambridge, 1998), chapter 2, Pairwise alignment", + "url": "https://doi.org/10.1017/CBO9780511790492.003" + } + ], + "source": "// MatchScore: the score of aligning letter A with letter B: Match when they\n// are the same letter, Mismatch otherwise. The simplest substitution score;\n// a substitution matrix such as BLOSUM62 replaces it for proteins.\nFUNCTION MatchScore(A : CHAR, B : CHAR, Match : INTEGER, Mismatch : INTEGER) RETURNS INTEGER\n ENSURE (A == B AND RESULT == Match) OR (A != B AND RESULT == Mismatch)\n IF A == B THEN\n RETURN Match\n ENDIF\n RETURN Mismatch\nENDFUNCTION\n", + "cases": [ + { + "call": "MatchScore('A', 'A', 1, -1)", + "result": "1" + }, + { + "call": "MatchScore('A', 'G', 1, -1)", + "result": "-1" + }, + { + "call": "MatchScore('T', 'T', 5, -4)", + "result": "5" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same test.", + "source": "def match_score(a, b, match, mismatch):\n return match if a == b else mismatch\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same test in Mojo; a letter is a String of one character.", + "source": "def match_score(a: String, b: String, match_: Int, mismatch: Int) -> Int:\n if a == b:\n return match_\n return mismatch\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 3 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 3 case(s) (exactly)" + } + } + } + ], + "proof": { + "lang": "dafny", + "file": "proof/match_score.dfy", + "note": "The score is Match exactly when the letters agree.", + "source": "method MatchScore(a: char, b: char, match_: int, mismatch: int) returns (r: int)\n ensures (a == b && r == match_) || (a != b && r == mismatch)\n{\n if a == b { r := match_; } else { r := mismatch; }\n}\n", + "check": { + "status": "pass", + "detail": "Dafny 4.11.0: 1 verified, 0 errors" + } + }, + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION MatchScore" + }, + "pseudo": { + "status": "pass", + "detail": "3 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 2 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 285 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + }, + { + "id": "align.NeedlemanWunschScore", + "module": "align", + "name": "NeedlemanWunschScore", + "dir": "modules/align/needleman-wunsch-score", + "summary": "The best score of a global alignment of two sequences with a linear gap model, by dynamic programming.", + "tags": [ + "alignment", + "dynamic programming" + ], + "sources": [ + { + "title": "Needleman and Wunsch, A general method applicable to the search for similarities in the amino acid sequence of two proteins, J. Mol. Biol. 48 (1970)", + "url": "https://doi.org/10.1016/0022-2836(70)90057-4" + } + ], + "source": "// NeedlemanWunschScore: the best score of a global alignment of S and T\n// (Needleman and Wunsch, 1970) with a linear gap model, by dynamic\n// programming. D[I, J] is the best score of aligning the first I letters of\n// S with the first J letters of T; each cell takes the best of three moves:\n// align S[I - 1] with T[J - 1], or put a gap in one sequence or the other.\nUSE align.MatchScore\nUSE align.LinearGapPenalty\nFUNCTION NeedlemanWunschScore(S : STRING, T : STRING, Match : INTEGER, Mismatch : INTEGER, Gap : INTEGER) RETURNS INTEGER\n DECLARE N : INTEGER <- LENGTH(S)\n DECLARE M : INTEGER <- LENGTH(T)\n DECLARE D : ARRAY[N + 1, M + 1] OF INTEGER\n FOR I <- 0 TO N + 1\n D[I, 0] <- LinearGapPenalty(I, Gap)\n NEXT I\n FOR J <- 1 TO M + 1\n D[0, J] <- LinearGapPenalty(J, Gap)\n NEXT J\n FOR I <- 1 TO N + 1\n FOR J <- 1 TO M + 1\n DECLARE Diagonal : INTEGER <- D[I - 1, J - 1] + MatchScore(S[I - 1], T[J - 1], Match, Mismatch)\n DECLARE Down : INTEGER <- D[I - 1, J] + Gap\n DECLARE Across : INTEGER <- D[I, J - 1] + Gap\n D[I, J] <- MAX(Diagonal, Down, Across)\n NEXT J\n NEXT I\n RETURN D[N, M]\nENDFUNCTION\n", + "cases": [ + { + "call": "NeedlemanWunschScore(\"GATTACA\", \"GCATGCU\", 1, -1, -1)", + "result": "0" + }, + { + "call": "NeedlemanWunschScore(\"ACGT\", \"ACGT\", 1, -1, -2)", + "result": "4" + }, + { + "call": "NeedlemanWunschScore(\"\", \"ACG\", 1, -1, -2)", + "result": "-6" + }, + { + "call": "NeedlemanWunschScore(\"AAAA\", \"TTTT\", 2, -3, -1)", + "result": "-8" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same table, filled row by row.", + "source": "def needleman_wunsch_score(s, t, match, mismatch, gap):\n n, m = len(s), len(t)\n d = [[0] * (m + 1) for _ in range(n + 1)]\n for i in range(n + 1):\n d[i][0] = i * gap\n for j in range(1, m + 1):\n d[0][j] = j * gap\n for i in range(1, n + 1):\n for j in range(1, m + 1):\n diagonal = d[i - 1][j - 1] + (match if s[i - 1] == t[j - 1] else mismatch)\n d[i][j] = max(diagonal, d[i - 1][j] + gap, d[i][j - 1] + gap)\n return d[n][m]\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) (exactly)" + } + } + }, + { + "lang": "python", + "level": "two-rows", + "file": "ref/python/two-rows.py", + "note": "Only the previous row is kept: memory grows with the length of T alone, the first step toward Hirschberg's linear-space alignment.", + "source": "def needleman_wunsch_score(s, t, match, mismatch, gap):\n m = len(t)\n prev = [j * gap for j in range(m + 1)]\n for i in range(1, len(s) + 1):\n cur = [i * gap] + [0] * m\n for j in range(1, m + 1):\n diagonal = prev[j - 1] + (match if s[i - 1] == t[j - 1] else mismatch)\n cur[j] = max(diagonal, prev[j] + gap, cur[j - 1] + gap)\n prev = cur\n return prev[m]\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same table in Mojo, as a list of rows.", + "source": "def needleman_wunsch_score(s: String, t: String, match_: Int, mismatch: Int, gap: Int) -> Int:\n var n = s.count_codepoints()\n var m = t.count_codepoints()\n var d = List[List[Int]]()\n for i in range(n + 1):\n var row = List[Int]()\n for j in range(m + 1):\n row.append(0)\n d.append(row^)\n for i in range(n + 1):\n d[i][0] = i * gap\n for j in range(1, m + 1):\n d[0][j] = j * gap\n for i in range(1, n + 1):\n for j in range(1, m + 1):\n var score = mismatch\n if String(s[codepoint=i - 1]) == String(t[codepoint=j - 1]):\n score = match_\n var best = d[i - 1][j - 1] + score\n if d[i - 1][j] + gap > best:\n best = d[i - 1][j] + gap\n if d[i][j - 1] + gap > best:\n best = d[i][j - 1] + gap\n d[i][j] = best\n return d[n][m]\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 4 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 4 case(s) (exactly)" + } + } + } + ], + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION NeedlemanWunschScore, using align.MatchScore, align.LinearGapPenalty" + }, + "pseudo": { + "status": "pass", + "detail": "4 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 11 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 300 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + } + ] + }, + { + "name": "md", + "title": "Molecular simulation", + "summary": "Pair potentials, forces and periodic boxes, the inner loop of molecular dynamics.", + "routines": [ + { + "id": "md.LennardJones", + "module": "md", + "name": "LennardJones", + "dir": "modules/md/lennard-jones", + "summary": "The Lennard-Jones pair potential at distance R, with well depth Epsilon and zero-crossing distance Sigma.", + "tags": [ + "molecular dynamics", + "pair potential" + ], + "sources": [ + { + "title": "Jones, On the determination of molecular fields. II. From the equation of state of a gas, Proc. R. Soc. A 106 (1924)", + "url": "https://doi.org/10.1098/rspa.1924.0082" + }, + { + "title": "Lennard-Jones, Cohesion, Proc. Phys. Soc. 43 (1931): the 12-6 form", + "url": "https://doi.org/10.1088/0959-5309/43/5/301" + }, + { + "title": "Allen and Tildesley, Computer Simulation of Liquids, 2nd edition (Oxford, 2017), section 1.3.2, Atomic systems" + } + ], + "source": "// LennardJones: the Lennard-Jones pair potential at distance R,\n// V(R) = 4 Epsilon ((Sigma / R)^12 - (Sigma / R)^6). Epsilon is the depth of\n// the well, reached at R = 2^(1/6) Sigma, and V is 0 at R = Sigma. The powers\n// are built by multiplication from (Sigma / R)^2, as simulation codes do,\n// instead of ^, which rounds a little differently.\nFUNCTION LennardJones(R : FLOAT, Epsilon : FLOAT, Sigma : FLOAT) RETURNS FLOAT\n REQUIRE R > 0.0\n DECLARE S2 : FLOAT <- (Sigma / R) * (Sigma / R)\n DECLARE S6 : FLOAT <- S2 * S2 * S2\n RETURN 4.0 * Epsilon * (S6 * S6 - S6)\nENDFUNCTION\n", + "cases": [ + { + "call": "LennardJones(1.0, 1.0, 1.0)", + "result": "0.0" + }, + { + "call": "LennardJones(1.122462048309373, 1.0, 1.0)", + "result": "-1.0" + }, + { + "call": "LennardJones(2.5, 0.238, 3.4)", + "result": "32.09189752256677" + }, + { + "call": "LennardJones(0.9, 1.0, 1.0)", + "result": "6.6361189532529234" + } + ], + "references": [ + { + "lang": "python", + "level": "multiply", + "file": "ref/python/multiply.py", + "note": "The same multiplications.", + "source": "def lennard_jones(r, epsilon, sigma):\n s2 = (sigma / r) * (sigma / r)\n s6 = s2 * s2 * s2\n return 4.0 * epsilon * (s6 * s6 - s6)\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) (exactly)" + } + } + }, + { + "lang": "python", + "level": "powers", + "file": "ref/python/powers.py", + "note": "The textbook formula with ** 12 and ** 6: the same function, rounded differently, held to agree to 12 digits on these cases. Near R = Sigma, where V is close to 0, the two roundings differ by far more relative to V (1.9e-8 at R = 1.000000001, Sigma = 1).", + "rel": 1e-12, + "source": "def lennard_jones(r, epsilon, sigma):\n return 4.0 * epsilon * ((sigma / r) ** 12 - (sigma / r) ** 6)\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) (within rel 1e-12, abs 0)" + } + } + }, + { + "lang": "c", + "level": "multiply", + "file": "ref/c/multiply.c", + "note": "The same multiplications in C.", + "source": "double lennard_jones(double r, double epsilon, double sigma) {\n double s2 = (sigma / r) * (sigma / r);\n double s6 = s2 * s2 * s2;\n return 4.0 * epsilon * (s6 * s6 - s6);\n}\n", + "checks": { + "cc -O0": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) (exactly)" + }, + "cc -O2": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "multiply", + "file": "ref/mojo/multiply.mojo", + "note": "The same multiplications in Mojo.", + "source": "def lennard_jones(r: Float64, epsilon: Float64, sigma: Float64) -> Float64:\n var s2 = (sigma / r) * (sigma / r)\n var s6 = s2 * s2 * s2\n return 4.0 * epsilon * (s6 * s6 - s6)\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 4 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 4 case(s) (exactly)" + } + } + } + ], + "proof": { + "lang": "dafny", + "file": "proof/lennard_jones.dfy", + "note": "Over real numbers the multiplications compute the textbook formula exactly, and V(Sigma) = 0.", + "source": "function Pow(x: real, n: nat): real\n{\n if n == 0 then 1.0 else x * Pow(x, n - 1)\n}\n\nfunction LJ(r: real, epsilon: real, sigma: real): real\n requires r > 0.0\n{\n 4.0 * epsilon * (Pow(sigma / r, 12) - Pow(sigma / r, 6))\n}\n\nlemma PowAdd(x: real, a: nat, b: nat)\n ensures Pow(x, a + b) == Pow(x, a) * Pow(x, b)\n{\n if a > 0 {\n PowAdd(x, a - 1, b);\n assert Pow(x, a + b) == x * Pow(x, a - 1 + b);\n }\n}\n\nlemma Pow2(x: real)\n ensures Pow(x, 2) == x * x\n{\n assert Pow(x, 1) == x;\n}\n\nlemma Pow6(s: real)\n ensures Pow(s, 6) == (s * s) * (s * s) * (s * s)\n{\n Pow2(s);\n PowAdd(s, 2, 2);\n PowAdd(s, 4, 2);\n}\n\nlemma Pow12(s: real)\n ensures Pow(s, 12) == Pow(s, 6) * Pow(s, 6)\n{\n PowAdd(s, 6, 6);\n}\n\nmethod LennardJones(r: real, epsilon: real, sigma: real) returns (v: real)\n requires r > 0.0\n ensures v == LJ(r, epsilon, sigma)\n{\n var s := sigma / r;\n var s2 := s * s;\n var s6 := s2 * s2 * s2;\n Pow6(s);\n Pow12(s);\n v := 4.0 * epsilon * (s6 * s6 - s6);\n}\n\n// The potential crosses zero at R = Sigma.\nlemma ZeroAtSigma(epsilon: real, sigma: real)\n requires sigma > 0.0\n ensures LJ(sigma, epsilon, sigma) == 0.0\n{\n assert sigma / sigma == 1.0;\n}\n", + "check": { + "status": "pass", + "detail": "Dafny 4.11.0: 14 verified, 0 errors" + } + }, + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION LennardJones" + }, + "pseudo": { + "status": "pass", + "detail": "4 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 2 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 252 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + }, + { + "id": "md.LennardJonesForce", + "module": "md", + "name": "LennardJonesForce", + "dir": "modules/md/lennard-jones-force", + "summary": "The force between two particles at distance R under the Lennard-Jones potential: positive pushes them apart.", + "tags": [ + "molecular dynamics", + "force" + ], + "sources": [ + { + "title": "Allen and Tildesley, Computer Simulation of Liquids, 2nd edition (Oxford, 2017), section 1.3.2, Atomic systems: the potential whose derivative this is" + }, + { + "title": "Lennard-Jones, Cohesion, Proc. Phys. Soc. 43 (1931): the 12-6 form", + "url": "https://doi.org/10.1088/0959-5309/43/5/301" + } + ], + "source": "// LennardJonesForce: the force between two particles at distance R under\n// the Lennard-Jones potential, F(R) = -dV/dR\n// = 24 Epsilon / R (2 (Sigma / R)^12 - (Sigma / R)^6).\n// Positive pushes the particles apart; it is 0 at the bottom of the well.\nFUNCTION LennardJonesForce(R : FLOAT, Epsilon : FLOAT, Sigma : FLOAT) RETURNS FLOAT\n REQUIRE R > 0.0\n DECLARE S2 : FLOAT <- (Sigma / R) * (Sigma / R)\n DECLARE S6 : FLOAT <- S2 * S2 * S2\n RETURN 24.0 * Epsilon * (2.0 * S6 * S6 - S6) / R\nENDFUNCTION\n", + "cases": [ + { + "call": "LennardJonesForce(1.0, 1.0, 1.0)", + "result": "24.0" + }, + { + "call": "LennardJonesForce(1.122462048309373, 1.0, 1.0)", + "result": "-2.3738310467722616e-15" + }, + { + "call": "LennardJonesForce(2.5, 0.238, 3.4)", + "result": "168.49822326347663" + } + ], + "references": [ + { + "lang": "python", + "level": "multiply", + "file": "ref/python/multiply.py", + "note": "The same multiplications.", + "source": "def lennard_jones_force(r, epsilon, sigma):\n s2 = (sigma / r) * (sigma / r)\n s6 = s2 * s2 * s2\n return 24.0 * epsilon * (2.0 * s6 * s6 - s6) / r\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "multiply", + "file": "ref/mojo/multiply.mojo", + "note": "The same multiplications in Mojo.", + "source": "def lennard_jones_force(r: Float64, epsilon: Float64, sigma: Float64) -> Float64:\n var s2 = (sigma / r) * (sigma / r)\n var s6 = s2 * s2 * s2\n return 24.0 * epsilon * (2.0 * s6 * s6 - s6) / r\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 3 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 3 case(s) (exactly)" + } + } + } + ], + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION LennardJonesForce" + }, + "pseudo": { + "status": "pass", + "detail": "3 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 3 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 252 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + }, + { + "id": "md.MinimumImage", + "module": "md", + "name": "MinimumImage", + "dir": "modules/md/minimum-image", + "summary": "A displacement brought to its nearest periodic image in a box of length Box: the minimum image convention. Over real numbers the result lies in [-Box / 2, Box / 2); in FLOAT arithmetic it can land a rounding error outside that range.", + "tags": [ + "molecular dynamics", + "periodic boundary conditions" + ], + "sources": [ + { + "title": "Allen and Tildesley, Computer Simulation of Liquids, 2nd edition (Oxford, 2017), sections 1.6.2, Periodic boundary conditions, and 1.6.3, Potential truncation: the minimum image convention" + } + ], + "source": "// MinimumImage: the displacement Dx brought to its nearest periodic image in\n// a box of length Box, the minimum image convention: Dx minus the whole\n// number of box lengths that brings it into [-Box / 2, Box / 2). FLOOR of\n// Dx / Box + 0.5 rounds halves upward, so exactly half a box maps to -Box / 2.\n// That range holds over exact numbers (the Dafny proof). In FLOAT arithmetic\n// Dx / Box and the product are rounded, so the result can land a rounding\n// error outside it (MinimumImage(1.35, 0.1) is -0.050000000000000044), and\n// within a rounding error of half a box it can give the farther of the two\n// closest images (MinimumImage(0.49999999999999994, 1.0) is -0.5).\nFUNCTION MinimumImage(Dx : FLOAT, Box : FLOAT) RETURNS FLOAT\n REQUIRE Box > 0.0\n RETURN Dx - Box * FLOOR(Dx / Box + 0.5)\nENDFUNCTION\n", + "cases": [ + { + "call": "MinimumImage(0.75, 1.0)", + "result": "-0.25" + }, + { + "call": "MinimumImage(-0.75, 1.0)", + "result": "0.25" + }, + { + "call": "MinimumImage(2.3, 1.0)", + "result": "0.2999999999999998" + }, + { + "call": "MinimumImage(0.5, 1.0)", + "result": "-0.5" + }, + { + "call": "MinimumImage(-12.0, 10.0)", + "result": "-2.0" + }, + { + "call": "MinimumImage(1.35, 0.1)", + "result": "-0.050000000000000044" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same formula with math.floor.", + "source": "import math\n\n\ndef minimum_image(dx, box):\n return dx - box * math.floor(dx / box + 0.5)\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 6 case(s) (exactly)" + } + } + }, + { + "lang": "c", + "level": "plain", + "file": "ref/c/plain.c", + "note": "The same formula with floor from math.h.", + "source": "#include <math.h>\n\ndouble minimum_image(double dx, double box) {\n return dx - box * floor(dx / box + 0.5);\n}\n", + "checks": { + "cc -O0": { + "status": "pass", + "detail": "agrees with the pseudo on 6 case(s) (exactly)" + }, + "cc -O2": { + "status": "pass", + "detail": "agrees with the pseudo on 6 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same formula in Mojo, with floor from std.math.", + "source": "from std.math import floor\n\n\ndef minimum_image(dx: Float64, box: Float64) -> Float64:\n return dx - box * floor(dx / box + 0.5)\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 6 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 6 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 6 case(s) (exactly)" + } + } + } + ], + "proof": { + "lang": "dafny", + "file": "proof/minimum_image.dfy", + "note": "Over real numbers the result lies in [-Box / 2, Box / 2) and differs from Dx by a whole number of box lengths.", + "source": "// MinimumImage, over real numbers: the image lies in [-box / 2, box / 2) and is a whole number of box lengths from dx.\nlemma ScaledBounds(g: real, box: real)\n requires box > 0.0\n requires -0.5 <= g < 0.5\n ensures -box / 2.0 <= g * box < box / 2.0\n{\n assert (g + 0.5) * box >= 0.0;\n assert (0.5 - g) * box > 0.0;\n}\n\nmethod MinimumImage(dx: real, box: real) returns (r: real, k: int)\n requires box > 0.0\n ensures r == dx - box * (k as real)\n ensures -box / 2.0 <= r < box / 2.0\n{\n var q := dx / box + 0.5;\n k := q.Floor;\n assert (k as real) <= q < (k as real) + 1.0;\n var g := q - (k as real) - 0.5;\n assert -0.5 <= g < 0.5;\n assert dx == (q - 0.5) * box;\n r := dx - box * (k as real);\n assert r == g * box;\n ScaledBounds(g, box);\n}\n", + "check": { + "status": "pass", + "detail": "Dafny 4.11.0: 4 verified, 0 errors" + } + }, + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION MinimumImage" + }, + "pseudo": { + "status": "pass", + "detail": "6 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 1 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 241 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + }, + { + "id": "md.PairDistance", + "module": "md", + "name": "PairDistance", + "dir": "modules/md/pair-distance", + "summary": "The distance between two points in a cubic periodic box, each coordinate difference taken by the minimum image convention.", + "tags": [ + "molecular dynamics", + "periodic boundary conditions" + ], + "sources": [ + { + "title": "Allen and Tildesley, Computer Simulation of Liquids, 2nd edition (Oxford, 2017), sections 1.6.2, Periodic boundary conditions, and 1.6.3, Potential truncation: the minimum image convention" + } + ], + "source": "// PairDistance: the distance between two points in a cubic periodic box of\n// side Box, each coordinate difference taken by the minimum image\n// convention. HYPOT adds the squares without overflowing on the way.\nUSE md.MinimumImage\nFUNCTION PairDistance(A : LIST OF FLOAT, B : LIST OF FLOAT, Box : FLOAT) RETURNS FLOAT\n REQUIRE LENGTH(A) == 3 AND LENGTH(B) == 3 AND Box > 0.0\n DECLARE Dx : FLOAT <- MinimumImage(A[0] - B[0], Box)\n DECLARE Dy : FLOAT <- MinimumImage(A[1] - B[1], Box)\n DECLARE Dz : FLOAT <- MinimumImage(A[2] - B[2], Box)\n RETURN HYPOT(HYPOT(Dx, Dy), Dz)\nENDFUNCTION\n", + "cases": [ + { + "call": "PairDistance([0.1, 0.1, 0.1], [0.9, 0.9, 0.9], 1.0)", + "result": "0.34641016151377535" + }, + { + "call": "PairDistance([0.0, 0.0, 0.0], [3.0, 4.0, 0.0], 20.0)", + "result": "5.0" + }, + { + "call": "PairDistance([1.0, 2.0, 3.0], [1.0, 2.0, 3.0], 5.0)", + "result": "0.0" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same steps, with math.hypot of all three differences at once.", + "rel": 1e-15, + "source": "import math\n\n\ndef pair_distance(a, b, box):\n d = [x - y for x, y in zip(a, b)]\n d = [v - box * math.floor(v / box + 0.5) for v in d]\n return math.hypot(*d)\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) (within rel 1e-15, abs 0)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same steps in Mojo, with the square root of the sum of squares.", + "rel": 1e-15, + "source": "from std.math import floor, sqrt\n\n\ndef pair_distance(a: List[Float64], b: List[Float64], box: Float64) -> Float64:\n var total: Float64 = 0.0\n for i in range(3):\n var d = a[i] - b[i]\n d = d - box * floor(d / box + 0.5)\n total = total + d * d\n return sqrt(total)\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) in core's interpreter (within rel 1e-15, abs 0)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 3 case(s) (within rel 1e-15, abs 0)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 3 case(s) (within rel 1e-15, abs 0)" + } + } + } + ], + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION PairDistance, using md.MinimumImage" + }, + "pseudo": { + "status": "pass", + "detail": "3 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 9 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 4 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints the pseudo's results to within 1.6e-16 of each value (the compiler's rounding and std.math), and in core's interpreter" + } + } + } + ] + }, + { + "name": "numerics", + "title": "Numerics", + "summary": "Summing, interpolating and integrating with FLOAT values, and, for summing, what rounding does.", + "routines": [ + { + "id": "numerics.KahanSum", + "module": "numerics", + "name": "KahanSum", + "dir": "modules/numerics/kahan-sum", + "summary": "The sum of a list of FLOAT values by compensated summation, whose error bound, to first order, does not grow with the length of the list.", + "tags": [ + "summation", + "rounding", + "precision" + ], + "sources": [ + { + "title": "Kahan, Pracniques: further remarks on reducing truncation errors, CACM 8(1), 1965", + "url": "https://doi.org/10.1145/363707.363723" + }, + { + "title": "Higham, Accuracy and Stability of Numerical Algorithms, 2nd edition (SIAM, 2002), section 4.3, Compensated summation" + } + ], + "source": "// KahanSum: the sum of a list of FLOAT values by compensated summation\n// (Kahan, 1965). C holds the low-order part the last addition lost, and the\n// next addition puts it back, so the error bound, about twice the unit\n// roundoff times the sum of the absolute values (Higham, section 4.3), does\n// not grow with the length of the list, to first order. It is relative to\n// that sum of absolute values, not to the result: KahanSum([1.0e100, 1.0,\n// -1.0e100]) is 0.0. In exact arithmetic C is always 0 and this is the\n// plain sum: the compensation exists only because of rounding. A compiler\n// allowed to reorder FLOAT arithmetic (-ffast-math) can simplify C away.\nFUNCTION KahanSum(X : LIST OF FLOAT) RETURNS FLOAT\n DECLARE Total : FLOAT <- 0.0\n DECLARE C : FLOAT <- 0.0\n FOR I <- 0 TO LENGTH(X)\n DECLARE Y : FLOAT <- X[I] - C\n DECLARE T : FLOAT <- Total + Y\n C <- (T - Total) - Y\n Total <- T\n NEXT I\n RETURN Total\nENDFUNCTION\n", + "cases": [ + { + "call": "KahanSum([1.0, 2.0, 3.0])", + "result": "6.0" + }, + { + "call": "KahanSum([1.0, 1.0e-16, 1.0e-16, 1.0e-16, 1.0e-16, 1.0e-16, 1.0e-16, 1.0e-16, 1.0e-16, 1.0e-16, 1.0e-16])", + "result": "1.000000000000001" + }, + { + "call": "KahanSum([0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1])", + "result": "1.0" + }, + { + "call": "KahanSum([])", + "result": "0.0" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same compensated loop.", + "source": "def kahan_sum(x):\n total = 0.0\n c = 0.0\n for v in x:\n y = v - c\n t = total + y\n c = (t - total) - y\n total = t\n return total\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) (exactly)" + } + } + }, + { + "lang": "c", + "level": "plain", + "file": "ref/c/plain.c", + "note": "The same loop in C. It survives -O2; it would not survive -ffast-math, which lets the compiler treat (t - total) - y as 0.", + "source": "double kahan_sum(const double *x, long n) {\n double total = 0.0, c = 0.0;\n for (long i = 0; i < n; i++) {\n double y = x[i] - c;\n double t = total + y;\n c = (t - total) - y;\n total = t;\n }\n return total;\n}\n", + "checks": { + "cc -O0": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) (exactly)" + }, + "cc -O2": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same loop in Mojo.", + "source": "def kahan_sum(x: List[Float64]) -> Float64:\n var total: Float64 = 0.0\n var c: Float64 = 0.0\n for i in range(len(x)):\n var y = x[i] - c\n var t = total + y\n c = (t - total) - y\n total = t\n return total\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 4 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 4 case(s) (exactly)" + } + } + } + ], + "proof": { + "lang": "dafny", + "file": "proof/kahan_sum.dfy", + "note": "Over real numbers the compensation C stays 0 and the result is exactly the sum: what Kahan summation corrects is rounding, and nothing else.", + "source": "function Sum(x: seq<real>): real\n{\n if |x| == 0 then 0.0 else Sum(x[..|x| - 1]) + x[|x| - 1]\n}\n\n// In exact arithmetic Kahan summation is the plain sum, and its compensation is always 0.\nmethod KahanSum(x: seq<real>) returns (total: real)\n ensures total == Sum(x)\n{\n total := 0.0;\n var c := 0.0;\n var i := 0;\n while i < |x|\n invariant 0 <= i <= |x|\n invariant total == Sum(x[..i])\n invariant c == 0.0\n {\n assert x[..i + 1][..i] == x[..i];\n var y := x[i] - c;\n var t := total + y;\n c := (t - total) - y;\n total := t;\n i := i + 1;\n }\n assert x[..|x|] == x;\n}\n", + "check": { + "status": "pass", + "detail": "Dafny 4.11.0: 2 verified, 0 errors" + } + }, + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION KahanSum" + }, + "pseudo": { + "status": "pass", + "detail": "4 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 1 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 263 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + }, + { + "id": "numerics.LinearInterp", + "module": "numerics", + "name": "LinearInterp", + "dir": "modules/numerics/linear-interp", + "summary": "The value at X on the straight line through (X0, Y0) and (X1, Y1).", + "tags": [ + "interpolation" + ], + "sources": [ + { + "title": "NIST Digital Library of Mathematical Functions, section 3.3(ii), linear interpolation (equation 3.3.14)", + "url": "https://dlmf.nist.gov/3.3.ii" + } + ], + "source": "// LinearInterp: the value at X on the straight line through (X0, Y0) and\n// (X1, Y1). X may lie outside [X0, X1]: that is extrapolation, and the line\n// carries on. The REQUIRE names the difference the division divides by.\nFUNCTION LinearInterp(X0 : FLOAT, Y0 : FLOAT, X1 : FLOAT, Y1 : FLOAT, X : FLOAT) RETURNS FLOAT\n REQUIRE X1 - X0 != 0.0\n RETURN Y0 + (Y1 - Y0) * (X - X0) / (X1 - X0)\nENDFUNCTION\n", + "cases": [ + { + "call": "LinearInterp(0.0, 0.0, 10.0, 5.0, 4.0)", + "result": "2.0" + }, + { + "call": "LinearInterp(1.0, 2.0, 3.0, 2.0, 2.5)", + "result": "2.0" + }, + { + "call": "LinearInterp(0.0, 1.0, 1.0, 3.0, 2.0)", + "result": "5.0" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same formula.", + "source": "def linear_interp(x0, y0, x1, y1, x):\n return y0 + (y1 - y0) * (x - x0) / (x1 - x0)\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same formula in Mojo.", + "source": "def linear_interp(x0: Float64, y0: Float64, x1: Float64, y1: Float64, x: Float64) -> Float64:\n return y0 + (y1 - y0) * (x - x0) / (x1 - x0)\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 3 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 3 case(s) (exactly)" + } + } + } + ], + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION LinearInterp" + }, + "pseudo": { + "status": "pass", + "detail": "3 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 1 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 300 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + }, + { + "id": "numerics.Trapezoid", + "module": "numerics", + "name": "Trapezoid", + "dir": "modules/numerics/trapezoid", + "summary": "The integral of samples Y taken H apart, by the trapezoidal rule.", + "tags": [ + "integration", + "quadrature" + ], + "sources": [ + { + "title": "NIST Digital Library of Mathematical Functions, section 3.5(i), the composite trapezoidal rule (equation 3.5.2)", + "url": "https://dlmf.nist.gov/3.5.i" + } + ], + "source": "// Trapezoid: the integral of samples Y taken H apart, by the composite\n// trapezoidal rule: the ends count half, every inside sample counts whole.\n// The error shrinks with H^2 for a smooth function.\nFUNCTION Trapezoid(Y : LIST OF FLOAT, H : FLOAT) RETURNS FLOAT\n REQUIRE LENGTH(Y) >= 2\n DECLARE Total : FLOAT <- (Y[0] + Y[LENGTH(Y) - 1]) / 2.0\n FOR I <- 1 TO LENGTH(Y) - 1\n Total <- Total + Y[I]\n NEXT I\n RETURN Total * H\nENDFUNCTION\n", + "cases": [ + { + "call": "Trapezoid([0.0, 1.0, 4.0, 9.0, 16.0], 1.0)", + "result": "22.0" + }, + { + "call": "Trapezoid([1.0, 1.0], 0.5)", + "result": "0.5" + }, + { + "call": "Trapezoid([0.0, 0.25, 1.0], 0.5)", + "result": "0.375" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same sums.", + "source": "def trapezoid(y, h):\n total = (y[0] + y[len(y) - 1]) / 2.0\n for i in range(1, len(y) - 1):\n total += y[i]\n return total * h\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same sums in Mojo.", + "source": "def trapezoid(y: List[Float64], h: Float64) -> Float64:\n var total: Float64 = (y[0] + y[len(y) - 1]) / 2.0\n for i in range(1, len(y) - 1):\n total = total + y[i]\n return total * h\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 3 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 3 case(s) (exactly)" + } + } + } + ], + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION Trapezoid" + }, + "pseudo": { + "status": "pass", + "detail": "3 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 4 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 227 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + } + ] + }, + { + "name": "seq", + "title": "Sequences", + "summary": "Counting and comparing the letters of biological sequences.", + "routines": [ + { + "id": "seq.GCContent", + "module": "seq", + "name": "GCContent", + "dir": "modules/seq/gc-content", + "summary": "The fraction of the letters of a sequence that are G or C.", + "tags": [ + "sequences", + "counting" + ], + "sources": [ + { + "title": "Biopython, Bio.SeqUtils.gc_fraction: the G+C fraction of a sequence. Biopython also counts S and lowercase letters and, by default, leaves ambiguous letters out of the length; this routine counts only G and C and every letter in the length", + "url": "https://biopython.org/docs/latest/api/Bio.SeqUtils.html#Bio.SeqUtils.gc_fraction" + } + ], + "source": "// GCContent: the fraction of the letters of S that are G or C. Other letters\n// (N for an unknown base, lowercase) count in the length and not in the count.\nFUNCTION GCContent(S : STRING) RETURNS FLOAT\n REQUIRE LENGTH(S) > 0\n DECLARE Count : INTEGER <- 0\n FOR I <- 0 TO LENGTH(S)\n IF S[I] == 'G' OR S[I] == 'C' THEN\n Count <- Count + 1\n ENDIF\n NEXT I\n RETURN Count / LENGTH(S)\nENDFUNCTION\n", + "cases": [ + { + "call": "GCContent(\"GATTACA\")", + "result": "0.2857142857142857" + }, + { + "call": "GCContent(\"GGCC\")", + "result": "1.0" + }, + { + "call": "GCContent(\"ATAT\")", + "result": "0.0" + }, + { + "call": "GCContent(\"GCNa\")", + "result": "0.5" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same count.", + "source": "def gc_content(s):\n count = 0\n for c in s:\n if c == \"G\" or c == \"C\":\n count += 1\n return count / len(s)\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same count in Mojo.", + "source": "def gc_content(s: String) -> Float64:\n var count = 0\n for i in range(s.count_codepoints()):\n var c = String(s[codepoint=i])\n if c == \"G\" or c == \"C\":\n count = count + 1\n return Float64(count) / Float64(s.count_codepoints())\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 4 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 4 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 4 case(s) (exactly)" + } + } + } + ], + "proof": { + "lang": "dafny", + "file": "proof/gc_content.dfy", + "note": "The count never exceeds the length, so the fraction lies in [0, 1].", + "source": "function GC(s: string, n: nat): nat\n requires n <= |s|\n{\n if n == 0 then 0 else GC(s, n - 1) + (if s[n - 1] == 'G' || s[n - 1] == 'C' then 1 else 0)\n}\n\nlemma GCAtMost(s: string, n: nat)\n requires n <= |s|\n ensures GC(s, n) <= n\n{\n if n > 0 { GCAtMost(s, n - 1); }\n}\n\nmethod GCContent(s: string) returns (f: real)\n requires |s| > 0\n ensures f == (GC(s, |s|) as real) / (|s| as real)\n ensures 0.0 <= f <= 1.0\n{\n var count := 0;\n var i := 0;\n while i < |s|\n invariant 0 <= i <= |s|\n invariant count == GC(s, i)\n {\n if s[i] == 'G' || s[i] == 'C' { count := count + 1; }\n i := i + 1;\n }\n GCAtMost(s, |s|);\n f := (count as real) / (|s| as real);\n}\n", + "check": { + "status": "pass", + "detail": "Dafny 4.11.0: 5 verified, 0 errors" + } + }, + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION GCContent" + }, + "pseudo": { + "status": "pass", + "detail": "4 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 3 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 469 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + }, + { + "id": "seq.HammingDistance", + "module": "seq", + "name": "HammingDistance", + "dir": "modules/seq/hamming-distance", + "summary": "How many positions of two sequences of the same length hold different letters.", + "tags": [ + "sequences", + "distance" + ], + "sources": [ + { + "title": "Hamming, Error detecting and error correcting codes, Bell System Technical Journal 29(2), 1950", + "url": "https://doi.org/10.1002/j.1538-7305.1950.tb00463.x" + } + ], + "source": "// HammingDistance: how many positions of two sequences of the same length\n// hold different letters. The INVARIANT is what makes the ENSURE provable:\n// after I positions, at most I mismatches can have been counted.\nFUNCTION HammingDistance(S : STRING, T : STRING) RETURNS INTEGER\n REQUIRE LENGTH(S) == LENGTH(T)\n ENSURE 0 <= RESULT AND RESULT <= LENGTH(S)\n DECLARE Count : INTEGER <- 0\n FOR I <- 0 TO LENGTH(S)\n INVARIANT 0 <= Count AND Count <= I\n IF S[I] != T[I] THEN\n Count <- Count + 1\n ENDIF\n NEXT I\n RETURN Count\nENDFUNCTION\n", + "cases": [ + { + "call": "HammingDistance(\"GATTACA\", \"GACTATA\")", + "result": "2" + }, + { + "call": "HammingDistance(\"\", \"\")", + "result": "0" + }, + { + "call": "HammingDistance(\"AAAA\", \"TTTT\")", + "result": "4" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same count.", + "source": "def hamming_distance(s, t):\n return sum(1 for a, b in zip(s, t) if a != b)\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same count in Mojo.", + "source": "def hamming_distance(s: String, t: String) -> Int:\n var count = 0\n for i in range(s.count_codepoints()):\n if String(s[codepoint=i]) != String(t[codepoint=i]):\n count = count + 1\n return count\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 3 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 3 case(s) (exactly)" + } + } + } + ], + "proof": { + "lang": "dafny", + "file": "proof/hamming_distance.dfy", + "note": "The loop computes the number of mismatched positions, which lies in [0, |S|].", + "source": "function Mismatches(s: string, t: string, n: nat): nat\n requires n <= |s| && n <= |t|\n{\n if n == 0 then 0 else Mismatches(s, t, n - 1) + (if s[n - 1] != t[n - 1] then 1 else 0)\n}\n\nlemma MismatchesAtMost(s: string, t: string, n: nat)\n requires n <= |s| && n <= |t|\n ensures Mismatches(s, t, n) <= n\n{\n if n > 0 { MismatchesAtMost(s, t, n - 1); }\n}\n\nmethod HammingDistance(s: string, t: string) returns (count: int)\n requires |s| == |t|\n ensures count == Mismatches(s, t, |s|)\n ensures 0 <= count <= |s|\n{\n count := 0;\n var i := 0;\n while i < |s|\n invariant 0 <= i <= |s|\n invariant count == Mismatches(s, t, i)\n {\n if s[i] != t[i] { count := count + 1; }\n i := i + 1;\n }\n MismatchesAtMost(s, t, |s|);\n}\n", + "check": { + "status": "pass", + "detail": "Dafny 4.11.0: 5 verified, 0 errors" + } + }, + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION HammingDistance" + }, + "pseudo": { + "status": "pass", + "detail": "3 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 5 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 291 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + } + ] + }, + { + "name": "special", + "title": "Special functions", + "summary": "Functions with no exact built-in, as approximations with a stated error bound.", + "routines": [ + { + "id": "special.ErfApprox", + "module": "special", + "name": "ErfApprox", + "dir": "modules/special/erf-approx", + "summary": "The error function by Abramowitz and Stegun's formula 7.1.26, whose stated absolute error is below 1.5e-7 for every X.", + "tags": [ + "special functions", + "approximation", + "statistics" + ], + "sources": [ + { + "title": "Abramowitz and Stegun, Handbook of Mathematical Functions (National Bureau of Standards, 1964), section 7.1, formula 7.1.26", + "url": "https://archive.org/details/HandbookOfMathematicalFunctions" + } + ], + "source": "// ErfApprox: the error function by Abramowitz and Stegun's formula 7.1.26,\n// with an absolute error below 1.5e-7 for every X. It is an approximation\n// with a stated bound, not the correctly rounded erf: the libm reference\n// holds it to that bound on the cases. ERF(X) for X < 0 is -ERF(-X), so the\n// sign is set aside first. The prover leaves one obligation open: that the\n// divisor 1.0 + 0.3275911 * A is not 0. It is at least 1, because A >= 0,\n// but A and the product are FLOAT values, and the prover does not see into\n// FLOAT arithmetic, so it cannot conclude it.\nFUNCTION ErfApprox(X : FLOAT) RETURNS FLOAT\n DECLARE Sign : FLOAT <- 1.0\n IF X < 0.0 THEN\n Sign <- -1.0\n ENDIF\n DECLARE A : FLOAT <- ABS(X)\n DECLARE T : FLOAT <- 1.0 / (1.0 + 0.3275911 * A)\n DECLARE Poly : FLOAT <- ((((1.061405429 * T - 1.453152027) * T + 1.421413741) * T - 0.284496736) * T + 0.254829592) * T\n RETURN Sign * (1.0 - Poly * EXP(-A * A))\nENDFUNCTION\n", + "cases": [ + { + "call": "ErfApprox(0.0)", + "result": "9.999999717180685e-10" + }, + { + "call": "ErfApprox(0.5)", + "result": "0.5205000163047472" + }, + { + "call": "ErfApprox(-1.0)", + "result": "-0.8427006897475899" + }, + { + "call": "ErfApprox(2.0)", + "result": "0.9953221395812188" + }, + { + "call": "ErfApprox(3.5)", + "result": "0.999999255782864" + }, + { + "call": "ErfApprox(0.0451)", + "result": "0.05085555733320535" + }, + { + "call": "ErfApprox(1.0e-5)", + "result": "0.00001128486356871683" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same formula.", + "source": "import math\n\n\ndef erf_approx(x):\n sign = -1.0 if x < 0.0 else 1.0\n a = abs(x)\n t = 1.0 / (1.0 + 0.3275911 * a)\n poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t\n return sign * (1.0 - poly * math.exp(-a * a))\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 7 case(s) (exactly)" + } + } + }, + { + "lang": "python", + "level": "libm", + "file": "ref/python/libm.py", + "note": "Python's math.erf, which uses the platform C library's erf: the approximation must stay within its stated 1.5e-7 of it. ErfApprox(0.0451) sits next to the largest error, about 1.39e-7.", + "abs": 1.5e-7, + "source": "import math\n\n\ndef erf_approx(x):\n return math.erf(x)\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 7 case(s) (within rel 0, abs 1.5e-7)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same formula in Mojo. The compiled program calls Mojo's own exp(), which in Mojo 1.2.0 is off by up to about 6.5e-12 of the value on [-16, 0], so the last digits are not the pseudo's. The result is 1 - Poly * exp(-A * A), so the difference is small in absolute terms (at most 1.2e-13 over 640,000 values of X between -8 and 8, many of them near 0, computed at run time by Mojo 1.2.0), but not in relative terms near 0, where erf is small: at X = 1.0e-5 the difference is about 8e-12 of the value.", + "abs": 2e-13, + "source": "from std.math import exp\n\n\ndef erf_approx(x: Float64) -> Float64:\n var sign: Float64 = 1.0\n if x < 0.0:\n sign = -1.0\n var a = abs(x)\n var t = 1.0 / (1.0 + 0.3275911 * a)\n var poly = ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t\n return sign * (1.0 - poly * exp(-a * a))\n", + "checks": { + "interpreter": { + "status": "skipped", + "detail": "core's Mojo interpreter declines exp(...), which Mojo computes with a routine of its own that Programming+ does not reproduce digit for digit" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 7 case(s) (within rel 0, abs 2e-13)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 7 case(s) (within rel 0, abs 2e-13)" + } + } + } + ], + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION ErfApprox" + }, + "pseudo": { + "status": "pass", + "detail": "7 case(s) run" + }, + "prover": { + "status": "open", + "detail": "0 of 1 obligation(s) proved; open: divide@15 (1.0 + 0.3275911 * A is not 0)" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 269 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints the pseudo's results to within 5.3e-14 of each value (the compiler's rounding and std.math) (core's interpreter declines exp(...), which Mojo computes with a routine of its own that Programming+ does not reproduce digit for digit)" + } + } + } + ] + }, + { + "name": "stats", + "title": "Statistics", + "summary": "Summaries of a list of measurements: where it sits and how far it spreads.", + "routines": [ + { + "id": "stats.Mean", + "module": "stats", + "name": "Mean", + "dir": "modules/stats/mean", + "summary": "The arithmetic mean of a list of numbers.", + "tags": [ + "statistics", + "summation" + ], + "sources": [ + { + "title": "NIST/SEMATECH e-Handbook of Statistical Methods, section 1.3.5.1, Measures of location: the mean is the sum of the data points divided by their number", + "url": "https://www.itl.nist.gov/div898/handbook/eda/section3/eda351.htm" + } + ], + "source": "// Mean: the arithmetic mean of a list of numbers.\n// The running total is a FLOAT, rounded at every addition, so the mean of\n// [0.1, 0.1, 0.1] comes out a little above 0.1. For long lists of values of\n// very different sizes, sum with numerics.KahanSum instead.\nFUNCTION Mean(X : LIST OF FLOAT) RETURNS FLOAT\n REQUIRE LENGTH(X) > 0\n DECLARE Total : FLOAT <- 0.0\n FOR I <- 0 TO LENGTH(X)\n Total <- Total + X[I]\n NEXT I\n RETURN Total / LENGTH(X)\nENDFUNCTION\n", + "cases": [ + { + "call": "Mean([1.0, 2.0, 3.0, 4.0])", + "result": "2.5" + }, + { + "call": "Mean([0.1, 0.1, 0.1])", + "result": "0.10000000000000002" + }, + { + "call": "Mean([-5.0, 5.0])", + "result": "0.0" + }, + { + "call": "Mean([1.0e300, 1.0e300])", + "result": "1.0e300" + }, + { + "call": "Mean([2.5])", + "result": "2.5" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same loop: one running total, added to in order.", + "source": "def mean(x):\n total = 0.0\n for v in x:\n total += v\n return total / len(x)\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 5 case(s) (exactly)" + } + } + }, + { + "lang": "python", + "level": "fsum", + "file": "ref/python/fsum.py", + "note": "math.fsum rounds the sum once, exactly; the plain loop rounds at every step. On these cases the two differ at most in the last digit. Under cancellation they can differ in every digit (Mean([1.0e16, 1.0, -1.0e16]) is 0.0 by the loop and 0.3333333333333333 by fsum), so the tolerance holds for cases like these only.", + "rel": 1e-15, + "source": "import math\n\n\ndef mean(x):\n return math.fsum(x) / len(x)\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 5 case(s) (within rel 1e-15, abs 0)" + } + } + }, + { + "lang": "c", + "level": "plain", + "file": "ref/c/plain.c", + "note": "The same loop in C. Without -ffast-math the compiler keeps the additions in order; the check builds it at -O0 and -O2.", + "source": "double mean(const double *x, long n) {\n double total = 0.0;\n for (long i = 0; i < n; i++) total += x[i];\n return total / (double)n;\n}\n", + "checks": { + "cc -O0": { + "status": "pass", + "detail": "agrees with the pseudo on 5 case(s) (exactly)" + }, + "cc -O2": { + "status": "pass", + "detail": "agrees with the pseudo on 5 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same loop in Mojo.", + "source": "def mean(x: List[Float64]) -> Float64:\n var total: Float64 = 0.0\n for i in range(len(x)):\n total = total + x[i]\n return total / Float64(len(x))\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 5 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 5 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 5 case(s) (exactly)" + } + } + }, + { + "lang": "mojo", + "level": "unrolled", + "file": "ref/mojo/unrolled.mojo", + "note": "Four running totals, the shape a vectorizing compiler gives the loop. The additions happen in a different order, so on these cases the last digit can differ; under cancellation every digit can (Mean([1.0e16, 1.0, 1.0, 1.0, -1.0e16, 0.0, 0.0, 0.0]) is 0.0 by the plain loop and 0.375 unrolled).", + "rel": 1e-15, + "source": "def mean(x: List[Float64]) -> Float64:\n var s0: Float64 = 0.0\n var s1: Float64 = 0.0\n var s2: Float64 = 0.0\n var s3: Float64 = 0.0\n var n = len(x)\n var i = 0\n while i + 4 <= n:\n s0 = s0 + x[i]\n s1 = s1 + x[i + 1]\n s2 = s2 + x[i + 2]\n s3 = s3 + x[i + 3]\n i = i + 4\n while i < n:\n s0 = s0 + x[i]\n i = i + 1\n return ((s0 + s1) + (s2 + s3)) / Float64(n)\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 5 case(s) in core's interpreter (within rel 1e-15, abs 0)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 5 case(s) (within rel 1e-15, abs 0)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 5 case(s) (within rel 1e-15, abs 0)" + } + } + } + ], + "proof": { + "lang": "dafny", + "file": "proof/mean.dfy", + "note": "Over real numbers: the loop computes Sum(X) / |X|, and the mean lies between the smallest and largest value (rounding can push the FLOAT result just outside, as Mean([0.1, 0.1, 0.1]) shows).", + "source": "// Mean, over real numbers. The FLOAT version rounds; this one does not.\nfunction Sum(x: seq<real>): real\n{\n if |x| == 0 then 0.0 else Sum(x[..|x| - 1]) + x[|x| - 1]\n}\n\nmethod Mean(x: seq<real>) returns (m: real)\n requires |x| > 0\n ensures m == Sum(x) / (|x| as real)\n{\n var total := 0.0;\n var i := 0;\n while i < |x|\n invariant 0 <= i <= |x|\n invariant total == Sum(x[..i])\n {\n assert x[..i + 1][..i] == x[..i];\n total := total + x[i];\n i := i + 1;\n }\n assert x[..|x|] == x;\n m := total / (|x| as real);\n}\n\n// Every value at most hi: the sum is at most |x| * hi.\nlemma SumAtMost(x: seq<real>, hi: real)\n requires forall i :: 0 <= i < |x| ==> x[i] <= hi\n ensures Sum(x) <= (|x| as real) * hi\n{\n if |x| > 0 { SumAtMost(x[..|x| - 1], hi); }\n}\n\n// Every value at least lo: the sum is at least |x| * lo.\nlemma SumAtLeast(x: seq<real>, lo: real)\n requires forall i :: 0 <= i < |x| ==> lo <= x[i]\n ensures (|x| as real) * lo <= Sum(x)\n{\n if |x| > 0 { SumAtLeast(x[..|x| - 1], lo); }\n}\n\n// In exact arithmetic the mean lies between any lower and upper bound of the values.\nlemma MeanBetween(x: seq<real>, lo: real, hi: real)\n requires |x| > 0\n requires forall i :: 0 <= i < |x| ==> lo <= x[i] <= hi\n ensures lo <= Sum(x) / (|x| as real) <= hi\n{\n SumAtMost(x, hi);\n SumAtLeast(x, lo);\n}\n", + "check": { + "status": "pass", + "detail": "Dafny 4.11.0: 9 verified, 0 errors" + } + }, + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION Mean" + }, + "pseudo": { + "status": "pass", + "detail": "5 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 2 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 262 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + }, + { + "id": "stats.Variance", + "module": "stats", + "name": "Variance", + "dir": "modules/stats/variance", + "summary": "The sample variance of a list of numbers (dividing by N - 1), computed in two passes.", + "tags": [ + "statistics", + "cancellation" + ], + "sources": [ + { + "title": "Chan, Golub and LeVeque, Algorithms for computing the sample variance: analysis and recommendations, The American Statistician 37(3), 1983", + "url": "https://doi.org/10.1080/00031305.1983.10483115" + } + ], + "source": "// Variance: the sample variance (dividing by N - 1), in two passes: the mean\n// first, then the squared distances from it. The one-pass formula\n// (SUM X^2 - N * MEAN^2) / (N - 1) subtracts two nearly equal large numbers\n// and loses every digit when the values sit far from 0: on the third case it\n// gives -170.66666666666666 in FLOAT arithmetic, where the two passes give 30.0.\nUSE stats.Mean\nFUNCTION Variance(X : LIST OF FLOAT) RETURNS FLOAT\n REQUIRE LENGTH(X) >= 2\n DECLARE M : FLOAT <- Mean(X)\n DECLARE Total : FLOAT <- 0.0\n FOR I <- 0 TO LENGTH(X)\n Total <- Total + (X[I] - M) * (X[I] - M)\n NEXT I\n RETURN Total / (LENGTH(X) - 1)\nENDFUNCTION\n", + "cases": [ + { + "call": "Variance([2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0])", + "result": "4.571428571428571" + }, + { + "call": "Variance([1.0, 2.0])", + "result": "0.5" + }, + { + "call": "Variance([1.0e9 + 4.0, 1.0e9 + 7.0, 1.0e9 + 13.0, 1.0e9 + 16.0])", + "result": "30.0" + } + ], + "references": [ + { + "lang": "python", + "level": "plain", + "file": "ref/python/plain.py", + "note": "The same two passes.", + "source": "def variance(x):\n m = 0.0\n for v in x:\n m += v\n m /= len(x)\n total = 0.0\n for v in x:\n total += (v - m) * (v - m)\n return total / (len(x) - 1)\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) (exactly)" + } + } + }, + { + "lang": "python", + "level": "statistics", + "file": "ref/python/statistics.py", + "note": "The standard library's statistics.variance, which works with exact fractions inside: a different method, held to agree to 12 digits.", + "rel": 1e-12, + "source": "import statistics\n\n\ndef variance(x):\n return statistics.variance(x)\n", + "checks": { + "cpython": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) (within rel 1e-12, abs 0)" + } + } + }, + { + "lang": "mojo", + "level": "plain", + "file": "ref/mojo/plain.mojo", + "note": "The same two passes in Mojo.", + "source": "def variance(x: List[Float64]) -> Float64:\n var m: Float64 = 0.0\n for i in range(len(x)):\n m = m + x[i]\n m = m / Float64(len(x))\n var total: Float64 = 0.0\n for i in range(len(x)):\n total = total + (x[i] - m) * (x[i] - m)\n return total / Float64(len(x) - 1)\n", + "checks": { + "interpreter": { + "status": "pass", + "detail": "agrees with the pseudo on 3 case(s) in core's interpreter (exactly)" + }, + "compiler": { + "status": "pass", + "detail": "built by Mojo 1.2.0, agrees with the pseudo on 3 case(s) (exactly)" + }, + "godbolt": { + "status": "pass", + "detail": "mojo_1_2_0 agrees with the pseudo on 3 case(s) (exactly)" + } + } + } + ], + "checks": { + "structure": { + "status": "pass", + "detail": "defines FUNCTION Variance, using stats.Mean" + }, + "pseudo": { + "status": "pass", + "detail": "3 case(s) run" + }, + "prover": { + "status": "pass", + "detail": "all 4 obligation(s) proved" + }, + "fuzz": { + "status": "pass", + "detail": "no failure in 221 generated input(s)" + }, + "mojo": { + "status": "pass", + "detail": "built by Mojo 1.2.0, the generated Mojo prints what the pseudo prints on every case, and in core's interpreter" + } + } + } + ] + } +]; diff --git a/src/library/registry.ts b/src/library/registry.ts new file mode 100644 index 0000000..892c04e --- /dev/null +++ b/src/library/registry.ts @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The library as core sees it: modules of verified routines that a program brings in with USE. + * + * The data comes from the generated module (scripts/library.mjs emit, from the library/ submodule), so the + * site needs no files at run time. The library's own tools read the routines from disk instead and install + * them here with setLibrary() before checking them. + */ +import { LIBRARY_MODULES } from './generated.js'; +import type { LibraryModule, LibraryRoutine } from './types.js'; + +export type { LibraryModule, LibraryRoutine } from './types.js'; + +let modules: LibraryModule[] = LIBRARY_MODULES; +let byModule = index(modules); + +function index(mods: LibraryModule[]): Map<string, LibraryModule> { + return new Map(mods.map((m) => [m.name.toUpperCase(), m])); +} + +/** Replace the library (the library's own tools, and tests). Returns the one it replaced. */ +export function setLibrary(mods: LibraryModule[]): LibraryModule[] { + const old = modules; + modules = mods; + byModule = index(mods); + generation++; + return old; +} + +/** Bumped by setLibrary, so a cache of studied routines knows to start again. */ +export let generation = 0; + +export function libraryModules(): LibraryModule[] { return modules; } + +export function libraryModule(name: string): LibraryModule | undefined { return byModule.get(String(name).toUpperCase()); } + +/** A routine by its id (stats.Mean), or by module and name. Names ignore case, as in pseudo. */ +export function libraryRoutine(module: string, name?: string): LibraryRoutine | undefined { + if (name === undefined) { const i = module.indexOf('.'); if (i < 0) return undefined; name = module.slice(i + 1); module = module.slice(0, i); } + return libraryModule(module)?.routines.find((r) => r.name.toUpperCase() === String(name).toUpperCase()); +} diff --git a/src/library/types.ts b/src/library/types.ts new file mode 100644 index 0000000..65cb1fb --- /dev/null +++ b/src/library/types.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The shapes of the library (pplus-library, the submodule library/): modules of verified pseudo routines, each + * with its cases, its reference implementations in other languages, an optional proof, and the status of every + * check, as `pixi run library-check` last recorded it. + */ + +/** One check of one routine: what it found, and when the stored answer was made. */ +export interface LibraryCheck { + /** + * 'pass'; 'fail'; 'open' (the prover could not finish every obligation: nothing is wrong, nothing is proved); + * 'unverified' (no stored answer for this exact text); 'skipped' (the tool is not available, or declines). + */ + status: 'pass' | 'fail' | 'open' | 'unverified' | 'skipped'; + /** One line: what was checked and what it found. */ + detail: string; +} + +export interface LibraryReference { + /** 'python', 'mojo', 'c', ... */ + lang: string; + /** 'plain', 'unrolled', 'fsum', ...: one implementation of the routine, at one level of care or optimization. */ + level: string; + /** The file, relative to the routine's folder. */ + file: string; + /** What this level does differently, in a sentence, and why it needs a tolerance if it has one. */ + note: string; + /** The tolerance this level is held to against the pseudo, relative and absolute; exact when absent. */ + rel?: number; + abs?: number; + source: string; + /** Per way of running it: 'interpreter' (core's Mojo interpreter), 'godbolt', 'cpython', 'cc -O0', 'cc -O2'. */ + checks: Record<string, LibraryCheck>; +} + +export interface LibraryCase { + /** A call in pseudo, such as Mean([1.0, 2.0, 3.0]). */ + call: string; + /** What the pseudo returns, written as pseudo writes values; filled in by library-check from a run. */ + result?: string; + /** The tolerance a reference implementation is held to, relative and absolute; 0 for exact. */ + rel?: number; + abs?: number; +} + +export interface LibraryRoutine { + /** module.Name, as USE writes it: stats.Mean. */ + id: string; + module: string; + name: string; + /** The routine's folder in pplus-library, such as modules/stats/mean. */ + dir: string; + summary: string; + tags: string[]; + /** Where the method comes from: a paper, a textbook section, a manual page. */ + sources: Array<{ title: string; url?: string }>; + /** The routine file, routine.pseudo: one FUNCTION or PROCEDURE with its contracts, and USE lines for what it calls. */ + source: string; + cases: LibraryCase[]; + references: LibraryReference[]; + proof?: { lang: string; file: string; note: string; source: string; check: LibraryCheck }; + /** pseudo (cases run), prover, fuzz, mojo (the generated Mojo), each a LibraryCheck. */ + checks: Record<string, LibraryCheck>; +} + +export interface LibraryModule { + name: string; + title: string; + summary: string; + routines: LibraryRoutine[]; +} diff --git a/src/mojo/att.ts b/src/mojo/att.ts new file mode 100644 index 0000000..4d86ed8 --- /dev/null +++ b/src/mojo/att.ts @@ -0,0 +1,392 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * godbolt's Mojo listing is AT&T syntax (mojo build has no Intel option, and + * godbolt passes none), and the x86-64 pane reads Intel syntax. This rewrites + * one listing line at a time: + * + * movq %rdi, 16(%rsp) -> mov qword ptr [rsp + 16], rdi + * addq %rsi, %rax -> add rax, rsi + * movzbl (%rdi,%rcx), %eax -> movzx eax, byte ptr [rdi + rcx*1] + * callq "example::add(...)" -> call add + * + * Operands reverse (AT&T writes the destination last), the size suffix + * becomes a `ptr` size on the memory operand, `$` and `%` go, and the + * compiler's quoted names become plain labels. Nothing is invented: an + * instruction or operand outside what the simulator models is still written + * out, and the x86-64 pane refuses it line by line, which is the honest answer + * (a call into the standard library cannot run on the model machine). + */ +import { demangle, type AsmLine, type ListingFunction } from '../services/godbolt.js'; + +const SIZE: Record<string, string> = { b: 'byte', w: 'word', l: 'dword', q: 'qword' }; + +/** Mnemonics that take a one-letter size suffix in AT&T. */ +const SUFFIXED = new Set(['mov', 'add', 'sub', 'and', 'or', 'xor', 'cmp', 'test', 'lea', 'push', 'pop', 'inc', 'dec', 'neg', 'not', 'imul', 'mul', 'div', 'idiv', 'shl', 'sal', 'shr', 'sar', 'rol', 'ror', 'adc', 'sbb', 'xchg', 'call', 'ret', 'jmp', 'bt', 'bts', 'btr', 'btc', 'popcnt', 'tzcnt', 'lzcnt', 'bsf', 'bsr', 'nop', 'leave', + // BMI and the atomics, which the compiler also writes with a size suffix + 'andn', 'mulx', 'shlx', 'shrx', 'sarx', 'bzhi', 'blsr', 'blsi', 'blsmsk', 'rorx', 'bextr', 'pdep', 'pext', 'xadd', 'cmpxchg', 'bswap', 'movbe', 'crc32', + // the double-width shifts, and the integer-to-double conversions, whose memory operand is a dword or a qword + 'shld', 'shrd', 'cvtsi2sd', 'vcvtsi2sd']); + +/** Whole-mnemonic renames: sign and zero extension, and the conversions. */ +const RENAME: Record<string, { op: string; src?: string }> = { + movzbl: { op: 'movzx', src: 'byte' }, movzbw: { op: 'movzx', src: 'byte' }, movzbq: { op: 'movzx', src: 'byte' }, + movzwl: { op: 'movzx', src: 'word' }, movzwq: { op: 'movzx', src: 'word' }, + movsbl: { op: 'movsx', src: 'byte' }, movsbw: { op: 'movsx', src: 'byte' }, movsbq: { op: 'movsx', src: 'byte' }, + movswl: { op: 'movsx', src: 'word' }, movswq: { op: 'movsx', src: 'word' }, + movslq: { op: 'movsxd', src: 'dword' }, + movabsq: { op: 'movabs' }, + cltq: { op: 'cdqe' }, cqto: { op: 'cqo' }, cltd: { op: 'cdq' }, cwtl: { op: 'cwde' }, +}; + +/** Labels the simulator accepts: letters, digits, `_`, `.`, `$`. */ +export type LabelMap = Map<string, string>; + +function bareName(sym: string): string { + return sym.replace(/@PLT$/, '').replace(/^"|"$/g, ''); +} + +/** A label for every name the listing uses: the user's functions by their Mojo names, the rest by a readable path. */ +export function labelMap(fns: ListingFunction[]): LabelMap { + const map: LabelMap = new Map(); + const used = new Set<string>(); + const claim = (want: string) => { let n = want || 'f', k = 2; while (used.has(n)) n = `${want}_${k++}`; used.add(n); return n; }; + for (const f of fns.filter((x) => x.kind === 'user')) map.set(bareName(f.label), claim(demangle(f.label).replace(/[(\[].*$/, '').replace(/[^\w.$]/g, '_'))); + for (const f of fns.filter((x) => x.kind !== 'user')) map.set(bareName(f.label), claim(f.kind === 'entry' ? 'entry' : simpleName(bareName(f.label)))); + return map; +} + +function simpleName(sym: string): string { + const path = sym.replace(/[(\[].*$/, '').split('::').filter(Boolean); + return (path.join('_') || 'sym').replace(/[^\w.$]/g, '_'); +} + +function label(sym: string, map: LabelMap): string { + const b = bareName(sym); + return map.get(b) ?? (/^[\w.$]+$/.test(b) ? b : simpleName(b)); +} + +function register(r: string): string { return r.replace(/^%/, '').toLowerCase(); } + +/** Split operands on commas outside parentheses and quotes. */ +export function splitOperands(s: string): string[] { + const out: string[] = []; + let depth = 0, quote = false, cur = ''; + for (const c of s) { + if (c === '"') quote = !quote; + if (!quote && c === '(') depth++; + if (!quote && c === ')') depth--; + if (!quote && depth === 0 && c === ',') { out.push(cur.trim()); cur = ''; continue; } + cur += c; + } + if (cur.trim()) out.push(cur.trim()); + return out; +} + +const MEM = /^(.*?)\(\s*(%\w+)?\s*(?:,\s*(%\w+)\s*(?:,\s*(\d+))?)?\s*\)$/; + +function memory(op: string, size: string | null, map: LabelMap): string { + const m = MEM.exec(op)!; + const disp = m[1].trim(); + const parts: string[] = []; + if (m[2]) parts.push(register(m[2])); + if (m[3]) parts.push(`${register(m[3])}*${m[4] ?? '1'}`); + let body = parts.join(' + '); + if (disp) { + const num = /^-?(?:0x[0-9a-f]+|\d+)$/i.test(disp); + const so = num ? null : /^([\w.$]+)\s*([+-])\s*(\d+)$/.exec(disp); // `.LCPI0_0+8`: a symbol and an offset + const d = num ? disp : so ? `${label(so[1], map)} ${so[2]} ${so[3]}` : label(disp, map); + body = !body ? d : num && d.startsWith('-') ? `${body} - ${d.slice(1)}` : `${body} + ${d}`; + } + return `${size ? `${size} ptr ` : ''}[${body}]`; +} + +function operand(op: string, memSize: string | null, map: LabelMap): string { + const t = op.trim().replace(/^\*/, ''); + if (t.startsWith('%')) return register(t); + if (t.startsWith('$')) { const v = t.slice(1); return /^-?(?:0x[0-9a-f]+|\d+)$/i.test(v) ? v : label(v, map); } + if (MEM.test(t)) return memory(t, memSize, map); + return label(t, map); +} + +/** One AT&T instruction as Intel, or null for a line that is not an instruction (a directive). */ +export function intelLine(text: string, map: LabelMap): string | null { + const t = text.trim(); + if (!t || t.startsWith('.') || t.startsWith('#')) return null; + // `lock` is a prefix on the instruction that follows; the model machine takes it the same way + if (/^lock\s+/i.test(t)) { const inner = intelLine(t.replace(/^lock\s+/i, ''), map); return inner ? `lock ${inner}` : null; } + const sp = t.search(/\s/); + const mnem = (sp < 0 ? t : t.slice(0, sp)).toLowerCase(); + const rest = sp < 0 ? '' : t.slice(sp + 1).trim(); + const ops = rest ? splitOperands(rest) : []; + let op = mnem; + let memSize: string | null = null; + let srcSize: string | null = null; + const ren = RENAME[mnem]; + if (ren) { op = ren.op; srcSize = ren.src ?? null; } + else if (/^cmov[a-z]+[wlq]$/.test(mnem) && !/^cmov(?:n?[abglepsoz]|n?[abgl]e|p[eo])$/.test(mnem)) op = mnem.slice(0, -1); + else if (mnem.length > 1 && SUFFIXED.has(mnem.slice(0, -1)) && SIZE[mnem.at(-1)!]) { op = mnem.slice(0, -1); memSize = SIZE[mnem.at(-1)!]; } + if (op === 'nop') return 'nop'; + if (op === 'lea' || op === 'call' || /^j[a-z]+$/.test(op)) memSize = null; // an address, not a load: no size + const intel = ops.map((o) => operand(o, srcSize && MEM.test(o.replace(/^\*/, '')) ? srcSize : memSize, map)).reverse(); + return intel.length ? `${op} ${intel.join(', ')}` : op; +} + +/** + * Standard-library functions whose body is empty, so the model machine can run past a call to them. + * `__mojo_debugger_raise_hook` is called on every raise for a debugger to break on; at -O0 a `for` loop's + * iterator ends by raising StopIteration, so every loop over a range calls it once, at its end. + */ +const NO_OP_CALLS = [/__mojo_debugger_raise_hook\b/]; + +/** Where the listing's read-only strings are placed in the model machine's memory, one after another. */ +const DATA_BASE = 0x6000; + +/** A data block longer than this is written into the setup as quoted text, not as a list of byte values. */ +export const LONG_BLOCK = 64; + +/** + * The functions to send: the chosen one first (the simulator starts at the + * first instruction), then the user's functions it calls (with `all`, every + * function of the program's own). Calls into the + * standard library get a stub that stops the run and says so, so the function + * assembles and steps up to that point. The read-only strings the code + * addresses through `[rip + static_string_…]` are placed in memory and the + * address written in their place; `data` holds the setup lines that put them + * there. + */ +export function forSimulator(fns: ListingFunction[], chosen: ListingFunction, opts: { all?: boolean } = {}): { source: string; refusedCalls: string[]; data: string[] } { + const map = labelMap(fns); + const byBare = new Map(fns.map((f) => [bareName(f.label), f])); + const order: ListingFunction[] = []; + const refused = new Set<string>(); + const hooks = new Set<string>(); + const visit = (f: ListingFunction) => { + if (order.includes(f)) return; + order.push(f); + for (const l of f.lines) { + const m = /^\s*(?:callq?|jmpq?)\s+\*?("(?:[^"\\]|\\.)*"(?:@PLT)?|[^\s,]+)/.exec(l.text); + if (!m) continue; + const target = byBare.get(bareName(m[1])); + if (target && target.kind === 'user') visit(target); + else if (!bareName(m[1]).startsWith('.L') && !m[1].startsWith('%') && !m[1].startsWith('*')) { + if (NO_OP_CALLS.some((re) => re.test(bareName(m[1])))) hooks.add(label(m[1], map)); + else refused.add(label(m[1], map)); + } + } + }; + visit(chosen); + // the whole program's own code, not only what the chosen function reaches: it runs first, the rest follow + if (opts.all) for (const f of fns) if (f.kind === 'user') visit(f); + // the strings: every `static_string_…:` label and the .asciz after it, anywhere in the listing. A block that is + // not one plain .asciz (several .ascii pieces, .byte lines, .zero for an empty string) is placed from its bytes. + const strings = new Map<string, string>(); + const blocks = new Map<string, number[]>(); + for (const f of fns) { + if (f.kind !== 'data') continue; + const texts = f.lines.map((l) => l.text.trim()).filter((t) => t && !/^\.set\s|^[\w.$]+\s*=/.test(t)); // an alias line is read below + const az = texts.length === 1 ? /^\.asciz\s+"((?:[^"\\]|\\.)*)"$/.exec(texts[0]) : null; + if (az) { strings.set(bareName(f.label), az[1]); continue; } + const bytes = dataBytes(texts); + if (bytes) blocks.set(bareName(f.label), bytes); + } + // `.set a, b+3` (or `a = b`): a name for another label's bytes + const aliases = new Map<string, { target: string; offset: number }>(); + for (const f of fns) for (const l of f.lines) { + const m = /^\s*(?:\.set\s+([\w.$]+)\s*,|([\w.$]+)\s*=)\s*([\w.$]+)\s*(?:([+-])\s*(\d+))?\s*$/.exec(l.text); + if (m) aliases.set(m[1] ?? m[2], { target: m[3], offset: m[5] ? (m[4] === '-' ? -1 : 1) * Number(m[5]) : 0 }); + } + // the constant pool: `.LCPI0_0:` / `.LCPI0_0: .quad 0x…` blocks that sit inside a function's lines, numbers only + const pool = constantPool(fns); + for (const [k, v] of blocks) pool.set(k, v); + const placed = new Map<string, number>(); + const data: string[] = []; + let next = DATA_BASE; + // A block's bytes as the setup writes them. A short one is `bytes 0x.. 0x..`, which reads as numbers. A long one + // is quoted text with \xHH for what is not a printable character (src/services/asm.ts reads it back to the + // same bytes): the table of source locations that the compiler's bounds checks point at is half a megabyte, and + // at five characters a byte it made a setup of 2.5 MB, more than a share link is allowed to hold. + const block = (bytes: number[]): string => { + if (bytes.length <= LONG_BLOCK) return `bytes ${bytes.map((b) => '0x' + b.toString(16).padStart(2, '0')).join(' ')}`; + const parts: string[] = []; + for (const b of bytes) parts.push(b === 0x22 ? '\\"' : b === 0x5c ? '\\\\' : b >= 32 && b < 127 ? String.fromCharCode(b) : `\\x${b.toString(16).padStart(2, '0')}`); + return `"${parts.join('')}"`; + }; + const place = (name: string, depth = 0): number | null => { + if (placed.has(name)) return placed.get(name)!; + const alias = aliases.get(name); + if (alias && depth < 8) { const at = place(alias.target, depth + 1); return at == null ? null : at + alias.offset; } + const text = strings.get(name); + const raw = pool.get(name); + if (text == null && raw == null) return null; + placed.set(name, next); + if (text == null) { + data.push(`memory 0x${next.toString(16)} ${block(raw!)} # ${name}`); + next = (next + Math.max(1, raw!.length) + 15) & ~15; + return placed.get(name)!; + } + const bytes = asciz(text); + const plain = bytes.every((b) => b >= 32 && b < 127 && b !== 0x22 && b !== 0x5c); + data.push(plain ? `memory 0x${next.toString(16)} "${String.fromCharCode(...bytes)}" # ${name}` : `memory 0x${next.toString(16)} ${block([...bytes, 0])} # ${name}`); + next += Math.max(1, bytes.length + 1); + next = (next + 15) & ~15; + return placed.get(name)!; + }; + const out: string[] = []; + const unplaced = new Set<string>(); + if (opts.all && order.length > 1) { + out.push(`; The program's own functions, as the compiler built them. ${chosen.name} runs first: set its arguments in Setup.`); + out.push(`; To run another one, put it first (or use "Send to x86-64 pane" in the Compiler tab).`); + out.push(''); + } + for (const f of order) { + out.push(`${map.get(bareName(f.label))}:`.padEnd(8) + `; ${f.name}`); + for (const l of f.lines) { + const lm = /^\s*([.\w$]+):\s*$/.exec(l.text); + if (lm) { if (!pool.has(lm[1])) out.push(`${lm[1]}:`); continue; } + let intel = intelLine(l.text, map); + if (intel) intel = intel.replace(/\[rip \+ ([\w.$]+)(?: ([+-]) (\d+))?\]/g, (whole, name, sign, off) => { + const addr = place(name); + if (addr == null) { if (!/^(rax|rbx|rcx|rdx|rsi|rdi|rbp|rsp|r\d+)$/.test(name)) unplaced.add(name); return whole; } + const at = addr + (off ? (sign === '-' ? -1 : 1) * Number(off) : 0); + return `[0x${at.toString(16)}]`; + }); + // each instruction keeps the Mojo line it came from, so the Machine view can total cost per line + if (intel) out.push(` ${intel}`.padEnd(40) + (l.line != null ? `; mojo line ${l.line}` : '')); + } + out.push(''); + } + if (hooks.size) { + out.push('; Hooks the standard library calls that do nothing and return: each is a plain ret here, as in the real program.'); + for (const name of hooks) { out.push(`${name}:`); out.push(` ret`.padEnd(40) + `; ${name}: does nothing (a place for a debugger to stop), so it returns at once`); } + out.push(''); + } + if (refused.size) { + out.push('; The standard library is not on the model machine. Each stub below stops the run where the real program would enter it.'); + for (const name of refused) { out.push(`${name}:`); out.push(` ud2`.padEnd(40) + `; ${name}: standard library, not on the model machine`); } + out.push(''); + } + if (unplaced.size) out.push(`; Data the code reads that the listing names but does not define in a form placed here: ${[...unplaced].join(', ')}`); + return { source: out.join('\n'), refusedCalls: [...refused], data }; +} + +/** + * The setup block that goes with `forSimulator`'s program: registers for the + * function's arguments, in the System V order the compiler used, then the + * listing's read-only strings (`data`). Shared by the "Send to x86-64 pane" + * button and `scripts/mojo/listings.mjs`, so a stored listing and a sent one + * start from the same setup. + */ +export function simulatorSetup(fn: ListingFunction, data: string[]): string { + const params = /\((.*)\)/.exec(fn.name)?.[1].split(',').map((x) => x.trim()).filter(Boolean) ?? []; + const regs = ['rdi', 'rsi', 'rdx', 'rcx', 'r8', 'r9']; + const ints = params.length <= regs.length && params.every((t) => /^U?Int\d*$|^Bool$/.test(t)); + const setup = !params.length ? `# ${fn.name} takes no arguments` + : ints ? [`# ${fn.name}: the arguments arrive in ${regs.slice(0, params.length).join(', ')} (System V); set them here`, ...params.map((_, i) => `${regs[i]} = 0`)].join('\n') + : `# ${fn.name}: its arguments are not all integers, so set up registers and memory by hand`; + return data.length ? `${setup}\n# the listing's read-only strings, where the code expects them\n${data.join('\n')}` : setup; +} + +/** Bytes of one numeric data directive, or null when its operands are not plain numbers (a symbol, an expression). */ +function directiveBytes(dir: string, args: string): number[] | null { + const width: Record<string, number> = { '.quad': 8, '.xword': 8, '.long': 4, '.int': 4, '.word': 2, '.short': 2, '.value': 2, '.hword': 2, '.byte': 1 }; + const out: number[] = []; + if (dir === '.zero' || dir === '.space' || dir === '.skip') { + const n = Number(args.split(',')[0]); + if (!Number.isInteger(n) || n < 0 || n > 4096) return null; + return new Array(n).fill(0); + } + if (dir === '.double' || dir === '.float') { + const dv = new DataView(new ArrayBuffer(8)); + for (const a of args.split(',').map((x) => x.trim())) { + const v = Number(a); + if (a === '' || Number.isNaN(v)) return null; + if (dir === '.double') { dv.setFloat64(0, v, true); out.push(...Array.from({ length: 8 }, (_, i) => dv.getUint8(i))); } + else { dv.setFloat32(0, v, true); out.push(...Array.from({ length: 4 }, (_, i) => dv.getUint8(i))); } + } + return out; + } + const w = width[dir]; + if (!w) return null; + for (const a of args.split(',').map((x) => x.trim())) { + if (!/^-?(?:0x[0-9a-f]+|\d+)$/i.test(a)) return null; + let v = BigInt(a.startsWith('-0x') ? '-' + a.slice(1) : a); + v = BigInt.asUintN(w * 8, v); + for (let i = 0; i < w; i++) { out.push(Number(v & 0xffn)); v >>= 8n; } + } + return out; +} + +/** The bytes of a block of data directives (.ascii, .asciz, .string, .byte, .quad, .zero, ...), or null if any line is something else. */ +function dataBytes(lines: string[]): number[] | null { + const out: number[] = []; + for (const t of lines) { + const sm = /^\.(ascii|asciz|string)\s+(.*)$/.exec(t); + if (sm) { + const parts = [...sm[2].matchAll(/"((?:[^"\\]|\\.)*)"/g)]; + if (!parts.length) return null; + for (const p of parts) { out.push(...asciz(p[1])); if (sm[1] !== 'ascii') out.push(0); } + continue; + } + const dm = /^(\.[a-z]+)\s*(.*?)(?:\s+#.*)?$/.exec(t); + if (!dm || /^\.(p2align|align|balign|type|size)$/.test(dm[1])) { if (dm) continue; return null; } + const b = directiveBytes(dm[1], dm[2]); + if (!b) return null; + out.push(...b); + } + return out.length ? out : null; +} + +/** + * The compiler's constant pool: `.LCPI0_0:` followed by `.quad 0x412e848000000000` + * and the like, which the listing leaves inside the preceding function. Maps the + * label to its bytes. A block with a symbol or expression among its operands + * (a jump table, say) is left out, so its label stays unresolved and the + * instruction that uses it is refused, as before. + */ +export function constantPool(fns: ListingFunction[]): Map<string, number[]> { + const pool = new Map<string, number[]>(); + for (const f of fns) { + let name: string | null = null, bytes: number[] = [], ok = true; + const close = () => { if (name && ok && bytes.length) pool.set(name, bytes); name = null; bytes = []; ok = true; }; + for (const l of f.lines) { + const t = l.text.replace(/\s+#.*$/, '').trim(); + if (!t) continue; + const lm = /^(\.L[\w.$]*):$/.exec(t); + if (lm) { close(); name = lm[1]; continue; } + const dm = /^(\.[a-z]+)\s*(.*)$/.exec(t); + if (name && dm && /^\.(quad|xword|long|int|word|short|value|hword|byte|zero|space|skip|double|float)$/.test(dm[1])) { + const b = directiveBytes(dm[1], dm[2]); + if (b) bytes.push(...b); else ok = false; + continue; + } + if (/^\.(p2align|align|balign)\b/.test(t)) continue; + close(); + } + close(); + } + return pool; +} + +/** The bytes of a `.asciz` literal: the assembler's escapes (\n, \t, \", \\, \ooo octal, \xhh), without the terminating zero. */ +function asciz(text: string): number[] { + const out: number[] = []; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + if (c !== '\\') { out.push(c.charCodeAt(0) & 0xff); continue; } + const n = text[++i]; + if (n === 'n') out.push(10); else if (n === 't') out.push(9); else if (n === 'r') out.push(13); else if (n === '0' || /[0-7]/.test(n)) { let o = n; while (o.length < 3 && /[0-7]/.test(text[i + 1] ?? '')) o += text[++i]; out.push(parseInt(o, 8) & 0xff); } + else if (n === 'x') { const h = /^[0-9a-fA-F]{1,2}/.exec(text.slice(i + 1))?.[0] ?? '0'; i += h.length; out.push(parseInt(h, 16)); } + else out.push(n.charCodeAt(0) & 0xff); + } + return out; +} + +/** Every line of a function as Intel, for display beside the AT&T original. */ +export function intelListing(fn: ListingFunction, fns: ListingFunction[]): Array<{ att: AsmLine; intel: string | null }> { + const map = labelMap(fns); + return fn.lines.map((l) => { + const lm = /^\s*([.\w$]+):\s*$/.exec(l.text); + return { att: l, intel: lm ? `${lm[1]}:` : intelLine(l.text, map) }; + }); +} diff --git a/src/mojo/check.ts b/src/mojo/check.ts new file mode 100644 index 0000000..436e3f8 --- /dev/null +++ b/src/mojo/check.ts @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * What Programming+ can say about a Mojo program before anyone compiles it. + * Stage 1 says one thing only, because it can say it exactly: a word the + * pinned compiler rejects as removed (`fn`, and `read` as a convention), in + * the compiler's own message, with its own fix. The messages come from the + * compiler's source through the corpus (MOJO_REMOVED_MESSAGES), and the + * probes dialect-fn and dialect-read confirm the compiler rejects both. + * + * Everything else waits for the Compile button, whose answer replaces these + * findings; Stage 2's checker grows this file. + */ +import type { Diagnostic } from '../common/types.js'; +import { MOJO_REMOVED_MESSAGES, MOJO_REMOVED_WORDS } from './generated/tokens.js'; +import { MOJO_VERSION } from './generated/pin.js'; +import { tokenizeMojo } from './tokenize.js'; +import { parseModule, type Module } from './parse.js'; +import { check } from './resolve.js'; +import { check2 } from './flow.js'; +import { check3 } from './typecheck.js'; +import { warnings } from './warnings.js'; + +export const CHECK_ORIGIN = `Programming+, in the Mojo ${MOJO_VERSION} compiler's words`; + +export function removedSyntax(source: string): Diagnostic[] { + const out: Diagnostic[] = []; + tokenizeMojo(source).forEach((toks, i) => { + let col = 1; + for (const t of toks) { + const msg = MOJO_REMOVED_MESSAGES[t.text]; + if (t.cls === 'tok-error' && msg) { + const repl = MOJO_REMOVED_WORDS[t.text]; + out.push({ + lang: 'mojo', severity: 'error', id: 'removed-syntax', line: i + 1, col, message: msg, origin: CHECK_ORIGIN, + fix: { label: `Replace with ${repl}`, line: i + 1, col, endCol: col + t.text.length, text: repl }, + }); + } + col += t.text.length; + } + }); + return out; +} + +/** + * Everything Programming+ can say about a Mojo program before it is compiled: + * removed syntax, what the parser certainly knows is wrong, and the structure + * and name checks (src/mojo/resolve.ts), all in the compiler's words. Code the + * parser cannot read yet gets one note, never an error. + */ +export function checkMojo(source: string): { diagnostics: Diagnostic[]; module: Module } { + const module = parseModule(source); + const diagnostics: Diagnostic[] = removedSyntax(source); + const origin = CHECK_ORIGIN; + for (const f of module.findings) diagnostics.push({ lang: 'mojo', severity: f.severity, id: f.id, line: f.line, col: f.col, message: f.message, origin }); + for (const c of check(module)) diagnostics.push({ lang: 'mojo', severity: 'error', id: c.id, line: c.line, col: c.col, message: c.message, origin }); + for (const c of check2(module)) diagnostics.push({ lang: 'mojo', severity: c.severity, id: c.id, line: c.line, col: c.col, message: c.message, origin }); + for (const w of warnings(module)) diagnostics.push({ lang: 'mojo', severity: 'warning', id: w.id, line: w.line, col: w.col, message: w.message, origin }); + for (const c of check3(module)) diagnostics.push({ lang: 'mojo', severity: c.severity, id: c.id, line: c.line, col: c.col, message: c.message, origin, ...(c.question ? { question: c.question } : {}) }); + return { diagnostics, module }; +} diff --git a/src/mojo/editing.ts b/src/mojo/editing.ts new file mode 100644 index 0000000..fdc3fb7 --- /dev/null +++ b/src/mojo/editing.ts @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Enter and comment behavior for the Mojo pane. The block-opening rule is + * the official VS Code extension's (MOJO_INDENT_AFTER, harvested); the + * extension predates `comptime if` and `comptime for`, which Mojo uses since 1.1.0 in + * place of the removed `@parameter if/for`, so they are added here, and a + * line that ends its block (return, pass, raise, break, continue) dedents + * the next, as Python editors do. + */ +import { MOJO_INDENT_AFTER, MOJO_LINE_COMMENT } from './generated/editor.js'; + +const OPENS = [...MOJO_INDENT_AFTER.map((r) => new RegExp(r)), /^\s*comptime\s+(?:if|for)\b.*?:\s*$/]; +const ENDS = /^\s*(?:return|pass|raise|break|continue)\b/; + +/** The indentation for the line after `line`, given one indent step. */ +export function nextIndent(line: string, step: string): string { + const code = line.replace(/\s+#.*$/, ''); + const indent = /^\s*/.exec(line)![0]; + if (OPENS.some((r) => r.test(code))) return indent + step; + if (ENDS.test(code) && indent.length >= step.length) return indent.slice(step.length); + return indent; +} + +export const LINE_COMMENT = MOJO_LINE_COMMENT; diff --git a/src/mojo/flow.ts b/src/mojo/flow.ts new file mode 100644 index 0000000..c5239b4 --- /dev/null +++ b/src/mojo/flow.ts @@ -0,0 +1,470 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Stage 2b: what the program's own signatures and argument conventions make + * certain, in the compiler's words (godbolt's verdicts on Mojo Quest's starters + * and the compiler's own tests, tests/mojo-check.test.mjs): + * + * invalid call to 'f': missing required argument: 'x' at the call's ( + * invalid call to 'f': unexpected argument at the first extra argument + * invalid call to 'f': value passed to mutable argument 'p' must be mutable + * no matching function in initialization a Copyable/Movable struct no init fits + * cannot call function that may raise in a context that cannot raise + * expression must be mutable for in-place operator destination (+=, on an immutable value) + * expression must be mutable in assignment (=, on an immutable value) + * use of uninitialized value 'x' after x^, in straight-line code + * 'self.f' is uninitialized at the implicit return from this function + * + * "Immutable" here is only what the evidence confirms: a parameter with no + * convention (or imm), a plain for-loop variable, and a field reached through + * an immutable self. Only the student's own functions and structs are judged, + * plus the prelude functions that raise (harvested); a call through a method, + * a generic, a star argument, or an overloaded name is left to the compiler. + */ +import type { Arg, Expr, Module, Param, Pos, Stmt } from './parse.js'; +import { MOJO_PRELUDE_RAISES } from './generated/stdlib.js'; + +export interface Finding2 { severity: 'error' | 'note'; line: number; col: number; message: string; id: string } + +type DefStmt = Extract<Stmt, { k: 'def' }>; +type StructStmt = Extract<Stmt, { k: 'struct' | 'trait' }>; +type Kind = 'imm' | 'mutable' | 'loop' | 'var' | 'other'; + +const MUTABLE_CONVENTIONS = new Set(['mut', 'inout', 'var', 'owned', 'out', 'deinit']); +const PRELUDE_RAISES = new Set<string>(MOJO_PRELUDE_RAISES); + +const realParams = (ps: Param[]) => ps.filter((p) => p.star !== 'marker' && p.star !== 'positional-marker'); + +function exprStart(e: Expr): Pos { + let x: Expr = e; + for (; ;) { + if (x.k === 'call') x = x.fn; + else if (x.k === 'index' || x.k === 'attr') x = x.obj; + else if (x.k === 'binary' || x.k === 'bool') x = x.l; + else if (x.k === 'compare') x = x.operands[0]; + else return x.pos; + } +} + +/** Positional and keyword-only parameters, the varargs and positional-only boundaries. */ +function shape(ps: Param[]) { + const pos: Param[] = [], kw: Param[] = []; + let varargs = false, kwargs = false, kwOnly = false; + for (const p of ps) { + if (p.star === 'marker') { kwOnly = true; continue; } + if (p.star === 'positional-marker') continue; + if (p.star === '*') { varargs = true; kwOnly = true; continue; } + if (p.star === '**') { kwargs = true; continue; } + (kwOnly ? kw : pos).push(p); + } + return { pos, kw, varargs, kwargs }; +} + +/** How many statements one function's moved-value analysis may walk (see movedReads). */ +const MAX_VISITS = 200_000; +class TooComplex extends Error {} + +export function check2(mod: Module): Finding2[] { + if (mod.unread) return []; + const out: Finding2[] = []; + const err = (p: Pos, message: string, id: string) => out.push({ severity: 'error', line: p.line, col: p.col, message, id }); + const note = (p: Pos, message: string) => out.push({ severity: 'note', line: p.line, col: p.col, message, id: 'note' }); + + // the module's own functions and structs; a name defined twice (overloads) is not judged + const count = new Map<string, number>(); + for (const s of mod.body) if (s.k === 'def' || s.k === 'struct' || s.k === 'trait') count.set(s.name, (count.get(s.name) ?? 0) + 1); + const functions = new Map<string, DefStmt>(); + const structs = new Map<string, StructStmt>(); + for (const s of mod.body) { + if (s.k === 'def' && count.get(s.name) === 1 && !s.typeParams.length) functions.set(s.name, s); + if (s.k === 'struct' && count.get(s.name) === 1 && !s.typeParams.length) structs.set(s.name, s as StructStmt); + } + + /* ---------------- calls ---------------- */ + + function checkCall(call: Extract<Expr, { k: 'call' }>, env: Map<string, Kind>, ctx: Ctx): void { + if (call.fn.k !== 'name' || env.has(call.fn.id)) return; + const name = call.fn.id; + if (call.args.some((a) => a.star)) return; + const fn = functions.get(name); + if (fn) { + matchArgs(call, fn, env); + const never = fn.raisesType?.k === 'name' && fn.raisesType.id === 'Never'; + if (fn.raises && !never) raising(call, ctx); + return; + } + if (PRELUDE_RAISES.has(name) && !structs.has(name)) { raising(call, ctx); return; } + const st = structs.get(name); + if (st) construct(call, st); + } + + function matchArgs(call: Extract<Expr, { k: 'call' }>, fn: DefStmt, env: Map<string, Kind>): void { + const { pos, kw, varargs, kwargs } = shape(fn.params.filter((p) => p.convention !== 'out')); // the callee fills an out parameter + const filled = new Set<string>(); + const positional = call.args.filter((a) => a.name === null); + positional.forEach((a, i) => { if (i < pos.length) filled.add(pos[i].name); }); + for (const a of call.args) if (a.name !== null) { if ([...pos, ...kw].some((p) => p.name === a.name)) filled.add(a.name); else if (!kwargs) return; } + const missing = [...pos, ...kw].find((p) => !p.default && !filled.has(p.name)); + if (missing) { + err(call.lparen, `invalid call to '${fn.name}': missing required argument: '${missing.name}'`, 'missing-argument'); + note(fn.pos, 'function declared here'); + return; + } + if (positional.length > pos.length && !varargs) { + err(exprStart(positional[pos.length].value), `invalid call to '${fn.name}': unexpected argument`, 'unexpected-argument'); + note(fn.pos, 'function declared here'); + return; + } + // an immutable value handed to a mut parameter + positional.forEach((a, i) => { + const p = pos[i]; + if (!p || (p.convention !== 'mut' && p.convention !== 'inout')) return; + if (a.value.k === 'name' && (env.get(a.value.id) === 'imm' || env.get(a.value.id) === 'loop')) + err(a.value.pos, `invalid call to '${fn.name}': value passed to mutable argument '${p.name}' must be mutable`, 'immutable-to-mut'); + }); + } + + /** A Copyable/Movable struct has the compiler's copy and move constructors too, so no fit is "no matching function". */ + function construct(call: Extract<Expr, { k: 'call' }>, st: StructStmt): void { + if (call.args.some((a) => a.name !== null)) return; + const synthesized = st.bases.some((b) => b.k === 'name' && ['Copyable', 'Movable', 'ImplicitlyCopyable'].includes(b.id)); + if (!synthesized) return; + const n = call.args.length; + const fieldwise = st.decorators.some((d) => d.k === 'name' && d.id === 'fieldwise_init'); + const fields = st.body.filter((b): b is Extract<Stmt, { k: 'var' }> => b.k === 'var' && !b.comptime).length; + const inits = st.body.filter((b): b is DefStmt => b.k === 'def' && b.name === '__init__'); + if (inits.some((d) => d.typeParams.length || d.params.some((p) => p.star === '*' || p.star === '**'))) return; + const fits = (d: DefStmt) => { + const ps = shape(d.params.filter((p) => !(p.convention === 'out' || p.name === 'self'))); + if (ps.kw.some((p) => !p.default)) return false; // copy=, move=: never by position + const required = ps.pos.filter((p) => !p.default).length; + return n >= required && n <= ps.pos.length; + }; + if ((fieldwise && n === fields) || inits.some(fits)) return; + err(call.lparen, 'no matching function in initialization', 'no-matching-init'); + } + + function raising(call: Extract<Expr, { k: 'call' }>, ctx: Ctx): void { + if (ctx.canRaise || !ctx.def) return; + err(call.lparen, 'cannot call function that may raise in a context that cannot raise', 'raises'); + note(call.lparen, "try surrounding the call in a 'try' block"); + note(ctx.def.pos, "or mark surrounding function as 'raises'"); + } + + /* ---------------- walking a function ---------------- */ + + interface Ctx { def: DefStmt | null; canRaise: boolean; selfKind: Kind | null } + + function exprs(e: Expr, env: Map<string, Kind>, ctx: Ctx): void { + switch (e.k) { + case 'call': checkCall(e, env, ctx); exprs(e.fn, env, ctx); for (const a of e.args) exprs(a.value, env, ctx); return; + case 'index': exprs(e.obj, env, ctx); for (const a of e.args) exprs(a.value, env, ctx); return; + case 'attr': exprs(e.obj, env, ctx); return; + case 'list': case 'set': case 'tuple': for (const x of e.items) exprs(x, env, ctx); return; + case 'dict': for (const [k, v] of e.entries) { exprs(k, env, ctx); exprs(v, env, ctx); } return; + case 'unary': case 'not': case 'transfer': exprs(e.x, env, ctx); return; + case 'binary': case 'bool': exprs(e.l, env, ctx); exprs(e.r, env, ctx); return; + case 'compare': for (const x of e.operands) exprs(x, env, ctx); return; + case 'ifexp': exprs(e.cond, env, ctx); exprs(e.a, env, ctx); exprs(e.b, env, ctx); return; + case 'record': for (const f of e.fields) exprs(f.value, env, ctx); return; + case 'walrus': exprs(e.value, env, ctx); return; + default: return; // comprehensions and lambdas have scopes of their own: not judged here + } + } + + /** Whether a target is immutable for certain, and where the compiler points at it. */ + function immutableTarget(t: Expr, env: Map<string, Kind>, ctx: Ctx): Pos | null { + if (t.k === 'name') { if (t.id === '_') return null; const k = env.get(t.id); return k === 'imm' || k === 'loop' ? t.pos : null; } + if (t.k === 'attr' && t.obj.k === 'name' && t.obj.id === 'self' && ctx.selfKind === 'imm' && env.get('self') === 'imm') return t.dot; + return null; + } + + function stmts(list: Stmt[], env: Map<string, Kind>, ctx: Ctx): void { + list.forEach((s) => stmt(s, env, ctx)); + } + + function stmt(s: Stmt, env: Map<string, Kind>, ctx: Ctx): void { + switch (s.k) { + case 'def': return; // a nested function captures: its conventions are not judged here + case 'struct': return; + case 'var': if (s.value) exprs(s.value, env, ctx); for (const n of names(s.targets)) env.set(n, 'var'); return; + case 'assign': { + exprs(s.value, env, ctx); + for (const t of s.targets) { const p = immutableTarget(t, env, ctx); if (p) err(p, 'expression must be mutable in assignment', 'immutable-assign'); else if (t.k !== 'name') exprs(t, env, ctx); } + return; + } + case 'aug': { + exprs(s.value, env, ctx); + const p = immutableTarget(s.target, env, ctx); + if (p) err(p, 'expression must be mutable for in-place operator destination', 'immutable-inplace'); + else if (s.target.k !== 'name') exprs(s.target, env, ctx); + return; + } + case 'expr': exprs(s.e, env, ctx); return; + case 'return': case 'raise': if (s.value) exprs(s.value, env, ctx); return; + case 'assert': exprs(s.test, env, ctx); if (s.msg) exprs(s.msg, env, ctx); return; + case 'if': exprs(s.test, env, ctx); stmts(s.body, env, ctx); stmts(s.orelse, env, ctx); return; + case 'while': exprs(s.test, env, ctx); stmts(s.body, env, ctx); stmts(s.orelse, env, ctx); return; + case 'for': { + exprs(s.iter, env, ctx); + for (const n of names(s.target)) env.set(n, s.convention ? 'other' : 'loop'); + stmts(s.body, env, ctx); stmts(s.orelse, env, ctx); + return; + } + case 'try': { + stmts(s.body, env, { ...ctx, canRaise: ctx.canRaise || s.handlers.length > 0 }); + for (const h of s.handlers) { if (h.name) env.set(h.name, 'other'); stmts(h.body, env, ctx); } + stmts(s.orelse, env, ctx); stmts(s.finalbody, env, ctx); + return; + } + case 'with': for (const it of s.items) { exprs(it.e, env, ctx); if (it.as) for (const n of names(it.as)) env.set(n, 'other'); } stmts(s.body, env, ctx); return; + default: return; + } + } + + function names(e: Expr): string[] { + if (e.k === 'name') return e.id === '_' ? [] : [e.id]; + if (e.k === 'tuple' || e.k === 'list') return e.items.flatMap(names); + return []; + } + + function walk(e: Expr, f: (x: Expr) => void): void { + f(e); + switch (e.k) { + case 'call': walk(e.fn, f); e.args.forEach((a: Arg) => walk(a.value, f)); return; + case 'index': walk(e.obj, f); e.args.forEach((a) => walk(a.value, f)); return; + case 'attr': walk(e.obj, f); return; + case 'list': case 'set': case 'tuple': e.items.forEach((x) => walk(x, f)); return; + case 'dict': e.entries.forEach(([k, v]) => { walk(k, f); walk(v, f); }); return; + case 'unary': case 'not': case 'transfer': walk(e.x, f); return; + case 'binary': case 'bool': walk(e.l, f); walk(e.r, f); return; + case 'compare': e.operands.forEach((x) => walk(x, f)); return; + case 'ifexp': walk(e.cond, f); walk(e.a, f); walk(e.b, f); return; + case 'record': e.fields.forEach((x) => walk(x.value, f)); return; + case 'walrus': walk(e.value, f); return; + default: return; + } + } + + function rootName(e: Expr): string | null { + let x: Expr = e; + while (x.k === 'attr' || x.k === 'index') x = x.obj; + return x.k === 'name' ? x.id : null; + } + + function assigns(s: Stmt, x: string): boolean { + if (s.k === 'assign') return s.targets.some((t) => names(t).includes(x)); + if (s.k === 'var') return names(s.targets).includes(x); + return false; + } + + const declaredAt = new Map<string, Pos>(); + + function fn(d: DefStmt, inStruct: StructStmt | null): void { + const env = new Map<string, Kind>(); + declaredAt.clear(); + let selfKind: Kind | null = null; + const ps = realParams(d.params); + ps.forEach((p, i) => { + const k: Kind = p.convention === null || p.convention === 'imm' || p.convention === 'read' || p.convention === 'borrowed' ? 'imm' : MUTABLE_CONVENTIONS.has(p.convention) ? 'mutable' : 'other'; + const kind: Kind = p.star ? 'other' : k; + env.set(p.name, kind); + if (inStruct && i === 0 && p.name === 'self') selfKind = kind; + }); + const collect = (list: Stmt[]) => { for (const s of list) { if (s.k === 'var' && s.targets.k === 'name') declaredAt.set(s.targets.id, s.targets.pos); if ('body' in s && Array.isArray((s as any).body) && s.k !== 'def' && s.k !== 'struct') collect((s as any).body); } }; + collect(d.body); + stmts(d.body, env, { def: d, canRaise: d.raises, selfKind }); + movedReads(d, env); + if (inStruct && d.name === '__init__') outSelf(d, inStruct); + } + + /** + * A value read where it may be uninitialized, on any path: moved out, or declared without a value + * and not yet assigned: "use of uninitialized value 'x'" + * (probes check-use-after-move, check-use-after-move-on-one-path; MQ-207), at the ( of the + * innermost call that reads it. A forward analysis of which owned variables may be moved out; + * a loop is judged on its converged entry state; a path that returns or raises adds nothing; + * names moved inside a try or with are not judged (their paths are not modeled). + */ + function movedReads(d: DefStmt, env: Map<string, Kind>): void { + const owned = (n: string) => env.get(n) === 'var' || env.get(n) === 'mutable'; + const reported = new Set<string>(); + const unjudged = new Set<string>(); + const moved = new Set<string>(); // names that some path moves out (judged everywhere) + type S = Set<string> | null; // null: no path reaches here + const join = (a: S, b: S): S => (a === null ? b && new Set(b) : b === null ? new Set(a) : new Set([...a, ...b])); + const same = (a: S, b: S) => (a === null || b === null ? a === b : a.size === b.size && [...a].every((x) => b.has(x))); + + /** Reads and moves in evaluation order, each read with the ( of its innermost enclosing call. */ + type Ev = { name: string; at: Pos; move: boolean; arg?: boolean }; + const events = (e: Expr, call: Pos | null, out: Ev[]) => { + const w = (x: Expr, c: Pos | null) => events(x, c, out); + switch (e.k) { + case 'name': out.push({ name: e.id, at: call ?? e.pos, move: false }); return; + case 'transfer': if (e.x.k === 'name') out.push({ name: e.x.id, at: e.x.pos, move: true }); else w(e.x, call); return; + case 'call': { + w(e.fn, e.lparen); + // a bare argument to a call other than print may bind a ref or out parameter, which needs no value + const loose = !(e.fn.k === 'name' && e.fn.id === 'print'); + e.args.forEach((a) => { if (a.value.k === 'name' && loose) out.push({ name: a.value.id, at: e.lparen, move: false, arg: true }); else w(a.value, e.lparen); }); + return; + } + case 'index': w(e.obj, call); e.args.forEach((a) => w(a.value, call)); return; + case 'attr': w(e.obj, call); return; + case 'list': case 'set': case 'tuple': e.items.forEach((x) => w(x, call)); return; + case 'dict': e.entries.forEach(([k, v]) => { w(k, call); w(v, call); }); return; + case 'unary': case 'not': w(e.x, call); return; + case 'binary': case 'bool': w(e.l, call); w(e.r, call); return; + case 'compare': e.operands.forEach((x) => w(x, call)); return; + case 'ifexp': w(e.cond, call); w(e.a, call); w(e.b, call); return; + case 'record': e.fields.forEach((x) => w(x.value, call)); return; + case 'walrus': w(e.value, call); return; + default: return; + } + }; + const scan = (e: Expr | null, st: Set<string>, report: boolean) => { + if (!e) return; + const evs: Ev[] = []; + events(e, null, evs); + for (const ev of evs) { + if (ev.move) { if (owned(ev.name)) { st.add(ev.name); moved.add(ev.name); } continue; } + // never assigned (not moved): judged only where the value is surely needed + if (ev.arg && st.has(ev.name) && !moved.has(ev.name)) { st.delete(ev.name); continue; } + if (st.has(ev.name) && !unjudged.has(ev.name)) { + if (report && !reported.has(ev.name)) { + reported.add(ev.name); + err(ev.at, `use of uninitialized value '${ev.name}'`, 'use-after-move'); + const decl = declaredAt.get(ev.name); + if (decl) note(decl, `'${ev.name}' declared here`); + } + st.delete(ev.name); // said once; as the compiler does, not again + } + } + }; + const names = (e: Expr): string[] => (e.k === 'name' ? [e.id] : e.k === 'tuple' ? e.items.flatMap(names) : []); + const rootOf = (e: Expr): string | null => { let x: Expr = e; while (x.k === 'attr' || x.k === 'index') x = x.obj; return x.k === 'name' ? x.id : null; }; + interface Loop { breaks: S; continues: S } + // a loop's body is walked until its state settles and once more to report, so loops nested n deep walk the + // innermost body 2^n times: past MAX_VISITS statements the rest of this function is not judged (no error is + // better than a guess), as src/mojo/lifetimes.ts gives up (tests/mojo-robustness.test.mjs) + let visits = 0; + const block = (list: Stmt[], st: S, loop: Loop | null, report: boolean): S => { + visits += list.length; + if (visits > MAX_VISITS) throw new TooComplex(); + for (const s of list) { if (st === null) return null; st = one(s, st, loop, report); } + return st; + }; + const one = (s: Stmt, st0: Set<string>, loop: Loop | null, report: boolean): S => { + const st = new Set(st0); + switch (s.k) { + case 'expr': scan(s.e, st, report); return st; + case 'var': + scan(s.value, st, report); + // declared without a value: uninitialized until assigned, on every path (the compiler's same words) + for (const n of names(s.targets)) { if (s.value || s.comptime || s.ref) st.delete(n); else st.add(n); } + return st; + case 'assign': { + scan(s.value, st, report); + for (const t of s.targets) { + if (t.k === 'name' || t.k === 'tuple') { for (const n of names(t)) st.delete(n); continue; } + // setting a field of a moved value is a different error: not judged here + const r = rootOf(t); if (r && st.has(r)) { st.delete(r); unjudged.add(r); continue; } + scan(t, st, report); + } + return st; + } + case 'aug': { const r = rootOf(s.target); if (r && st.has(r) && s.target.k !== 'name') { st.delete(r); unjudged.add(r); } scan(s.value, st, report); scan(s.target, st, report); return st; } + case 'return': case 'raise': scan(s.value, st, report); return null; + case 'assert': return st; + case 'if': { scan(s.test, st, report); return join(block(s.body, st, loop, report), block(s.orelse, st, loop, report)); } + case 'break': if (loop) loop.breaks = join(loop.breaks, st); return null; + case 'continue': if (loop) loop.continues = join(loop.continues, st); return null; + case 'while': case 'for': { + if (s.k === 'for') scan(s.iter, st, report); + // the entry state of a pass: the state before the loop, or after any pass; judged once converged + let head: S = st; + for (let n = 0; n < 50; n++) { + const l: Loop = { breaks: null, continues: null }; + const after = block(s.body, s.k === 'for' ? withoutTarget(head, s.target) : head, l, false); + const next = join(join(head, after), l.continues); + if (same(next, head)) break; + head = next; + } + if (s.k === 'while') scan(s.test, head ?? new Set(), report); + const l: Loop = { breaks: null, continues: null }; + block(s.body, s.k === 'for' ? withoutTarget(head, s.target) : head, l, report); + // `while True` never ends through its test: its only ways out are its breaks + const forever = s.k === 'while' && s.test.k === 'name' && s.test.id === 'True'; + return forever ? l.breaks : join(join(head, l.breaks), block(s.orelse, head, loop, report)); + } + case 'try': case 'with': { + // their paths are not modeled: whatever they assign or move leaves the analysis + const touched = new Set<string>(); + const visit = (x: any) => { + if (!x || typeof x !== 'object') return; + if (Array.isArray(x)) { x.forEach(visit); return; } + if (x.k === 'transfer' && x.x?.k === 'name') touched.add(x.x.id); + if ((x.k === 'assign' || x.k === 'var')) for (const t of (x.targets && x.k === 'assign' ? x.targets : [x.targets])) for (const n of t ? names(t) : []) touched.add(n); + if (x.k === 'aug' && x.target) { const r = rootOf(x.target); if (r) touched.add(r); } + for (const v of Object.values(x)) if (v && typeof v === 'object') visit(v); + }; + visit(s); + for (const n of touched) { unjudged.add(n); st.delete(n); } + return st; + } + default: return st; + } + }; + const withoutTarget = (st: S, t: Expr): S => { if (!st) return st; const c = new Set(st); for (const n of names(t)) c.delete(n); return c; }; + try { block(d.body, new Set(), null, true); } catch (e) { if (!(e instanceof TooComplex)) throw e; } + } + + /** An `out self` constructor that never sets a field, with no return and nothing hiding the assignment. */ + function outSelf(d: DefStmt, st: StructStmt): void { + const self = realParams(d.params).find((p) => p.name === 'self' && p.convention === 'out'); + if (!self) return; + const fields = st.body.filter((b): b is Extract<Stmt, { k: 'var' }> => b.k === 'var' && !b.comptime && !b.value && b.targets.k === 'name'); + let opaque = false; + const set = new Set<string>(); + const scan = (list: Stmt[]) => { + for (const s of list) { + if (s.k === 'return' || s.k === 'unread' || s.k === 'def') opaque = true; + if (s.k === 'assign') for (const t of s.targets) { + if (t.k === 'name' && t.id === 'self') opaque = true; + for (const x of t.k === 'tuple' ? t.items : [t]) if (x.k === 'attr' && x.obj.k === 'name' && x.obj.id === 'self') set.add(x.name); + } + for (const e of exprsOf(s)) walk(e, (x) => { if (x.k === 'call' && x.fn.k === 'attr' && (x.fn.obj.k === 'name' && (x.fn.obj.id === 'self' || x.fn.obj.id === 'Self'))) opaque = true; if (x.k === 'name' && x.id === 'self') { /* self passed along: may initialize it */ } }); + for (const e of exprsOf(s)) walk(e, (x) => { if (x.k === 'call' && x.args.some((a) => a.value.k === 'name' && a.value.id === 'self')) opaque = true; }); + if ('body' in s && Array.isArray((s as any).body)) scan((s as any).body); + if ('orelse' in s && Array.isArray((s as any).orelse)) scan((s as any).orelse); + if (s.k === 'try') { s.handlers.forEach((h) => scan(h.body)); scan(s.finalbody); } + } + }; + scan(d.body); + if (opaque) return; + const unset = fields.map((f) => (f.targets as Extract<Expr, { k: 'name' }>).id).find((n) => !set.has(n)); + if (unset) { + err(d.pos, `'self.${unset}' is uninitialized at the implicit return from this function`, 'out-self-field'); + note(self.pos, "'self' declared here"); + } + } + + function exprsOf(s: Stmt): Expr[] { + switch (s.k) { + case 'expr': return [s.e]; + case 'var': return s.value ? [s.value] : []; + case 'assign': return [s.value, ...s.targets]; + case 'aug': return [s.value, s.target]; + case 'return': case 'raise': return s.value ? [s.value] : []; + case 'if': case 'while': return [s.test]; + case 'for': return [s.iter]; + default: return []; + } + } + + for (const s of mod.body) { + if (s.k === 'def') fn(s, null); + if (s.k === 'struct') for (const b of s.body) if (b.k === 'def') fn(b, s); + } + return out; +} diff --git a/src/mojo/flowir.ts b/src/mojo/flowir.ts new file mode 100644 index 0000000..6d14790 --- /dev/null +++ b/src/mojo/flowir.ts @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * A Mojo function in the shapes the flowchart and data-flow views read + * (src/ui/views/viz.ts, written for pseudo's syntax tree), so those views draw + * Mojo without a second implementation (Stage 5). + * + * var / = / += Assign print(...) Output + * var x = input() Input return Return + * if / elif / else If (one branch per test) + * for x in xs ForEach while While + * f(a) with a mut param MojoCall: a changes obj.append(x), a mut self method: the receiver changes + * + * A `try` is drawn as its body followed by each handler's; a nested function is not drawn. + */ +import type { Expr, Module, Stmt } from './parse.js'; + +type DefStmt = Extract<Stmt, { k: 'def' }>; + +export interface FlowRoutine { name: string; kind: 'Function' | 'main'; params: Array<{ name: string; type: string }>; body: any[]; line: number; returns: string | null; symbols: Array<{ name: string; type: string }> } + +const MUTATORS = new Set(['append', 'insert', 'extend', 'pop', 'clear', 'add', 'remove', 'discard', 'reverse', 'sort']); + +function typeText(e: Expr | null): string { + if (!e) return ''; + if (e.k === 'name') return e.id; + if (e.k === 'index' && e.obj.k === 'name') return `${e.obj.id}[${e.args.map((a) => typeText(a.value)).join(', ')}]`; + return ''; +} + +function ex(e: Expr | null): any { + if (!e) return null; + switch (e.k) { + case 'name': return { kind: 'Name', name: e.id }; + case 'call': return { kind: 'CallExpr', name: e.fn.k === 'name' ? e.fn.id : e.fn.k === 'attr' ? e.fn.name : '', args: [...(e.fn.k === 'attr' ? [ex(e.fn.obj)] : []), ...e.args.map((a) => ex(a.value))] }; + case 'index': return { kind: 'Index', target: ex(e.obj), indices: e.args.map((a) => ex(a.value)) }; + case 'attr': return { kind: 'Field', target: ex(e.obj), field: e.name }; + case 'transfer': return ex(e.x); + default: { + // operators and literals: only the names inside matter to the views + const parts: any[] = []; + for (const v of Object.values(e)) { + if (Array.isArray(v)) for (const x of v) { if (x && typeof x === 'object' && 'k' in x) parts.push(ex(x as Expr)); else if (x && typeof x === 'object' && 'value' in x) parts.push(ex((x as any).value)); } + else if (v && typeof v === 'object' && 'k' in (v as object)) parts.push(ex(v as Expr)); + } + return { kind: 'Op', parts }; + } + } +} + +/** The functions a flowchart can show, in source order (struct methods as Type.method). */ +export function mojoFunctions(mod: Module): string[] { + const out: string[] = []; + for (const s of mod.body) { if (s.k === 'def') out.push(s.name); if (s.k === 'struct') for (const b of s.body) if (b.k === 'def') out.push(`${s.name}.${b.name}`); } + return out; +} + +export function mojoRoutine(mod: Module, name?: string): FlowRoutine | null { + const defs = mod.body.filter((s): s is DefStmt => s.k === 'def'); + const methods = new Map<string, DefStmt>(); + for (const s of mod.body) if (s.k === 'struct') for (const b of s.body) if (b.k === 'def') methods.set(`${s.name}.${b.name}`, b); + const main = (name && (defs.find((d) => d.name === name) ?? methods.get(name))) || defs.find((d) => d.name === 'main') || defs[0]; + if (!main) return null; + const userDefs = new Map(defs.map((d) => [d.name, d])); + const mutSelf = new Set<string>(); + for (const s of mod.body) if (s.k === 'struct') for (const b of s.body) if (b.k === 'def' && b.params[0]?.name === 'self' && b.params[0].convention === 'mut') mutSelf.add(b.name); + const symbols: FlowRoutine['symbols'] = []; + + const stmts = (list: Stmt[]): any[] => list.flatMap((s) => stmt(s)); + const stmt = (s: Stmt): any[] => { + const line = s.pos.line; + switch (s.k) { + case 'var': { + if (s.targets.k === 'name' && s.type) symbols.push({ name: s.targets.id, type: typeText(s.type) }); + if (!s.value) return [{ kind: 'Declare', line }]; + if (s.value.k === 'call' && s.value.fn.k === 'name' && s.value.fn.id === 'input') return [{ kind: 'Input', line, target: ex(s.targets) }]; + return [{ kind: 'Assign', line, target: ex(s.targets), value: ex(s.value) }]; + } + case 'assign': return s.targets.map((t) => (s.value.k === 'call' && s.value.fn.k === 'name' && s.value.fn.id === 'input' ? { kind: 'Input', line, target: ex(t) } : { kind: 'Assign', line, target: ex(t), value: ex(s.value) })); + case 'aug': return [{ kind: 'Assign', line, target: ex(s.target), value: { kind: 'Op', parts: [ex(s.target), ex(s.value)] } }]; + case 'expr': { + const e = s.e; + if (e.k === 'call' && e.fn.k === 'name' && e.fn.id === 'print') return [{ kind: 'Output', line, values: e.args.filter((a) => !a.name).map((a) => ex(a.value)) }]; + if (e.k === 'call' && e.fn.k === 'name' && userDefs.has(e.fn.id)) { + const d = userDefs.get(e.fn.id)!; + const params = d.params.filter((p) => p.star !== 'marker' && p.star !== 'positional-marker'); + const outs = e.args.map((a, i) => (!a.name && (params[i]?.convention === 'mut' || params[i]?.convention === 'inout') ? ex(a.value) : null)).filter(Boolean); + return [{ kind: 'MojoCall', line, name: e.fn.id, args: e.args.map((a) => ex(a.value)), outs }]; + } + if (e.k === 'call' && e.fn.k === 'attr' && (MUTATORS.has(e.fn.name) || mutSelf.has(e.fn.name))) return [{ kind: 'BuiltinCall', line, args: [ex(e.fn.obj), ...e.args.map((a) => ex(a.value))] }]; + return [{ kind: 'ExprStmt', line }]; + } + case 'return': return [{ kind: 'Return', line, value: ex(s.value) }]; + case 'break': return [{ kind: 'Break', line }]; + case 'continue': return [{ kind: 'Continue', line }]; + case 'pass': return [{ kind: 'BlankStmt', line }]; + case 'assert': return [{ kind: 'Assert', line }]; + case 'raise': return [{ kind: 'Return', line, value: ex(s.value) }]; + case 'if': { + const branches = [{ line, cond: ex(s.test), body: stmts(s.body) }]; + let rest = s.orelse; + // an elif is an if in the else, on its own line + while (rest.length === 1 && rest[0].k === 'if' && rest[0].pos.line !== line && !rest[0].comptime) { + const r = rest[0]; + branches.push({ line: r.pos.line, cond: ex(r.test), body: stmts(r.body) }); + rest = r.orelse; + } + return [{ kind: 'If', line, branches, elseBody: rest.length ? stmts(rest) : null }]; + } + case 'for': return [{ kind: 'ForEach', line, var: s.target.k === 'name' ? s.target.id : '_', iterable: ex(s.iter), body: stmts(s.body) }]; + case 'while': return [{ kind: 'While', line, cond: ex(s.test), body: stmts(s.body) }]; + case 'try': return [{ kind: 'ExprStmt', line }, ...stmts(s.body), ...s.handlers.flatMap((h) => stmts(h.body)), ...stmts(s.finalbody)]; + case 'with': return [{ kind: 'ExprStmt', line }, ...stmts(s.body)]; + default: return []; + } + }; + const body = stmts(main.body); + return { + name: name && methods.has(name) ? name : main.name, kind: main.name === 'main' ? 'main' : 'Function', + params: main.params.filter((p) => !p.star && p.convention !== 'out').map((p) => ({ name: p.name, type: typeText(p.type) })), + body, line: main.pos.line, returns: typeText(main.ret) || null, symbols, + }; +} diff --git a/src/mojo/generated/editor.ts b/src/mojo/generated/editor.ts new file mode 100644 index 0000000..e5f00d8 --- /dev/null +++ b/src/mojo/generated/editor.ts @@ -0,0 +1,26 @@ +// Generated by scripts/mojo/corpus.py emit. Do not edit; change library/mojo/pin.json or re-harvest. +// From modular/vscode-mojo@e7f673213c2d: language-configuration.json (VS Code extension 26.6.1). + +/** The line comment, for toggling comments. */ +export const MOJO_LINE_COMMENT = "#"; + +/** A line matching one of these opens a block: Enter indents the next line (the extension's onEnterRules). */ +export const MOJO_INDENT_AFTER: readonly string[] = [ + "^\\s*(?:def|class|fn|struct|trait|for|if|elif|else|while|try|with|finally|except|async).*?:\\s*$" +]; + +/** Bracket pairs. */ +export const MOJO_BRACKETS: ReadonlyArray<readonly [string, string]> = [ + [ + "{", + "}" + ], + [ + "[", + "]" + ], + [ + "(", + ")" + ] +]; diff --git a/src/mojo/generated/listings.ts b/src/mojo/generated/listings.ts new file mode 100644 index 0000000..3409383 --- /dev/null +++ b/src/mojo/generated/listings.ts @@ -0,0 +1,24 @@ +// Generated by scripts/mojo/listings.mjs (run by `pixi run play`). Do not edit. +// The compiler's listing for each program in library/mojo/local/ that has a godbolt answer with assembly, +// already in the x86-64 pane's syntax and with the setup block that goes with it. +// An exercise that names a `mojoProgram` and leaves its x86-64 pane empty gets this one. + +export interface MojoListing { + /** The function the listing starts from; the user functions it calls follow it. */ + entry: string; + /** The pane's program: Intel syntax, each instruction tagged with the line of the Mojo it came from. */ + asm: string; + /** The setup block: argument registers (set to 0, to be edited) and the listing's read-only strings. */ + setup: string; + /** The compiler arguments the listing was built with. */ + build: string; +} + +export const MOJO_LISTINGS: Readonly<Record<string, MojoListing>> = { + "demo": { + "entry": "cell(Int, Int, Int)", + "asm": "cell: ; cell(Int, Int, Int)\n xor eax, eax ; mojo line 9\n cmp rdi, 0 ; mojo line 9\n cmovg rax, rdi ; mojo line 9\n cmp rsi, rax ; mojo line 11\n cmovg rax, rsi ; mojo line 11\n cmp rdx, rax ; mojo line 13\n cmovg rax, rdx ; mojo line 13\n ret ; mojo line 15\n", + "setup": "# cell(Int, Int, Int): the arguments arrive in rdi, rsi, rdx (System V); set them here\nrdi = 0\nrsi = 0\nrdx = 0", + "build": "--target-cpu x86-64-v3 --fp-mode contract=off -O0 --debug-level=line-tables" + } +}; diff --git a/src/mojo/generated/pin.ts b/src/mojo/generated/pin.ts new file mode 100644 index 0000000..3d35b1f --- /dev/null +++ b/src/mojo/generated/pin.ts @@ -0,0 +1,15 @@ +// Generated by scripts/mojo/corpus.py emit. Do not edit; change library/mojo/pin.json or re-harvest. +/** The Mojo version Programming+ models, and the compiler that is its ground truth. */ +export const MOJO_VERSION = "1.2.0"; +export const GODBOLT_BASE = "https://godbolt.org"; +export const GODBOLT_COMPILER = "mojo_1_2_0"; +/** The arguments every program is built with, on godbolt and on this machine, before a program's own (library/mojo/pin.json, build.args). */ +export const MOJO_BUILD_ARGS = "--target-cpu x86-64-v3 --fp-mode contract=off"; +/** The commit (or local zip hash) of each source the corpus was harvested from. */ +export const MOJO_SOURCES: Record<string, string> = { + "modular": "527d069ee11a41ac5320af1f9402db0a66b2b868", + "mojo-quest": "8e884a70986676cb45b0b149c073be79d9925a90", + "mojo-syntax": "48cdeac4f6b542e0b50666a5725fdfadf6188836", + "skills": "c18d0e2b34e90c1acb6fd20ab6e77605f1e91a5d", + "vscode-mojo": "e7f673213c2d44c4494af10cd307afe3b4bde0b3" +}; diff --git a/src/mojo/generated/programs.ts b/src/mojo/generated/programs.ts new file mode 100644 index 0000000..011785b --- /dev/null +++ b/src/mojo/generated/programs.ts @@ -0,0 +1,116 @@ +// Generated by scripts/mojo/corpus.py emit. Do not edit; change library/mojo/pin.json or re-harvest. +// Our own Mojo programs (library/mojo/local/), each checked on the pinned compiler by `corpus.py verify`. +// `verified` is true only when godbolt's answer passed the program's own expectation. + +export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string } + +export const MOJO_PROGRAMS: Readonly<Record<string, MojoProgram>> = { + "best-position": { + "source": "def best_position(q: List[Int]) -> Int:\n # requires: len(q) >= 1\n # ensures: 0 <= result < len(q)\n # ensures: forall k in [0, len(q)) : q[k] <= q[result]\n var best: Int = 0\n for i in range(1, len(q)):\n # invariant: 0 <= best < i\n # invariant: forall k in [0, i) : q[k] <= q[best]\n if q[i] > q[best]:\n best = i\n return best\n\n\ndef main():\n var scores: List[Int] = [12, 30, 25, 30, 8]\n print(best_position(scores))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "binary-search": { + "source": "def contains_sorted(l: List[Int], x: Int) -> Bool:\n # requires: forall k in [0, len(l) - 1) : l[k] <= l[k + 1]\n var lo: Int = 0\n var hi: Int = len(l)\n while lo < hi:\n # invariant: 0 <= lo <= hi <= len(l)\n # decreases: hi - lo\n var mid = (lo + hi) // 2\n if l[mid] == x:\n return True\n elif l[mid] < x:\n lo = mid + 1\n else:\n hi = mid\n return False\n\n\ndef main():\n var sorted_values: List[Int] = [2, 5, 8, 13, 21]\n print(contains_sorted(sorted_values, 13))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "count-base-proved": { + "source": "def count_base(b: String, t: String) -> Int:\n # ensures: result == |{ k in [0, |t|) : t[k] == b }|\n var total: Int = 0\n for i in range(t.count_codepoints()):\n # invariant: total == |{ k in [0, i) : t[k] == b }|\n if String(t[codepoint=i]) == b:\n total += 1\n return total\n\n\ndef main():\n print(count_base(\"A\", \"GATTACA\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "count-base": { + "source": "def count_base(b: String, t: String) -> Int:\n var total = 0\n for i in range(t.count_codepoints()):\n if String(t[codepoint=i]) == b:\n total += 1\n return total\n\n\ndef main():\n print(count_base(\"A\", \"GATTACA\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "demo": { + "source": "# Far below any real score: a gap state that cannot have been reached yet.\ncomptime NEG_INF = -1_000_000\n\n\ndef cell(diagonal: Int, gap_in_reference: Int, gap_in_read: Int) -> Int:\n \"\"\"The Smith-Waterman recurrence for one cell: the best of the three ways in, but never\n below 0, because a local alignment may start afresh anywhere.\"\"\"\n var best: Int = 0\n if diagonal > best:\n best = diagonal\n if gap_in_reference > best:\n best = gap_in_reference\n if gap_in_read > best:\n best = gap_in_read\n return best\n\n\ndef larger(a: Int, b: Int) -> Int:\n if a > b:\n return a\n return b\n\n\n@fieldwise_init\nstruct Scoring(Copyable, Movable):\n \"\"\"How an alignment is scored. With gap_open == gap_extend every gap letter costs the\n same (linear gaps); with gap_open below gap_extend one long gap is cheaper than\n several short ones (affine gaps), which is what real insertions and deletions look like.\n \"\"\"\n\n var name: String\n var match_score: Int\n var mismatch_score: Int\n var gap_open: Int\n var gap_extend: Int\n\n def pair(self, same: Bool) -> Int:\n if same:\n return self.match_score\n return self.mismatch_score\n\n\nstruct Grid(Movable):\n \"\"\"A dynamic-programming table that owns its memory, row by row in one List,\n and says when that memory is allocated and freed.\"\"\"\n\n var name: String\n var cols: Int\n var cells: List[Int]\n\n def __init__(out self, name: String, rows: Int, cols: Int, start: Int):\n self.name = name\n self.cols = cols\n self.cells = List[Int]()\n for _ in range(rows * cols):\n self.cells.append(start)\n print(\" + alloc\", name, String(rows) + \"x\" + String(cols))\n\n def __init__(out self, *, deinit move: Self):\n self.name = move.name^\n self.cols = move.cols\n self.cells = move.cells^\n\n def __deinit__(deinit self):\n print(\" - free \", self.name)\n\n def at(self, i: Int, j: Int) -> Int:\n return self.cells[i * self.cols + j]\n\n def put(mut self, i: Int, j: Int, value: Int):\n self.cells[i * self.cols + j] = value\n\n\n@fieldwise_init\nstruct Alignment(Copyable, Movable):\n \"\"\"The best local alignment: its score, where it sits in the reference, and the three\n printed rows (reference, match marks, read).\"\"\"\n\n var score: Int\n var start: Int\n var end: Int\n var top: String\n var marks: String\n var bottom: String\n\n\ndef align(reference: String, seq_read: String, s: Scoring) -> Alignment:\n \"\"\"Smith-Waterman local alignment with affine gaps (Gotoh's three tables).\n\n H[i][j]: the best score of an alignment ending at reference letter i and read letter j.\n E[i][j]: the same, but ending in a gap in the reference (a letter of the read left over).\n F[i][j]: the same, but ending in a gap in the read (a letter of the reference skipped).\n \"\"\"\n var n = reference.count_codepoints()\n var m = seq_read.count_codepoints()\n var h = Grid(\"H (best score ending here)\", n + 1, m + 1, 0)\n var e = Grid(\"E (ends in a gap in the reference)\", n + 1, m + 1, NEG_INF)\n var f = Grid(\"F (ends in a gap in the read)\", n + 1, m + 1, NEG_INF)\n\n # fill: every cell from its three neighbors\n var best = 0\n var best_i = 0\n var best_j = 0\n for i in range(1, n + 1):\n for j in range(1, m + 1):\n e.put(\n i,\n j,\n larger(\n e.at(i, j - 1) + s.gap_extend, h.at(i, j - 1) + s.gap_open\n ),\n )\n f.put(\n i,\n j,\n larger(\n f.at(i - 1, j) + s.gap_extend, h.at(i - 1, j) + s.gap_open\n ),\n )\n var same = reference[codepoint=i - 1] == seq_read[codepoint=j - 1]\n var diagonal = h.at(i - 1, j - 1) + s.pair(same)\n var here = cell(diagonal, e.at(i, j), f.at(i, j))\n h.put(i, j, here)\n if here > best:\n best = here\n best_i = i\n best_j = j\n\n # traceback: walk back from the best cell, switching tables where a gap opened\n var top = String(\"\")\n var marks = String(\"\")\n var bottom = String(\"\")\n var i = best_i\n var j = best_j\n var table = 0 # 0: in H, 1: in E, 2: in F\n while i > 0 and j > 0:\n if table == 0:\n if h.at(i, j) == 0:\n break\n var same = reference[codepoint=i - 1] == seq_read[codepoint=j - 1]\n if h.at(i, j) == h.at(i - 1, j - 1) + s.pair(same):\n top = String(reference[codepoint=i - 1]) + top\n bottom = String(seq_read[codepoint=j - 1]) + bottom\n if same:\n marks = \"|\" + marks\n else:\n marks = \".\" + marks\n i -= 1\n j -= 1\n elif h.at(i, j) == e.at(i, j):\n table = 1\n else:\n table = 2\n elif table == 1:\n top = \"-\" + top\n marks = \" \" + marks\n bottom = String(seq_read[codepoint=j - 1]) + bottom\n if e.at(i, j) == h.at(i, j - 1) + s.gap_open:\n table = 0\n j -= 1\n else:\n top = String(reference[codepoint=i - 1]) + top\n marks = \" \" + marks\n bottom = \"-\" + bottom\n if f.at(i, j) == h.at(i - 1, j) + s.gap_open:\n table = 0\n i -= 1\n return Alignment(best, i + 1, best_i, top^, marks^, bottom^)\n\n\ndef report(name: String, a: Alignment):\n print(\" \" + name)\n print(\" score\", a.score, \"| reference letters\", a.start, \"to\", a.end)\n print(\" reference \", a.top)\n print(\" \", a.marks)\n print(\" read \", a.bottom)\n\n\ndef main():\n # A sequencing read that lost two letters (TA) of the reference: a 2-letter deletion.\n var reference = \"TTGATTACAGGTCC\"\n var read = \"GATCAGGT\"\n print(\"read\", read, \"against reference\", reference)\n\n var affine = Scoring(\n \"affine gaps: opening a gap costs 5, each further letter 1\",\n 3,\n -2,\n -5,\n -1,\n )\n var linear = Scoring(\"linear gaps: every gap letter costs 2\", 3, -2, -2, -2)\n\n print(\"1. align with affine gaps\")\n var first = align(reference, read, affine)\n report(affine.name, first)\n\n print(\"2. align the same read with linear gaps\")\n var second = align(reference, read, linear)\n report(linear.name, second)\n\n if first.top == second.top and first.bottom == second.bottom:\n print(\"the two models agree\")\n else:\n print(\n \"the models disagree: one 2-letter deletion, or two 1-letter\"\n \" deletions?\"\n )\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "edit-distance": { + "source": "def edit_distance(a: String, b: String) -> Int:\n var n = a.count_codepoints()\n var m = b.count_codepoints()\n var d = List[List[Int]]()\n for _ in range(n + 1):\n var row = List[Int]()\n for _ in range(m + 1):\n row.append(0)\n d.append(row^)\n for i in range(n + 1):\n d[i][0] = i\n for j in range(m + 1):\n d[0][j] = j\n for i in range(1, n + 1):\n for j in range(1, m + 1):\n var cost = 1\n if a[codepoint=i - 1] == b[codepoint=j - 1]:\n cost = 0\n d[i][j] = min(\n d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + cost\n )\n return d[n][m]\n\n\ndef main():\n print(edit_distance(\"GATTACA\", \"GCATGCA\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "first-unknown": { + "source": "def first_unknown(t: String) -> Int:\n # ensures: 0 <= result <= |t|\n # ensures: forall k in [0, result) : t[k] != 'N'\n var n: Int = t.count_codepoints()\n var i: Int = 0\n while i < n and String(t[codepoint=i]) != \"N\":\n # invariant: forall k in [0, i) : t[k] != 'N'\n i += 1\n return i\n\n\ndef main():\n print(first_unknown(\"GATNACA\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "five-bugs": { + "source": "def count_gc(t: String) -> Int:\n var total: Int = 0\n for i in range(t.count_codepoints() - 1):\n total = 0\n var b: String = String(t[codepoint=i])\n if b == \"G\" or b == \"C\":\n total += 1\n return total\n\n\ndef main():\n print(count_gc(\"GATTACA\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "gc-fraction": { + "source": "def gc_fraction(t: String) -> Float64:\n # requires: |t| > 0\n var gc = 0\n for cp in t.codepoint_slices():\n var b = String(cp)\n if b == \"G\" or b == \"C\":\n gc += 1\n return Float64(gc) / Float64(t.count_codepoints())\n\n\ndef main():\n print(gc_fraction(\"GATTACA\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "hamming": { + "source": "def hamming(a: String, b: String) -> Int:\n # requires: |a| == |b|\n # ensures: result == |{ k in [0, |a|) : a[k] != b[k] }|\n var differences = 0\n for i in range(a.count_codepoints()):\n # invariant: differences == |{ k in [0, i) : a[k] != b[k] }|\n if a[codepoint=i] != b[codepoint=i]:\n differences += 1\n return differences\n\n\ndef main():\n print(hamming(\"GATTACA\", \"GACTATA\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "kmers": { + "source": "def kmer_counts(t: String, k: Int) raises -> Dict[String, Int]:\n # requires: k >= 1\n var counts = Dict[String, Int]()\n var n = t.count_codepoints()\n for i in range(n - k + 1):\n var word = String(\"\")\n for j in range(k):\n word += String(t[codepoint=i + j])\n if word in counts:\n counts[word] += 1\n else:\n counts[word] = 1\n return counts^\n\n\ndef main() raises:\n print(kmer_counts(\"GATTACAGATT\", 3))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "match-off-by-one": { + "source": "def match_pattern(p: String, t: String) -> List[Int]:\n var n = t.count_codepoints()\n var m = p.count_codepoints()\n var answer = List[Int]()\n for i in range(n):\n var matched = True\n for j in range(m):\n if t[codepoint=i + j] != p[codepoint=j]:\n matched = False\n break\n if matched:\n answer.append(i)\n return answer^\n\n\ndef main():\n print(match_pattern(\"ATT\", \"GATTACA\"))\n", + "verdict": "pass", + "verified": true, + "why": "fails as expected: exited abnormally (exit code 132)" + }, + "match-proved": { + "source": "def match_pattern(p: String, t: String) -> List[Int]:\n # requires: |p| >= 1\n # ensures: result == sorted({ s in [0, |t| - |p| + 1) : forall k in [0, |p|) : t[s + k] == p[k] })\n var n = t.count_codepoints()\n var m = p.count_codepoints()\n var answer = List[Int]()\n for i in range(n - m + 1):\n # invariant: answer == sorted({ s in [0, i) : forall k in [0, m) : t[s + k] == p[k] })\n var matched = True\n for j in range(m):\n # invariant: matched == (forall k in [0, j) : t[i + k] == p[k])\n if t[codepoint=i + j] != p[codepoint=j]:\n matched = False\n break\n if matched:\n answer.append(i)\n return answer^\n\n\ndef main():\n print(match_pattern(\"ATT\", \"GATTACA\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "match-reset": { + "source": "def match_pattern(p: String, t: String) -> List[Int]:\n var n = t.count_codepoints()\n var m = p.count_codepoints()\n var answer = List[Int]()\n var matched = True\n for i in range(n - m + 1):\n for j in range(m):\n if t[codepoint=i + j] != p[codepoint=j]:\n matched = False\n break\n if matched:\n answer.append(i)\n return answer^\n\n\ndef main():\n print(match_pattern(\"GC\", \"GATGCA\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "match": { + "source": "def match_pattern(p: String, t: String) -> List[Int]:\n var n = t.count_codepoints()\n var m = p.count_codepoints()\n var answer = List[Int]()\n for i in range(n - m + 1):\n var matched = True\n for j in range(m):\n if t[codepoint=i + j] != p[codepoint=j]:\n matched = False\n break\n if matched:\n answer.append(i)\n return answer^\n\n\ndef main():\n print(match_pattern(\"ATT\", \"GATTACA\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "revcomp": { + "source": "def reverse_complement(t: String) -> String:\n # ensures: |result| == |t|\n var answer = String(\"\")\n var i = t.count_codepoints() - 1\n while i >= 0:\n var b = String(t[codepoint=i])\n var partner = String(\"N\")\n if b == \"A\":\n partner = \"T\"\n elif b == \"T\":\n partner = \"A\"\n elif b == \"G\":\n partner = \"C\"\n elif b == \"C\":\n partner = \"G\"\n answer += partner\n i -= 1\n return answer^\n\n\ndef main():\n print(reverse_complement(\"GATTACA\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "tally": { + "source": "def tally_bases(t: String) raises -> Dict[String, Int]:\n var tally = Dict[String, Int]()\n for i in range(t.count_codepoints()):\n var b = String(t[codepoint=i])\n if b in tally:\n tally[b] += 1\n else:\n tally[b] = 1\n return tally^\n\n\ndef main() raises:\n print(tally_bases(\"GATTACA\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + }, + "translate-codons": { + "source": "def translate(t: String) raises -> String:\n var stops = [\"TAA\", \"TAG\", \"TGA\"]\n var table = {\"ATG\": \"M\", \"GCC\": \"A\", \"AAA\": \"K\", \"TTT\": \"F\", \"GGC\": \"G\"}\n var protein = String(\"\")\n var n = t.count_codepoints()\n var i = 0\n while i + 3 <= n:\n var codon = String(\"\")\n for j in range(3):\n codon += String(t[codepoint=i + j])\n if codon in stops:\n break\n elif codon in table:\n protein += table[codon]\n else:\n protein += \"X\"\n i += 3\n return protein^\n\n\ndef main() raises:\n print(translate(\"ATGGCCAAATAGTTT\"))\n", + "verdict": "pass", + "verified": true, + "why": "ok" + } +}; diff --git a/src/mojo/generated/stdlib.ts b/src/mojo/generated/stdlib.ts new file mode 100644 index 0000000..f86681a --- /dev/null +++ b/src/mojo/generated/stdlib.ts @@ -0,0 +1,652 @@ +// Generated by scripts/mojo/corpus.py emit. Do not edit; change library/mojo/pin.json or re-harvest. +// From modular/modular@527d069ee11a41ac53: stdlib/std, each package's __init__.mojo and each module's public definitions. + +/** Which standard-library package (or module) exports each public name; a name exported from two places is left out. */ +export const MOJO_STD_EXPORTS: Readonly<Record<string, string>> = { + "ASSERT_MODE": "std.builtin.debug_assert", + "Absable": "std.math", + "AddressSpace": "std.memory", + "Alignment": "std.memory", + "Allocation": "std.memory", + "AnyOrigin": "std.origin", + "AnyType": "std.traits", + "ArcPointer": "std.memory", + "Array": "std.collections", + "Atomic": "std.atomic", + "BFloat16": "std.simd", + "Batch": "std.benchmark", + "Bench": "std.benchmark", + "BenchConfig": "std.benchmark", + "BenchId": "std.benchmark", + "BenchMetric": "std.benchmark", + "Bencher": "std.benchmark", + "BenchmarkInfo": "std.benchmark", + "BinaryHeap": "std.collections", + "BitSet": "std.collections", + "BlockingScopedLock": "std.utils", + "BlockingSpinLock": "std.utils", + "Bool": "std.builtin.bool", + "Boolable": "std.builtin.bool", + "Byte": "std.simd", + "BytesIter": "std.collections.string", + "CStringSlice": "std.ffi", + "CStringSpan": "std.ffi", + "CeilDivable": "std.math", + "CeilDivableRaising": "std.math", + "Ceilable": "std.math", + "Codepoint": "std.collections.string", + "CodepointSliceIter": "std.collections.string.iterators", + "CodepointsIter": "std.collections.string", + "Comparable": "std.builtin.comparable", + "CompiledFunctionInfo": "std.compile", + "ComplexFloat32": "std.complex", + "ComplexFloat64": "std.complex", + "ComplexSIMD": "std.complex", + "ComplexScalar": "std.complex", + "ComptimeInt": "std.utils.coord", + "ContiguousSlice": "std.builtin.builtin_slice", + "ConvertibleFromPython": "std.python", + "ConvertibleToPython": "std.python", + "Coord": "std.utils.coord", + "CoordLike": "std.utils.coord", + "Copyable": "std.traits", + "CountTuple": "std.collections.counter", + "Counter": "std.collections", + "DEFAULT_LEVEL": "std.logger.logger", + "DEFAULT_RTLD": "std.ffi", + "DIR_SEPARATOR": "std.pathlib", + "DType": "std.builtin.dtype", + "DebugLevel": "std.sys.compile", + "Defaultable": "std.builtin.value", + "Deinitable": "std.traits", + "Deque": "std.collections", + "DevicePassable": "std.builtin.device_passable", + "DevicePointerLike": "std.builtin.device_passable", + "DeviceTypeEncoder": "std.builtin.device_passable", + "Dict": "std.collections", + "DictEntry": "std.collections.dict", + "DictKeyError": "std.collections.dict", + "DivModable": "std.math", + "DynamicCoord": "std.utils.coord", + "EllipsisType": "std.builtin.type_aliases", + "EmptyDictError": "std.collections.dict", + "EmptyOptionalError": "std.collections.optional", + "Endian": "std.sys", + "EnumLike": "std.format", + "Equatable": "std.builtin.comparable", + "ErrNo": "std.ffi", + "Error": "std.builtin.error", + "ExceptionType": "std.python.bindings", + "FPUtils": "std.utils.numerics", + "FastDiv": "std.utils.fast_div", + "FastMathFlag": "std.simd", + "FileHandle": "std.io", + "Float16": "std.simd", + "Float32": "std.simd", + "Float4_e2m1fn": "std.simd", + "Float64": "std.simd", + "Float8_e4m3fn": "std.simd", + "Float8_e4m3fnuz": "std.simd", + "Float8_e5m2": "std.simd", + "Float8_e5m2fnuz": "std.simd", + "Float8_e8m0fnu": "std.simd", + "FloatLiteral": "std.builtin.float_literal", + "Floatable": "std.builtin.floatable", + "FloatableRaising": "std.builtin.floatable", + "Floorable": "std.math", + "FlushDenormals": "std.utils.numerics", + "Format": "std.benchmark", + "GenericPyFunction": "std.python.bindings", + "GraphemeIndicesIter": "std.collections.string.iterators", + "GraphemeSliceIter": "std.collections.string", + "Hashable": "std.hash", + "Hasher": "std.hash", + "Identifiable": "std.builtin.identifiable", + "Idx": "std.utils.coord", + "ImmOpaquePointer": "std.memory", + "ImmOrigin": "std.origin", + "ImmPointer": "std.memory", + "ImmSpan": "std.collections", + "ImmStaticOrigin": "std.origin", + "ImmStringSlice": "std.collections.string", + "ImmStringSpan": "std.collections.string", + "ImmUnsafeAnyOrigin": "std.origin", + "ImmUntrackedOrigin": "std.origin", + "ImmutAnyOrigin": "std.origin", + "ImplicitlyCopyable": "std.traits", + "Index": "std.utils", + "IndexList": "std.utils", + "Indexer": "std.builtin.int", + "InlineLevel": "std.builtin.inline_level", + "Int": "std.simd", + "Int128": "std.simd", + "Int16": "std.simd", + "Int256": "std.simd", + "Int32": "std.simd", + "Int64": "std.simd", + "Int8": "std.simd", + "IntLiteral": "std.builtin.int_literal", + "Intable": "std.builtin.int", + "IntableRaising": "std.builtin.int", + "Interval": "std.collections.interval", + "IntervalElement": "std.collections.interval", + "IntervalTree": "std.collections.interval", + "IsTriviallyCopyable": "std.traits", + "IsTriviallyDeinitable": "std.traits", + "IsTriviallyMovable": "std.traits", + "Iterable": "std.iter", + "IterableOwned": "std.iter", + "Iterator": "std.iter", + "KGENString": "std.builtin.enum_like", + "KeyElement": "std.collections", + "Layout": "std.memory", + "Level": "std.logger", + "LinkedList": "std.collections", + "List": "std.collections", + "Logger": "std.logger", + "MAX_PATH": "std.ffi", + "MOJO_PYTHON_TYPE_OBJECTS": "std.python.bindings", + "MOJO_VERSION": "std.sys.defines", + "ManagedAllocation": "std.memory.alloc", + "MaybeUninit": "std.memory", + "Mode": "std.benchmark", + "MojoVersion": "std.sys.defines", + "Movable": "std.traits", + "MutAnyOrigin": "std.origin", + "MutOpaquePointer": "std.memory", + "MutOrigin": "std.origin", + "MutPointer": "std.memory", + "MutSpan": "std.collections", + "MutUnsafeAnyOrigin": "std.origin", + "MutUntrackedOrigin": "std.origin", + "NamedTemporaryFile": "std.tempfile", + "Never": "std.builtin.type_aliases", + "Node": "std.collections.linked_list", + "NoneType": "std.builtin.none", + "NormalRandom": "std.random", + "NumPyView": "std.python.numpy", + "O_APPEND": "std.io.file", + "O_CLOEXEC": "std.io.file", + "O_CREAT": "std.io.file", + "O_RDONLY": "std.io.file", + "O_RDWR": "std.io.file", + "O_TRUNC": "std.io.file", + "O_WRONLY": "std.io.file", + "OpaquePointer": "std.memory", + "OptimizationLevel": "std.sys.compile", + "Optional": "std.collections", + "OptionalImmPointer": "std.memory", + "OptionalMutPointer": "std.memory", + "OptionalPointer": "std.memory", + "Ordering": "std.atomic", + "Origin": "std.origin", + "OriginSet": "std.origin", + "OwnedDLHandle": "std.ffi", + "ParameterList": "std.format", + "Passwd": "std.pwd.pwd", + "PathLike": "std.os", + "Pipe": "std.os", + "Powable": "std.math", + "PrefetchCache": "std.sys", + "PrefetchLocality": "std.sys", + "PrefetchOptions": "std.sys", + "PrefetchRW": "std.sys", + "Process": "std.os", + "ProcessStatus": "std.os.process", + "PropTest": "std.testing.prop", + "PropTestConfig": "std.testing.prop", + "PyFunctionRaising": "std.python.bindings", + "PyFunctionWithKeywordsRaising": "std.python.bindings", + "PyMojoObject": "std.python.bindings", + "Python": "std.python", + "PythonModuleBuilder": "std.python.bindings", + "PythonObject": "std.python", + "PythonTypeBuilder": "std.python.bindings", + "QuickBench": "std.benchmark", + "RTLD": "std.ffi", + "Random": "std.random", + "RawFileDescriptor": "std.io", + "Reflected": "std.reflection", + "ReflectedFn": "std.reflection", + "RegisterPassable": "std.builtin.value", + "Report": "std.benchmark", + "ReversibleRange": "std.builtin.reversed", + "Roundable": "std.math", + "SEEK_CUR": "std.os", + "SEEK_END": "std.os", + "SEEK_SET": "std.os", + "SIMD": "std.simd", + "SIMDLength": "std.builtin.simd_length", + "S_IFBLK": "std.stat", + "S_IFCHR": "std.stat", + "S_IFDIR": "std.stat", + "S_IFIFO": "std.stat", + "S_IFLNK": "std.stat", + "S_IFMT": "std.stat", + "S_IFREG": "std.stat", + "S_IFSOCK": "std.stat", + "S_ISBLK": "std.stat", + "S_ISCHR": "std.stat", + "S_ISDIR": "std.stat", + "S_ISFIFO": "std.stat", + "S_ISLNK": "std.stat", + "S_ISREG": "std.stat", + "S_ISSOCK": "std.stat", + "SanitizeAddress": "std.sys.compile", + "Scalar": "std.simd", + "Set": "std.collections", + "Sized": "std.builtin.len", + "SizedRaising": "std.builtin.len", + "Slice": "std.builtin.builtin_slice", + "Some": "std.builtin.anytype", + "SomeTypeList": "std.builtin.anytype", + "SourceLocation": "std.reflection", + "SpinWaiter": "std.utils", + "StackTrace": "std.builtin.error", + "StaticCoord": "std.utils.coord", + "StaticString": "std.collections.string", + "StaticTuple": "std.utils", + "StopIteration": "std.iter", + "StridedSlice": "std.builtin.builtin_slice", + "String": "std.collections.string", + "StringDict": "std.collections", + "StringLiteral": "std.builtin.string_literal", + "StringSlice": "std.collections.string", + "StringSpan": "std.collections.string", + "TMP_MAX": "std.tempfile.tempfile", + "TString": "std.format.tstring", + "TemporaryDirectory": "std.tempfile", + "TestReport": "std.testing.suite", + "TestResult": "std.testing.suite", + "TestSuite": "std.testing", + "TestSuiteReport": "std.testing.suite", + "ThroughputMeasure": "std.benchmark", + "TrivialRegisterPassable": "std.builtin.value", + "Truncable": "std.math", + "Tuple": "std.builtin.tuple", + "TypeDict": "std.collections", + "U8x16": "std.simd", + "UInt": "std.simd", + "UInt128": "std.simd", + "UInt16": "std.simd", + "UInt256": "std.simd", + "UInt32": "std.simd", + "UInt64": "std.simd", + "UInt8": "std.simd", + "Unit": "std.benchmark", + "UnsafeAnyOrigin": "std.origin", + "UnsafePointer": "std.memory", + "UnsafeUnion": "std.ffi", + "UntrackedOrigin": "std.origin", + "VariadicList": "std.builtin.variadics", + "VariadicPack": "std.builtin.variadics", + "Variant": "std.utils", + "WeakPointer": "std.memory.arc_pointer", + "acos": "std.math", + "acosh": "std.math", + "align_down": "std.math", + "align_of": "std.sys", + "align_up": "std.math", + "all": "std.builtin.bool", + "alloc": "std.memory", + "any": "std.builtin.bool", + "argv": "std.sys", + "ascii": "std.collections.string", + "asin": "std.math", + "asinh": "std.math", + "assert_almost_equal": "std.testing", + "assert_equal": "std.testing", + "assert_equal_pyobj": "std.testing", + "assert_false": "std.testing", + "assert_is": "std.testing", + "assert_is_not": "std.testing", + "assert_not_equal": "std.testing", + "assert_raises": "std.testing", + "assert_true": "std.testing", + "assume": "std.sys.intrinsics", + "atan": "std.math", + "atan2": "std.math", + "atanh": "std.math", + "atof": "std.collections.string", + "atol": "std.collections.string", + "b16decode": "std.base64", + "b16encode": "std.base64", + "b64decode": "std.base64", + "b64encode": "std.base64", + "ballot": "std.sys.intrinsics", + "basename": "std.os.path", + "bin": "std.builtin.format_int", + "bit_not": "std.bit", + "bit_reverse": "std.bit", + "bit_width": "std.bit", + "bit_width_of": "std.sys", + "bitcast": "std.memory", + "black_box": "std.benchmark", + "breakpoint": "std.builtin.breakpoint", + "breakpointhook": "std.sys", + "byte_swap": "std.bit", + "c_char": "std.ffi", + "c_double": "std.ffi", + "c_float": "std.ffi", + "c_int": "std.ffi", + "c_long": "std.ffi", + "c_long_long": "std.ffi", + "c_pid_t": "std.ffi", + "c_short": "std.ffi", + "c_size_t": "std.ffi", + "c_ssize_t": "std.ffi", + "c_uchar": "std.ffi", + "c_uint": "std.ffi", + "c_ulong": "std.ffi", + "c_ulong_long": "std.ffi", + "c_ushort": "std.ffi", + "call_location": "std.reflection", + "cbrt": "std.math", + "ceil": "std.math", + "ceildiv": "std.math", + "chain": "std.iter", + "chdir": "std.os", + "check_and_get_arg": "std.python.bindings", + "check_and_get_or_convert_arg": "std.python.bindings", + "check_arguments_arity": "std.python.bindings", + "check_bounds": "std.collections", + "check_slice_bounds": "std.collections", + "chr": "std.collections.string", + "clamp": "std.math", + "clobber_memory": "std.benchmark", + "codegen_unreachable": "std.sys", + "comb": "std.math", + "compile_info": "std.compile", + "compressed_store": "std.sys", + "coord": "std.utils.coord", + "coord_to_index_list": "std.utils.coord", + "copy_to_numpy_array": "std.python.numpy", + "copy_to_numpy_tensor": "std.python.numpy", + "copysign": "std.math", + "cos": "std.math", + "cosh": "std.math", + "count": "std.itertools", + "count_leading_zeros": "std.bit", + "count_trailing_zeros": "std.bit", + "crd2idx": "std.utils.coord", + "current_accelerator": "std.sys", + "cwd": "std.pathlib", + "cycle": "std.itertools", + "debug_assert": "std.builtin.debug_assert", + "default_accelerator": "std.sys", + "default_comp_time_hasher": "std.hash", + "default_hasher": "std.hash", + "deinit": "std.traits.deinitable", + "dirname": "std.os.path", + "divmod": "std.math", + "dlclose": "std.ffi", + "dlerror": "std.ffi", + "dlopen": "std.ffi", + "dlsym": "std.ffi", + "doc_hidden": "std.documentation", + "downcast": "std.iter", + "drop": "std.itertools", + "drop_while": "std.itertools", + "dyn_coord": "std.utils.coord", + "e": "std.math", + "empty": "std.iter", + "enumerate": "std.iter", + "erf": "std.math", + "erfc": "std.math", + "exists": "std.os.path", + "exit": "std.sys", + "exp": "std.math", + "exp2": "std.math", + "exp_approx_f32": "std.math.math", + "expanduser": "std.os.path", + "expandvars": "std.os.path", + "expect": "std.sys.intrinsics", + "expm1": "std.math", + "factorial": "std.math", + "fence": "std.atomic", + "floor": "std.math", + "fma": "std.math", + "foo": "std.builtin.anytype", + "forget_deinit": "std.memory.memory", + "frexp": "std.math", + "from_numpy_array": "std.python.numpy", + "from_numpy_tensor": "std.python.numpy", + "gamma": "std.math", + "gather": "std.sys", + "gcd": "std.math", + "get_accum_type": "std.utils.numerics", + "get_defined_bool": "std.sys", + "get_defined_dtype": "std.sys", + "get_defined_int": "std.sys", + "get_defined_string": "std.sys", + "get_errno": "std.ffi", + "get_function_name": "std.reflection", + "get_linkage_name": "std.reflection", + "get_random": "std.ffi", + "get_static_string": "std.ffi", + "getenv": "std.os", + "getpwnam": "std.pwd", + "getpwuid": "std.pwd", + "getsize": "std.os.path", + "gettempdir": "std.tempfile", + "getuid": "std.os", + "global_constant": "std.builtin.globals", + "global_perf_counter_ns": "std.time", + "has_accelerator": "std.sys", + "has_amd_rdna_gpu_accelerator": "std.sys", + "hash": "std.hash", + "hash_bytes": "std.hash", + "hex": "std.builtin.format_int", + "hypot": "std.math", + "idx2crd": "std.utils.coord", + "implicitarg_ptr": "std.sys.intrinsics", + "index": "std.builtin.int", + "inf": "std.math", + "initialize_runtime": "std.runtime", + "inlined_assembly": "std.sys", + "input": "std.io", + "insertion_sort_threshold": "std.builtin.sort", + "iota": "std.math", + "is_32bit": "std.ffi", + "is_64bit": "std.ffi", + "is_absolute": "std.os.path", + "is_amd_gpu": "std.sys", + "is_apple_gpu": "std.sys", + "is_apple_m5": "std.sys.info", + "is_big_endian": "std.sys", + "is_defined": "std.sys", + "is_gpu": "std.sys", + "is_little_endian": "std.sys", + "is_negative": "std.bit.mask", + "is_nvidia_gpu": "std.sys", + "is_triple": "std.sys.info", + "is_trivially_copyable": "std.memory.memory", + "is_trivially_deletable": "std.memory.memory", + "is_trivially_movable": "std.memory.memory", + "isatty": "std.os", + "isclose": "std.math", + "isdir": "std.os.path", + "isfile": "std.os.path", + "isfinite": "std.math", + "isinf": "std.math", + "islink": "std.os.path", + "isnan": "std.math", + "iter": "std.iter", + "j0": "std.math", + "j1": "std.math", + "join": "std.os.path", + "keep": "std.benchmark", + "lcm": "std.math", + "ldexp": "std.math", + "len": "std.builtin.len", + "lexists": "std.os.path", + "lgamma": "std.math", + "likely": "std.sys.intrinsics", + "link": "std.os", + "listdir": "std.os", + "llvm_intrinsic": "std.sys", + "log": "std.math", + "log10": "std.math", + "log1p": "std.math", + "log2": "std.math", + "log2_ceil": "std.bit", + "log2_floor": "std.bit", + "log2e": "std.math.constants", + "logb": "std.math", + "lookup_py_type_object": "std.python.bindings", + "lstat": "std.os", + "makedirs": "std.os", + "masked_load": "std.sys", + "masked_store": "std.sys", + "materialize": "std.builtin.value", + "max": "std.math", + "max_finite": "std.utils.numerics", + "max_or_inf": "std.utils.numerics", + "memmove": "std.memory.memory", + "min": "std.math", + "min_finite": "std.utils.numerics", + "min_or_neg_inf": "std.utils.numerics", + "mkdir": "std.os", + "mkdtemp": "std.tempfile", + "modf": "std.math", + "monotonic": "std.time", + "mulhi": "std.math", + "mulwide": "std.math", + "my_assert": "std.reflection.location", + "my_func": "std.compile", + "my_function": "std.reflection.type_info", + "nan": "std.math", + "neg_inf": "std.utils.numerics", + "next": "std.iter", + "next_power_of_two": "std.bit", + "nextafter": "std.math", + "num_logical_cores": "std.sys", + "num_performance_cores": "std.sys", + "num_physical_cores": "std.sys", + "oct": "std.builtin.format_int", + "once": "std.iter", + "open": "std.io.file", + "ord": "std.collections.string", + "pack_bits": "std.memory", + "parallelism_level": "std.runtime", + "partition": "std.builtin.sort", + "path": "std.os", + "peekable": "std.iter", + "perf_counter": "std.time", + "perf_counter_ns": "std.time", + "perm": "std.math", + "pi": "std.math", + "platform_map": "std.sys.info", + "platform_type_map": "std.sys.info", + "pointer_to_int": "std.memory.unsafe_pointer", + "polynomial_evaluate": "std.math.polynomial", + "pop_count": "std.bit", + "pow": "std.math", + "prefetch": "std.sys", + "prev_power_of_two": "std.bit", + "print": "std.io", + "print_fields": "std.reflection", + "process": "std.memory.alloc", + "raise_python_exception": "std.python.bindings", + "rand": "std.random", + "randint": "std.random", + "randn": "std.random", + "randn_float64": "std.random", + "random_float64": "std.random", + "random_si64": "std.random", + "random_ui64": "std.random", + "range": "std.builtin.range", + "readfirstlane": "std.sys.intrinsics", + "realpath": "std.os.path", + "rebind": "std.builtin.rebind", + "rebind_var": "std.builtin.rebind", + "recip": "std.math", + "reflect_fn": "std.reflection", + "remainder": "std.math", + "remove": "std.os", + "removedirs": "std.os", + "repeat": "std.itertools", + "repr": "std.format", + "reversed": "std.builtin.reversed", + "rmdir": "std.os", + "rotate_bits_left": "std.bit", + "rotate_bits_right": "std.bit", + "round": "std.math", + "rsqrt": "std.math", + "scalb": "std.math", + "scatter": "std.sys", + "seed": "std.random", + "sendmsg": "std.sys.intrinsics", + "sep": "std.os", + "set_errno": "std.ffi", + "setenv": "std.os", + "shuffle": "std.random", + "simd_bit_width": "std.sys", + "simd_byte_width": "std.sys", + "simd_width_of": "std.sys", + "sin": "std.math", + "sinh": "std.math", + "sleeper": "std.benchmark", + "slice": "std.builtin.builtin_slice", + "sort": "std.builtin.sort", + "source_location": "std.reflection", + "splat": "std.bit.mask", + "split": "std.os.path", + "split_extension": "std.os.path", + "splitroot": "std.os.path", + "sqrt": "std.math", + "stack_allocation": "std.memory", + "stat": "std.os", + "stat_result": "std.os", + "stdPathLike": "std.ffi", + "stderr": "std.sys", + "stdin": "std.sys", + "stdlib_plugin": "std.sys.info", + "stdout": "std.sys", + "strided_load": "std.sys", + "strided_store": "std.sys", + "swap": "std.builtin.swap", + "symlink": "std.os", + "take": "std.itertools", + "take_while": "std.itertools", + "tan": "std.math", + "tanh": "std.math", + "tau": "std.math", + "time_function": "std.time", + "trunc": "std.math", + "ualign_down": "std.math.uutils", + "ualign_up": "std.math.uutils", + "uceildiv": "std.math.uutils", + "udiv_unchecked": "std.math.uutils", + "udivmod": "std.math.uutils", + "udivmod_unchecked": "std.math.uutils", + "ufloordiv": "std.math.uutils", + "ulp": "std.math", + "umod": "std.math.uutils", + "unlikely": "std.sys.intrinsics", + "unlink": "std.os", + "unsafe_alloc": "std.memory.alloc", + "unsafe_cast": "std.ffi", + "unsafe_destroy_n": "std.memory.memory", + "unsafe_memcmp": "std.memory", + "unsafe_memcpy": "std.memory", + "unsafe_memmove": "std.memory.memory", + "unsafe_memset": "std.memory.memory", + "unsafe_memset_zero": "std.memory.memory", + "unsafe_stack_allocation": "std.memory", + "unsafe_uninit_copy_n": "std.memory.memory", + "unsafe_uninit_move_n": "std.memory.memory", + "unsetenv": "std.os", + "y0": "std.math", + "y1": "std.math", + "zip": "std.iter" +}; + +/** Prelude functions every overload of which raises (input, atol, ...): calling one needs raises or a try. */ +export const MOJO_PRELUDE_RAISES: readonly string[] = [ + "atof", + "atol", + "input", + "next", + "open" +]; diff --git a/src/mojo/generated/tokens.ts b/src/mojo/generated/tokens.ts new file mode 100644 index 0000000..7d068a3 --- /dev/null +++ b/src/mojo/generated/tokens.ts @@ -0,0 +1,347 @@ +// Generated by scripts/mojo/corpus.py emit. Do not edit; change library/mojo/pin.json or re-harvest. +// From modular/modular@527d069ee11a41ac53: TokenKinds.def, lib/MojoParser, the decorator reference, and the stdlib prelude. + +/** Hard keywords: reserved words in every context. */ +export const MOJO_KEYWORDS: readonly string[] = [ + "False", + "None", + "Self", + "True", + "_", + "and", + "as", + "assert", + "break", + "case", + "class", + "comptime", + "conforms_to", + "continue", + "def", + "del", + "elif", + "else", + "except", + "finally", + "for", + "from", + "global", + "if", + "import", + "in", + "is", + "lambda", + "match", + "nonlocal", + "not", + "or", + "origin_of", + "pass", + "raise", + "ref", + "return", + "struct", + "trait", + "try", + "type_of", + "var", + "while", + "with", + "yield" +]; + +/** Compiler-internal keywords (they start with __); valid, but not for students. */ +export const MOJO_INTERNAL_KEYWORDS: readonly string[] = [ + "__async", + "__await", + "__enum", + "__extension", + "__functions_in_module", + "__generator_type", + "__get_address_as_owned_value", + "__get_address_as_uninit_lvalue", + "__get_current_function_name", + "__get_litref_as_mvalue", + "__get_mvalue_as_litref", + "__is_run_in_comptime_interpreter", + "__match", + "__mlir_region", + "__mojo_crash", + "__struct_field_ref" +]; + +/** Words the parser accepts by spelling in one context, such as argument conventions. */ +export const MOJO_SOFT_KEYWORDS: readonly string[] = [ + "deinit", + "imm", + "mut", + "out", + "where" +]; + +/** Words the pinned compiler rejects as removed, and what to write instead. */ +export const MOJO_REMOVED_WORDS: Readonly<Record<string, string>> = { + "fn": "def", + "read": "imm" +}; + +/** Removed words that are tokens of their own: an error wherever they appear. */ +export const MOJO_REMOVED_KEYWORDS: readonly string[] = [ + "fn" +]; + +/** Removed argument conventions: an error only in front of a parameter. */ +export const MOJO_REMOVED_CONVENTIONS: readonly string[] = [ + "read" +]; + +/** The compiler's own words for each removed word, from its source. */ +export const MOJO_REMOVED_MESSAGES: Readonly<Record<string, string>> = { + "fn": "'fn' has been removed; use 'def' instead", + "read": "'read' was removed; use 'imm'" +}; + +/** Contextual words in a function signature. */ +export const MOJO_EFFECTS: readonly string[] = [ + "raises", + "capturing", + "abi", + "thin", + "where", + "escaping" +]; + +/** Decorators documented in the language reference. */ +export const MOJO_DECORATORS: readonly string[] = [ + "align", + "always_inline", + "copy_capture", + "deprecated", + "doc_hidden", + "export", + "fieldwise_init", + "implicit", + "inline", + "no_inline", + "parameter", + "staticmethod" +]; + +/** Names every module sees without an import. */ +export const MOJO_PRELUDE: readonly string[] = [ + "Absable", + "AddressSpace", + "AnyOrigin", + "AnyType", + "Array", + "BFloat16", + "Bool", + "Boolable", + "Byte", + "Codepoint", + "Comparable", + "Copyable", + "DType", + "Defaultable", + "Deinitable", + "Dict", + "EnumLike", + "Equatable", + "Error", + "FileHandle", + "Float16", + "Float32", + "Float4_e2m1fn", + "Float64", + "Float8_e4m3fn", + "Float8_e4m3fnuz", + "Float8_e5m2", + "Float8_e5m2fnuz", + "Float8_e8m0fnu", + "FloatLiteral", + "Floatable", + "FloatableRaising", + "Hashable", + "Identifiable", + "ImmOpaquePointer", + "ImmOrigin", + "ImmPointer", + "ImmSpan", + "ImmStaticOrigin", + "ImmStringSlice", + "ImmStringSpan", + "ImmUnsafeAnyOrigin", + "ImmUntrackedOrigin", + "ImmutAnyOrigin", + "ImplicitlyCopyable", + "Indexer", + "InlineLevel", + "Int", + "Int128", + "Int16", + "Int256", + "Int32", + "Int64", + "Int8", + "IntLiteral", + "Intable", + "IntableRaising", + "Iterable", + "IterableOwned", + "Iterator", + "KeyElement", + "List", + "Movable", + "MutAnyOrigin", + "MutOpaquePointer", + "MutOrigin", + "MutPointer", + "MutSpan", + "MutUnsafeAnyOrigin", + "MutUntrackedOrigin", + "Never", + "NoneType", + "OpaquePointer", + "Optional", + "OptionalImmPointer", + "OptionalMutPointer", + "OptionalPointer", + "Origin", + "OriginSet", + "ParameterList", + "Pointer", + "Powable", + "RawFileDescriptor", + "RegisterPassable", + "ReversibleRange", + "Roundable", + "SIMD", + "SIMDLength", + "Scalar", + "Sized", + "SizedRaising", + "Slice", + "Some", + "SomeTypeList", + "Span", + "StaticString", + "StopIteration", + "String", + "StringLiteral", + "StringSlice", + "StringSpan", + "TrivialRegisterPassable", + "Tuple", + "TypeList", + "UInt", + "UInt128", + "UInt16", + "UInt256", + "UInt32", + "UInt64", + "UInt8", + "UnsafeAnyOrigin", + "UnsafePointer", + "UntrackedOrigin", + "VariadicList", + "VariadicPack", + "Writable", + "Writer", + "abs", + "all", + "alloc", + "any", + "ascii", + "atof", + "atol", + "bin", + "breakpoint", + "chr", + "debug_assert", + "deinit", + "divmod", + "doc_hidden", + "enumerate", + "hash", + "hash_bytes", + "hex", + "index", + "input", + "iter", + "len", + "map", + "materialize", + "max", + "min", + "next", + "oct", + "open", + "ord", + "partition", + "pow", + "print", + "range", + "rebind", + "rebind_var", + "reflect", + "repr", + "reversed", + "round", + "slice", + "sort", + "swap", + "zip" +]; + +/** Operator and punctuation spellings, longest first, for a maximal-munch lexer. */ +export const MOJO_PUNCTUATION: readonly string[] = [ + "**=", + "...", + "//=", + "<<=", + ">>=", + "!=", + "%=", + "&=", + "**", + "*=", + "+=", + "-=", + "->", + "//", + "/=", + ":=", + "<<", + "<=", + "<>", + "==", + ">=", + ">>", + "@=", + "^=", + "|=", + "%", + "&", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + ":", + ";", + "<", + "=", + ">", + "@", + "[", + "]", + "^", + "_", + "{", + "|", + "}", + "~" +]; diff --git a/src/mojo/generated/types.ts b/src/mojo/generated/types.ts new file mode 100644 index 0000000..3f7674b --- /dev/null +++ b/src/mojo/generated/types.ts @@ -0,0 +1,1772 @@ +// Generated by scripts/mojo/corpus.py emit. Do not edit; change library/mojo/pin.json or re-harvest. +// From modular/modular@527d069ee11a41ac53: the struct and trait declarations in stdlib/std. + +/** Each standard trait: the traits it refines, and the methods it gives a conforming struct by default. */ +export const MOJO_STD_TRAITS: Readonly<Record<string, { bases: readonly string[]; defaults: readonly string[] }>> = { + "Absable": { + "bases": [], + "defaults": [] + }, + "AnyType": { + "bases": [], + "defaults": [] + }, + "Boolable": { + "bases": [], + "defaults": [] + }, + "CeilDivable": { + "bases": [], + "defaults": [] + }, + "CeilDivableRaising": { + "bases": [], + "defaults": [] + }, + "Ceilable": { + "bases": [], + "defaults": [] + }, + "Comparable": { + "bases": [ + "Equatable" + ], + "defaults": [ + "__ge__", + "__gt__", + "__le__" + ] + }, + "ConvertibleFromPython": { + "bases": [ + "Copyable", + "Deinitable" + ], + "defaults": [] + }, + "ConvertibleToPython": { + "bases": [ + "Deinitable" + ], + "defaults": [] + }, + "CoordLike": { + "bases": [ + "Defaultable", + "ImplicitlyCopyable", + "Intable", + "TrivialRegisterPassable", + "Writable" + ], + "defaults": [ + "__int__" + ] + }, + "Copyable": { + "bases": [ + "Movable" + ], + "defaults": [ + "copy" + ] + }, + "Defaultable": { + "bases": [], + "defaults": [ + "__init__" + ] + }, + "Deinitable": { + "bases": [], + "defaults": [] + }, + "DevicePassable": { + "bases": [], + "defaults": [ + "_is_convertible_to_device_type", + "_is_implicitly_encodable_to" + ] + }, + "DevicePointerLike": { + "bases": [], + "defaults": [ + "_buffer_handle" + ] + }, + "DeviceTypeEncoder": { + "bases": [], + "defaults": [ + "encode", + "encode_array", + "encode_closure_state", + "encode_fields", + "encode_static_tuple", + "target" + ] + }, + "DivModable": { + "bases": [ + "ImplicitlyCopyable" + ], + "defaults": [] + }, + "EnumLike": { + "bases": [], + "defaults": [] + }, + "Equatable": { + "bases": [], + "defaults": [ + "__eq__", + "__ne__" + ] + }, + "Floatable": { + "bases": [], + "defaults": [] + }, + "FloatableRaising": { + "bases": [], + "defaults": [] + }, + "Floorable": { + "bases": [], + "defaults": [] + }, + "Hashable": { + "bases": [], + "defaults": [ + "__hash__" + ] + }, + "Hasher": { + "bases": [], + "defaults": [] + }, + "Identifiable": { + "bases": [], + "defaults": [ + "__isnot__" + ] + }, + "ImplicitlyCopyable": { + "bases": [ + "Copyable" + ], + "defaults": [] + }, + "Indexer": { + "bases": [], + "defaults": [] + }, + "Intable": { + "bases": [], + "defaults": [] + }, + "IntableRaising": { + "bases": [], + "defaults": [] + }, + "IntervalElement": { + "bases": [ + "Comparable", + "Copyable", + "Deinitable", + "Intable", + "Writable" + ], + "defaults": [] + }, + "Iterable": { + "bases": [], + "defaults": [] + }, + "IterableOwned": { + "bases": [], + "defaults": [] + }, + "Iterator": { + "bases": [ + "Deinitable", + "Movable" + ], + "defaults": [ + "bounds", + "nth" + ] + }, + "Movable": { + "bases": [], + "defaults": [] + }, + "PathLike": { + "bases": [], + "defaults": [] + }, + "Powable": { + "bases": [], + "defaults": [] + }, + "RegisterPassable": { + "bases": [ + "Movable" + ], + "defaults": [] + }, + "ReversibleRange": { + "bases": [], + "defaults": [] + }, + "Roundable": { + "bases": [], + "defaults": [] + }, + "Sized": { + "bases": [], + "defaults": [] + }, + "SizedRaising": { + "bases": [], + "defaults": [] + }, + "Strategy": { + "bases": [ + "Deinitable", + "Movable" + ], + "defaults": [] + }, + "TrivialRegisterPassable": { + "bases": [ + "Deinitable", + "ImplicitlyCopyable", + "Movable", + "RegisterPassable" + ], + "defaults": [] + }, + "Truncable": { + "bases": [], + "defaults": [] + }, + "UnsafeCustomNicheStorage": { + "bases": [], + "defaults": [] + }, + "UnsafeNicheable": { + "bases": [], + "defaults": [] + }, + "UnsafeSingleNicheable": { + "bases": [ + "UnsafeNicheable" + ], + "defaults": [ + "classify_niche", + "niche_count", + "write_niche" + ] + }, + "Writable": { + "bases": [], + "defaults": [ + "write_repr_to", + "write_to" + ] + }, + "Writer": { + "bases": [], + "defaults": [ + "write" + ] + } +}; + +/** Each standard struct: the traits it declares, and whether they reach ImplicitlyCopyable. */ +export const MOJO_STD_STRUCTS: Readonly<Record<string, { bases: readonly string[]; implicitlyCopyable: boolean }>> = { + "AHasher": { + "bases": [ + "Defaultable", + "Hasher" + ], + "implicitlyCopyable": false + }, + "AddressSpace": { + "bases": [ + "EnumLike", + "Equatable", + "ImplicitlyCopyable", + "Intable", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Alignment": { + "bases": [ + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Allocation": { + "bases": [ + "RegisterPassable", + "Sized", + "Writable" + ], + "implicitlyCopyable": false + }, + "ArcPointer": { + "bases": [ + "Equatable", + "Hashable", + "Identifiable", + "ImplicitlyCopyable", + "RegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Array": { + "bases": [ + "Comparable", + "Copyable", + "Defaultable", + "Deinitable", + "DevicePassable", + "Equatable", + "Hashable", + "Iterable", + "Movable", + "Sized", + "Writable" + ], + "implicitlyCopyable": false + }, + "Atomic": { + "bases": [], + "implicitlyCopyable": false + }, + "Batch": { + "bases": [ + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Bench": { + "bases": [ + "Writable" + ], + "implicitlyCopyable": false + }, + "BenchConfig": { + "bases": [ + "Copyable" + ], + "implicitlyCopyable": false + }, + "BenchId": { + "bases": [ + "Copyable", + "Equatable", + "Hashable", + "Writable" + ], + "implicitlyCopyable": false + }, + "BenchMetric": { + "bases": [ + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Bencher": { + "bases": [ + "RegisterPassable" + ], + "implicitlyCopyable": false + }, + "BenchmarkInfo": { + "bases": [ + "Copyable" + ], + "implicitlyCopyable": false + }, + "BinaryHeap": { + "bases": [ + "Copyable", + "Defaultable", + "Sized" + ], + "implicitlyCopyable": false + }, + "BitSet": { + "bases": [ + "Boolable", + "Copyable", + "Defaultable", + "Sized", + "Writable" + ], + "implicitlyCopyable": false + }, + "BlockingScopedLock": { + "bases": [], + "implicitlyCopyable": false + }, + "BlockingSpinLock": { + "bases": [ + "Defaultable" + ], + "implicitlyCopyable": false + }, + "Bool": { + "bases": [ + "Boolable", + "Comparable", + "ConvertibleFromPython", + "Defaultable", + "EnumLike", + "Floatable", + "Hashable", + "ImplicitlyCopyable", + "Intable", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Buffer": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "BufferMode": { + "bases": [], + "implicitlyCopyable": false + }, + "BytesIter": { + "bases": [ + "ImplicitlyCopyable", + "Iterable", + "Iterator", + "Sized" + ], + "implicitlyCopyable": true + }, + "CPython": { + "bases": [ + "Defaultable", + "Movable" + ], + "implicitlyCopyable": false + }, + "CStringSpan": { + "bases": [ + "Equatable", + "ImplicitlyCopyable", + "Sized", + "TrivialRegisterPassable", + "UnsafeNicheable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Codepoint": { + "bases": [ + "Comparable", + "ImplicitlyCopyable", + "Intable", + "Movable", + "Writable" + ], + "implicitlyCopyable": true + }, + "CodepointSliceIter": { + "bases": [ + "ImplicitlyCopyable", + "Iterable", + "Iterator", + "Sized" + ], + "implicitlyCopyable": true + }, + "CodepointsIter": { + "bases": [ + "ImplicitlyCopyable", + "Iterable", + "Iterator", + "Sized" + ], + "implicitlyCopyable": true + }, + "Color": { + "bases": [ + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "CompilationTarget": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "CompiledFunctionInfo": { + "bases": [ + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "ComplexSIMD": { + "bases": [ + "Equatable", + "TrivialRegisterPassable", + "Writable", + "_Expable" + ], + "implicitlyCopyable": true + }, + "ComptimeInt": { + "bases": [ + "CoordLike" + ], + "implicitlyCopyable": true + }, + "ContiguousSlice": { + "bases": [ + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "ControlOffset": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "ControlWidth": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "Coord": { + "bases": [ + "CoordLike", + "DevicePassable", + "Sized", + "Writable" + ], + "implicitlyCopyable": true + }, + "Coroutine": { + "bases": [ + "RegisterPassable" + ], + "implicitlyCopyable": false + }, + "CountTuple": { + "bases": [ + "Comparable", + "Copyable" + ], + "implicitlyCopyable": false + }, + "Counter": { + "bases": [ + "Boolable", + "Copyable", + "Defaultable", + "Equatable", + "Iterable", + "IterableOwned", + "Sized", + "Writable" + ], + "implicitlyCopyable": false + }, + "DType": { + "bases": [ + "Equatable", + "Hashable", + "ImplicitlyCopyable", + "KeyElement", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Deque": { + "bases": [ + "Boolable", + "Copyable", + "Deinitable", + "Equatable", + "Hashable", + "Iterable", + "IterableOwned", + "Movable", + "Sized", + "Writable" + ], + "implicitlyCopyable": false + }, + "DescriptorOffset": { + "bases": [], + "implicitlyCopyable": false + }, + "DescriptorWidth": { + "bases": [], + "implicitlyCopyable": false + }, + "Dict": { + "bases": [ + "Boolable", + "Copyable", + "Defaultable", + "Deinitable", + "Equatable", + "Hashable", + "Iterable", + "Movable", + "Sized", + "Writable" + ], + "implicitlyCopyable": false + }, + "DictKeyError": { + "bases": [ + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "EmptyDictError": { + "bases": [ + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "EmptyOptionalError": { + "bases": [ + "ImplicitlyCopyable", + "RegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Endian": { + "bases": [ + "EnumLike", + "Equatable", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "ErrNo": { + "bases": [ + "Equatable", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Error": { + "bases": [ + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "ExceptionType": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "ExternalFunction": { + "bases": [], + "implicitlyCopyable": false + }, + "FP": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "FPUtils": { + "bases": [], + "implicitlyCopyable": false + }, + "FastDiv": { + "bases": [ + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "FastMathFlag": { + "bases": [ + "EnumLike", + "Equatable", + "ImplicitlyCopyable", + "RegisterPassable" + ], + "implicitlyCopyable": true + }, + "FcntlCommands": { + "bases": [], + "implicitlyCopyable": false + }, + "FcntlFDFlags": { + "bases": [], + "implicitlyCopyable": false + }, + "FileHandle": { + "bases": [ + "Defaultable", + "Movable", + "Writer" + ], + "implicitlyCopyable": false + }, + "FloatLiteral": { + "bases": [ + "Boolable", + "Defaultable", + "Floatable", + "Intable", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "FlushDenormals": { + "bases": [ + "Defaultable" + ], + "implicitlyCopyable": false + }, + "Fnv1a": { + "bases": [ + "Defaultable", + "Hasher" + ], + "implicitlyCopyable": false + }, + "Format": { + "bases": [ + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "FormatStruct": { + "bases": [ + "Movable" + ], + "implicitlyCopyable": false + }, + "FprintfCtrl": { + "bases": [], + "implicitlyCopyable": false + }, + "GILAcquired": { + "bases": [ + "Movable" + ], + "implicitlyCopyable": false + }, + "GILReleased": { + "bases": [ + "Movable" + ], + "implicitlyCopyable": false + }, + "GraphemeIndicesIter": { + "bases": [ + "ImplicitlyCopyable", + "Iterable", + "Iterator" + ], + "implicitlyCopyable": true + }, + "GraphemeSliceIter": { + "bases": [ + "ImplicitlyCopyable", + "Iterable", + "Iterator", + "Sized" + ], + "implicitlyCopyable": true + }, + "Group": { + "bases": [ + "Copyable", + "Movable" + ], + "implicitlyCopyable": false + }, + "Header": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "IndexList": { + "bases": [ + "Comparable", + "Defaultable", + "DevicePassable", + "Hashable", + "ImplicitlyCopyable", + "Sized", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "InlineLevel": { + "bases": [ + "Equatable", + "ImplicitlyCopyable", + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "IntLiteral": { + "bases": [ + "Boolable", + "Ceilable", + "Defaultable", + "Floorable", + "Indexer", + "Intable", + "TrivialRegisterPassable", + "Truncable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Interval": { + "bases": [ + "Boolable", + "Equatable", + "ImplicitlyCopyable", + "Sized", + "Writable" + ], + "implicitlyCopyable": true + }, + "IntervalTree": { + "bases": [ + "Defaultable", + "Writable" + ], + "implicitlyCopyable": false + }, + "Layout": { + "bases": [ + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Level": { + "bases": [ + "Comparable", + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "LinkedList": { + "bases": [ + "Boolable", + "Copyable", + "Defaultable", + "Deinitable", + "Equatable", + "Hashable", + "Iterable", + "IterableOwned", + "Movable", + "Sized", + "Writable" + ], + "implicitlyCopyable": false + }, + "List": { + "bases": [ + "Boolable", + "Copyable", + "Defaultable", + "Deinitable", + "Equatable", + "Hashable", + "Iterable", + "IterableOwned", + "Movable", + "Sized", + "Writable" + ], + "implicitlyCopyable": false + }, + "Logger": { + "bases": [ + "ImplicitlyCopyable" + ], + "implicitlyCopyable": true + }, + "ManagedAllocation": { + "bases": [ + "RegisterPassable", + "Writable" + ], + "implicitlyCopyable": false + }, + "MaybeUninit": { + "bases": [ + "Defaultable", + "Deinitable", + "ImplicitlyCopyable", + "Movable", + "RegisterPassable" + ], + "implicitlyCopyable": true + }, + "Mode": { + "bases": [ + "ImplicitlyCopyable" + ], + "implicitlyCopyable": true + }, + "MojoVersion": { + "bases": [ + "ImplicitlyCopyable", + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "Named": { + "bases": [ + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "NamedTemporaryFile": { + "bases": [ + "Movable" + ], + "implicitlyCopyable": false + }, + "NicheIndex": { + "bases": [ + "Equatable", + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "Node": { + "bases": [ + "Deinitable", + "Movable" + ], + "implicitlyCopyable": false + }, + "NoneType": { + "bases": [ + "Defaultable", + "ImplicitlyCopyable", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "NormalRandom": { + "bases": [ + "Copyable" + ], + "implicitlyCopyable": false + }, + "NumPyView": { + "bases": [ + "ImplicitlyCopyable" + ], + "implicitlyCopyable": true + }, + "Optional": { + "bases": [ + "Boolable", + "Copyable", + "Defaultable", + "Deinitable", + "DevicePassable", + "EnumLike", + "Equatable", + "Hashable", + "ImplicitlyCopyable", + "Iterable", + "IterableOwned", + "Movable", + "RegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "OptionalReg": { + "bases": [ + "Boolable", + "Defaultable", + "DevicePassable", + "EnumLike", + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "Ordering": { + "bases": [ + "Equatable", + "ImplicitlyCopyable", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Origin": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "OwnedDLHandle": { + "bases": [ + "Boolable", + "Movable" + ], + "implicitlyCopyable": false + }, + "OwnedPointer": { + "bases": [ + "Deinitable", + "RegisterPassable", + "Writable" + ], + "implicitlyCopyable": false + }, + "ParameterList": { + "bases": [ + "Sized", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Passwd": { + "bases": [ + "Copyable", + "Writable" + ], + "implicitlyCopyable": false + }, + "Path": { + "bases": [ + "Boolable", + "Comparable", + "Hashable", + "ImplicitlyCopyable", + "KeyElement", + "PathLike", + "Writable" + ], + "implicitlyCopyable": true + }, + "Payload": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "Pipe": { + "bases": [], + "implicitlyCopyable": false + }, + "Pointer": { + "bases": [ + "Comparable", + "DevicePassable", + "ImplicitlyCopyable", + "Intable", + "TrivialRegisterPassable", + "UnsafeCustomNicheStorage", + "UnsafeSingleNicheable", + "Writable" + ], + "implicitlyCopyable": true + }, + "PrefetchCache": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "PrefetchLocality": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "PrefetchOptions": { + "bases": [ + "Defaultable", + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "PrefetchRW": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "Process": { + "bases": [], + "implicitlyCopyable": false + }, + "ProcessStatus": { + "bases": [ + "Copyable", + "ImplicitlyCopyable", + "Movable" + ], + "implicitlyCopyable": true + }, + "Progress": { + "bases": [ + "ImplicitlyCopyable" + ], + "implicitlyCopyable": true + }, + "PropTest": { + "bases": [ + "Movable" + ], + "implicitlyCopyable": false + }, + "PropTestConfig": { + "bases": [ + "Copyable" + ], + "implicitlyCopyable": false + }, + "PyGILState_STATE": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "PyMethodDef": { + "bases": [ + "Defaultable", + "ImplicitlyCopyable" + ], + "implicitlyCopyable": true + }, + "PyModuleDef": { + "bases": [ + "Movable", + "Writable" + ], + "implicitlyCopyable": false + }, + "PyModuleDef_Base": { + "bases": [ + "Defaultable", + "Movable", + "Writable" + ], + "implicitlyCopyable": false + }, + "PyModuleDef_Slot": { + "bases": [], + "implicitlyCopyable": false + }, + "PyMojoObject": { + "bases": [ + "Movable" + ], + "implicitlyCopyable": false + }, + "PyObject": { + "bases": [ + "Defaultable", + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "PyObjectPtr": { + "bases": [ + "Boolable", + "Defaultable", + "Equatable", + "ImplicitlyCopyable", + "Intable", + "RegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "PyThreadState": { + "bases": [], + "implicitlyCopyable": false + }, + "PyTypeObject": { + "bases": [], + "implicitlyCopyable": false + }, + "PyType_Slot": { + "bases": [ + "ImplicitlyCopyable", + "RegisterPassable" + ], + "implicitlyCopyable": true + }, + "PyType_Spec": { + "bases": [ + "ImplicitlyCopyable", + "RegisterPassable" + ], + "implicitlyCopyable": true + }, + "Python": { + "bases": [ + "Defaultable", + "ImplicitlyCopyable" + ], + "implicitlyCopyable": true + }, + "PythonModuleBuilder": { + "bases": [], + "implicitlyCopyable": false + }, + "PythonObject": { + "bases": [ + "Boolable", + "Defaultable", + "Identifiable", + "ImplicitlyCopyable", + "Movable", + "RegisterPassable", + "SizedRaising", + "Writable" + ], + "implicitlyCopyable": true + }, + "PythonTypeBuilder": { + "bases": [ + "Copyable" + ], + "implicitlyCopyable": false + }, + "PythonVersion": { + "bases": [ + "ImplicitlyCopyable", + "RegisterPassable" + ], + "implicitlyCopyable": true + }, + "QuickBench": { + "bases": [], + "implicitlyCopyable": false + }, + "RTLD": { + "bases": [], + "implicitlyCopyable": false + }, + "RaisingCoroutine": { + "bases": [ + "RegisterPassable" + ], + "implicitlyCopyable": false + }, + "RaisingTask": { + "bases": [], + "implicitlyCopyable": false + }, + "Random": { + "bases": [ + "Copyable" + ], + "implicitlyCopyable": false + }, + "RawFileDescriptor": { + "bases": [ + "TrivialRegisterPassable", + "Writer" + ], + "implicitlyCopyable": true + }, + "Reflected": { + "bases": [], + "implicitlyCopyable": false + }, + "ReflectedFn": { + "bases": [], + "implicitlyCopyable": false + }, + "Report": { + "bases": [ + "Copyable", + "Defaultable" + ], + "implicitlyCopyable": false + }, + "Repr": { + "bases": [ + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Rng": { + "bases": [ + "Movable" + ], + "implicitlyCopyable": false + }, + "SIMD": { + "bases": [ + "Absable", + "Boolable", + "CeilDivable", + "Ceilable", + "Comparable", + "ConvertibleFromPython", + "CoordLike", + "Defaultable", + "DevicePassable", + "DivModable", + "Equatable", + "Floorable", + "Hashable", + "Indexer", + "Intable", + "IntervalElement", + "Powable", + "Roundable", + "Sized", + "TrivialRegisterPassable", + "Truncable", + "Writable", + "_FromInt" + ], + "implicitlyCopyable": true + }, + "SIMDLength": { + "bases": [ + "Comparable", + "Equatable", + "ImplicitlyCopyable", + "Indexer", + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "ServiceId": { + "bases": [], + "implicitlyCopyable": false + }, + "Set": { + "bases": [ + "Boolable", + "Comparable", + "Copyable", + "Deinitable", + "Equatable", + "Hashable", + "Iterable", + "IterableOwned", + "Movable", + "Sized", + "Writable" + ], + "implicitlyCopyable": false + }, + "SignalCodes": { + "bases": [], + "implicitlyCopyable": false + }, + "Slice": { + "bases": [ + "Equatable", + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "SourceLocation": { + "bases": [ + "TrivialRegisterPassable", + "UnsafeSingleNicheable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Span": { + "bases": [ + "Boolable", + "Defaultable", + "DevicePassable", + "Hashable", + "ImplicitlyCopyable", + "Iterable", + "IterableOwned", + "Sized", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "SpinWaiter": { + "bases": [ + "Defaultable" + ], + "implicitlyCopyable": false + }, + "StackTrace": { + "bases": [ + "ImplicitlyCopyable", + "Movable", + "Writable" + ], + "implicitlyCopyable": true + }, + "StaticTuple": { + "bases": [ + "Defaultable", + "DevicePassable", + "Sized", + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "StopIteration": { + "bases": [ + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "StridedSlice": { + "bases": [ + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "String": { + "bases": [ + "Boolable", + "Comparable", + "ConvertibleFromPython", + "Defaultable", + "FloatableRaising", + "ImplicitlyCopyable", + "IntableRaising", + "KeyElement", + "PathLike", + "Writable", + "Writer" + ], + "implicitlyCopyable": true + }, + "StringDict": { + "bases": [ + "Copyable", + "Defaultable", + "Deinitable", + "Iterable", + "Movable", + "Sized", + "Writable" + ], + "implicitlyCopyable": false + }, + "StringLiteral": { + "bases": [ + "Boolable", + "Defaultable", + "FloatableRaising", + "ImplicitlyCopyable", + "IntableRaising", + "PathLike", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "StringSpan": { + "bases": [ + "Boolable", + "Defaultable", + "Equatable", + "FloatableRaising", + "Hashable", + "ImplicitlyCopyable", + "IntableRaising", + "KeyElement", + "PathLike", + "TrivialRegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "SwissTable": { + "bases": [ + "Copyable", + "Deinitable", + "Movable" + ], + "implicitlyCopyable": false + }, + "SwissTableEntry": { + "bases": [ + "Copyable", + "Deinitable", + "Movable" + ], + "implicitlyCopyable": false + }, + "TString": { + "bases": [ + "RegisterPassable", + "Writable" + ], + "implicitlyCopyable": false + }, + "Task": { + "bases": [], + "implicitlyCopyable": false + }, + "TaskGroup": { + "bases": [ + "Defaultable" + ], + "implicitlyCopyable": false + }, + "TaskGroupContext": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "TemporaryDirectory": { + "bases": [], + "implicitlyCopyable": false + }, + "TestReport": { + "bases": [ + "Copyable", + "Writable" + ], + "implicitlyCopyable": false + }, + "TestResult": { + "bases": [ + "EnumLike", + "Equatable", + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "TestSuite": { + "bases": [ + "Movable" + ], + "implicitlyCopyable": false + }, + "TestSuiteReport": { + "bases": [ + "Copyable", + "Writable" + ], + "implicitlyCopyable": false + }, + "Text": { + "bases": [ + "Writable" + ], + "implicitlyCopyable": false + }, + "ThinAllocation": { + "bases": [ + "RegisterPassable", + "Writable" + ], + "implicitlyCopyable": false + }, + "ThroughputMeasure": { + "bases": [ + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "Token": { + "bases": [ + "__Enum" + ], + "implicitlyCopyable": false + }, + "Tuple": { + "bases": [ + "Comparable", + "Copyable", + "Defaultable", + "Deinitable", + "Equatable", + "Hashable", + "ImplicitlyCopyable", + "Movable", + "RegisterPassable", + "Sized", + "Writable" + ], + "implicitlyCopyable": true + }, + "TypeDict": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "TypeList": { + "bases": [ + "Sized", + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "TypeNames": { + "bases": [ + "ImplicitlyCopyable", + "Writable" + ], + "implicitlyCopyable": true + }, + "UTF8Chunk": { + "bases": [ + "ImplicitlyCopyable" + ], + "implicitlyCopyable": true + }, + "UTF8Chunks": { + "bases": [ + "ImplicitlyCopyable", + "Iterable", + "Iterator" + ], + "implicitlyCopyable": true + }, + "Unit": { + "bases": [], + "implicitlyCopyable": false + }, + "UnsafeUnion": { + "bases": [ + "ImplicitlyCopyable", + "Movable", + "Writable" + ], + "implicitlyCopyable": true + }, + "VariadicList": { + "bases": [ + "RegisterPassable", + "Sized", + "Writable" + ], + "implicitlyCopyable": false + }, + "VariadicPack": { + "bases": [ + "Copyable", + "RegisterPassable", + "Sized" + ], + "implicitlyCopyable": false + }, + "Variant": { + "bases": [ + "Copyable", + "Deinitable", + "Equatable", + "Hashable", + "ImplicitlyCopyable", + "Movable", + "RegisterPassable", + "Writable" + ], + "implicitlyCopyable": true + }, + "WaitFlags": { + "bases": [], + "implicitlyCopyable": false + }, + "WeakPointer": { + "bases": [ + "ImplicitlyCopyable", + "RegisterPassable" + ], + "implicitlyCopyable": true + }, + "amd_signal_t": { + "bases": [ + "Copyable" + ], + "implicitlyCopyable": false + }, + "assert_raises": { + "bases": [], + "implicitlyCopyable": false + }, + "buffer_t": { + "bases": [ + "Copyable", + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "header_t": { + "bases": [ + "TrivialRegisterPassable" + ], + "implicitlyCopyable": true + }, + "payload_t": { + "bases": [ + "Copyable" + ], + "implicitlyCopyable": false + }, + "stat_result": { + "bases": [ + "Copyable", + "Writable" + ], + "implicitlyCopyable": false + } +}; diff --git a/src/mojo/interp.ts b/src/mojo/interp.ts new file mode 100644 index 0000000..ab93858 --- /dev/null +++ b/src/mojo/interp.ts @@ -0,0 +1,2231 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Programming+'s Mojo interpreter (Stage 3): runs a parsed program (src/mojo/parse.ts) + * and records the trace every stepping view reads (the stepper, the table of + * values, breakpoints, predict mode), so a Mojo program can be stepped line by + * line like the pseudo. + * + * It is judged against the real compiler: for every program godbolt compiled and + * ran (library/mojo/golden), what this prints, and how it ends, must be what godbolt's + * run printed and how it ended (tests/mojo-run.test.mjs). Where it does not model + * something, it says so and stops ("Programming+ cannot run this yet"), rather + * than guess: Compile & run asks the real compiler. + * + * Conventions follow Mojo's: an argument with no convention (or imm) is read in + * place; `mut` and `ref` alias the caller's variable; `var` takes a copy, or the + * value itself when the caller writes `x^`, after which x is empty until assigned. + * Values have value semantics: a copy shares nothing (values.ts, deepCopy). + * + * Not modeled yet, and declined: the order in which values are destroyed and + * copied (a struct whose __deinit__, copy or move constructor does anything: + * Stage 4), SIMD, pointers, files and `with`, Python, comptime parameters + * beyond plain constants, and most of the standard library beyond its core. + */ +import type { Arg, Expr, Module, Param, Pos, Stmt } from './parse.js'; +import { parseModule } from './parse.js'; +import { lifetimes, type Lifetimes } from './lifetimes.js'; +import type { Trace } from '../common/types.js'; +import { + type V, INT_TYPES, int, float, bool, str, NONE, floordiv, mod, truncdiv, show, display, typeName, + keyOf, deepCopy as copyValue, elementsIn, truthy, equal, utf8, codepoints, roundHalfEven, Unprintable, +} from './values.js'; + +/* ---------------------------------------------------------------- limits */ + +/** + * What one run may hold and do besides its steps. A program from a share link can grow a value without bound in a + * few steps (t = t + t doubles a String each pass, xs.extend(xs) a List); past any of these the run stops as it does + * at its step limit (`limited`: how the compiled program would have ended is not known), before the engine + * underneath runs out of memory, throws an error of its own, or works for minutes (tests/mojo-robustness.test.mjs, + * docs/limits.md). Each is far beyond what a program written to be stepped through needs. + */ +export const RUN_LIMITS = { + /** characters in one String */ + text: 10_000_000, + /** values in one List, Dict or Set */ + values: 2_000_000, + /** characters printed in one run */ + output: 10_000_000, + /** values made or copied into collections in one run: what it can come to hold */ + made: 5_000_000, + /** characters and values a run builds, copies or searches through: how long it can take */ + work: 100_000_000, + /** variables' values the trace records, one per variable per step: what stepping through it keeps (each is an object) */ + recorded: 1_000_000, +} as const; +/** Lower limits for one run (the tests prove each limit with a small one); none can be raised past RUN_LIMITS. */ +export type RunLimits = Partial<Record<keyof typeof RUN_LIMITS, number>>; +const count = (n: number): string => n.toLocaleString('en-US'); + +/** The run whose budget a copy is charged to (one Interp runs at a time; set by run() and callOne()). */ +let charging: { made(n: number, pos: Pos | null): void } | null = null; + +/** Mojo's value semantics copy a whole collection: the copy is charged to the run's budget of values made. */ +function deepCopy(v: V): V { + if (charging && v.k !== 'int' && v.k !== 'float' && v.k !== 'bool' && v.k !== 'str') { const n = elementsIn(v); if (n) charging.made(n, null); } + return copyValue(v); +} + +/** The first `n` items of a String (its codepoints), a list or a map, without listing the rest. */ +function* firstItems<T>(xs: Iterable<T>, n: number): Generator<T> { if (n <= 0) return; for (const x of xs) { yield x; if (--n <= 0) return; } } + +/** + * How a run ends when the engine underneath fails: its stack or a size it refuses is a limit, like the step limit; + * anything else is a fault in the interpreter, and the run is declined rather than answered. + */ +function engineStop(e: unknown): { limited: boolean; message: string } { + if (e instanceof RangeError) { + return { limited: true, message: /call stack/i.test(e.message) + ? 'Stopped: the run went deeper (calls inside calls, or expressions inside expressions) than Programming+ can follow.' + : 'Stopped: a value grew larger than Programming+ can hold.' }; + } + return { limited: false, message: `a fault in Programming+ (${e instanceof Error ? e.message : String(e)})` }; +} + +/* ---------------------------------------------------------------- outcomes */ + +class Refuse extends Error { constructor(readonly reason: string, readonly pos: Pos | null) { super(reason); } } +class Raised { constructor(readonly value: V, readonly pos: Pos | null) {} } +class Halt { constructor(readonly code: number) {} } +class ReturnSignal { constructor(readonly value: V | null) {} } +class BreakSignal {} +class ContinueSignal {} + +/** What happened to a value whose lifecycle can be seen, during one step; `id` is the value's identity in this run. */ +export interface LifeEvent { kind: 'made' | 'copied' | 'moved' | 'named' | 'destroyed'; id: number; label: string; from?: number; name?: string } +export interface MojoStep { line: number | null; kind: string; text: string; reads: Access[]; writes: Access[]; depth: number; sub: string; life?: LifeEvent[] } +interface Access { name: string; frame: number; path?: string[] } +export interface MojoState { frames: Array<{ id: number; name: string; vars: Record<string, { name: string; value: V | undefined; type: string; text: string }> }>; loops: Array<{ kind: string; line: number; pass: number }>; outputLength: number } + +/** + * Where a value lives: a variable, a field, a list slot. `mut` arguments and `ref` bindings share one. `type`: the + * type the place was declared with, where it is known, which a value assigned there takes (a list in brackets + * becomes a List: probes/list-literal-into-place). + */ +interface Loc { get(): V | undefined; set(v: V | undefined): void; name: string; frame: number; path?: string[]; type?: string } + +/** `owned`: this frame owns the value (a var, or a var/owned parameter), so it destroys it at its last use. */ +interface Cell { v: V | undefined; type: string; owned?: boolean; destroyed?: boolean } +interface Frame { id: number; name: string; cells: Map<string, Cell>; aliases: Map<string, Loc>; selfType?: string; def?: DefStmt; life?: Lifetimes } + +/** A value due to be destroyed: when its bucket's call returns, or its statement ends. */ +interface Doomed { v: V; label: string; frame?: Frame; name?: string } + +type DefStmt = Extract<Stmt, { k: 'def' }>; +type StructStmt = Extract<Stmt, { k: 'struct' | 'trait' }>; + +/** `limited`: the run hit its step limit, so how it would have ended is not known (exitCode is then -1). */ +export interface RunResult { trace: Trace<MojoState, MojoStep>; stdout: string; stderr: string; exitCode: number; refused: string | null; limited: boolean } + +export function runMojo(mod: Module, source: string, opts: { input?: string[]; maxSteps?: number; record?: boolean; limits?: RunLimits } = {}): RunResult { + return new Interp(mod, source, opts).run(); +} + +/** How one call ended: it returned, it raised, it aborted (an index out of bounds), it ran out of steps, or it was declined. */ +export type CallOutcome = + | { kind: 'returned'; value: V | null } + | { kind: 'raised'; message: string; line: number | null } + | { kind: 'aborted'; message: string; line: number } + | { kind: 'limit' } + | { kind: 'declined'; reason: string; line: number | null }; + +/** + * One call, with what the fuzzer needs besides its outcome: the lines it ran (and how often) and how many steps it + * took. Same interpreter, same limits as `callMojo`. + */ +export function probeMojo(mod: Module, source: string, name: string, args: V[], opts: { maxSteps?: number } = {}): { outcome: CallOutcome; hits: Map<number, number>; steps: number } { + const it = new Interp(mod, source, { record: false, maxSteps: opts.maxSteps ?? 5000 }); + const outcome = it.callOne(name, args); + return { outcome, hits: it.lineHits(), steps: it.stepCount() }; +} + +/** Call one of the module's functions with argument values (property tests, Stage 5). */ +export function callMojo(mod: Module, source: string, name: string, args: V[], opts: { maxSteps?: number } = {}): CallOutcome { + return new Interp(mod, source, { record: false, maxSteps: opts.maxSteps ?? 5000 }).callOne(name, args); +} + +const SCALAR_TYPES = new Set([...Object.keys(INT_TYPES), 'Float64', 'Float32', 'Bool', 'String']); + +class Interp { + private steps: MojoStep[] = []; + private states: MojoState[] = []; + private hits = new Map<number, number>(); + private stdout = ''; + private stderr = ''; + private frames: Frame[] = []; + private nextFrameId = 1; + private reads: Access[] = []; + private writes: Access[] = []; + private loops: Array<{ kind: string; line: number; pass: number }> = []; + private functions = new Map<string, DefStmt>(); + private overloads = new Map<string, DefStmt[]>(); + private structs = new Map<string, StructStmt>(); + private traits = new Map<string, StructStmt>(); + private globals = new Map<string, V>(); + private lines: string[]; + private input: string[]; + private maxSteps: number; + private count = 0; + private imported = new Set<string>(); + + private limits: Record<keyof typeof RUN_LIMITS, number>; + + constructor(private mod: Module, source: string, private opts: { input?: string[]; maxSteps?: number; record?: boolean; limits?: RunLimits }) { + this.lines = source.split('\n'); + this.input = [...(opts.input ?? [])]; + this.maxSteps = opts.maxSteps ?? 20000; + this.limits = { ...RUN_LIMITS }; + for (const [k, v] of Object.entries(opts.limits ?? {}) as Array<[keyof typeof RUN_LIMITS, number]>) if (k in RUN_LIMITS && v < RUN_LIMITS[k]) this.limits[k] = v; + } + + /* ---------------- the run ---------------- */ + + run(): RunResult { + let exitCode = 0, refused: string | null = null, limited = false; + let error: Trace['error'] = null; + const outer = charging; + charging = this; + try { + if (this.mod.unread) throw new Refuse('code Programming+ cannot read yet', this.mod.findings.find((f) => f.id === 'unread') ?? null); + this.declare(); + const main = this.functions.get('main'); + if (!main) throw new Refuse("a program with no main function", null); + this.states.push(this.snapshot()); // before anything runs: no frames yet + try { + this.callDef(main, [], null, main.pos); + } catch (e) { + if (e instanceof Raised) { + const msg = `Unhandled exception caught during execution: ${show(e.value)}`; + this.stderr += msg + '\n'; + exitCode = 1; + error = { kind: 'raise', message: msg, line: e.pos?.line ?? null }; + } else throw e; + } + } catch (e) { + if (e instanceof Halt) exitCode = e.code; + else if (e instanceof Refuse) { + refused = e.reason; + error = { kind: 'unsupported', message: `Programming+ cannot run this yet: ${e.reason}. Compile & run asks the real compiler.`, line: e.pos?.line ?? null }; + } else if (e instanceof OutOfBounds) { + this.write(e.message + '\n'); + exitCode = 132; + error = { kind: 'bounds', message: e.message, line: e.pos.line }; + } else if (e instanceof Limit) { + error = { kind: 'limit', message: e.message, line: e.pos?.line ?? null }; + limited = true; exitCode = -1; + } else if (e instanceof Unprintable) { + refused = `printing a value of this kind (${e.kind})`; + error = { kind: 'unsupported', message: `Programming+ cannot run this yet: ${refused}. Compile & run asks the real compiler.`, line: null }; + } else { + // Anything else is the interpreter's own failure, not the program's answer: a run ends in a stop or a + // refusal, never in an exception for the caller (tests/mojo-robustness.test.mjs). + const stop = engineStop(e); + if (stop.limited) { limited = true; exitCode = -1; error = { kind: 'limit', message: stop.message, line: null }; } + else { refused = stop.message; error = { kind: 'unsupported', message: `Programming+ cannot run this yet: ${refused}. Compile & run asks the real compiler.`, line: null }; } + } + } finally { charging = outer; } + const output = this.stdout.split('\n'); + if (output[output.length - 1] === '') output.pop(); + const trace: Trace<MojoState, MojoStep> = { + lang: 'mojo', steps: this.steps, states: this.states, count: this.steps.length, output, + error, truncated: limited, hits: this.hits, raw: { exitCode, stdout: this.stdout, stderr: this.stderr, refused, limited }, + }; + return { trace, stdout: this.stdout, stderr: this.stderr, exitCode, refused, limited }; + } + + /** The lines run so far, with their counts, and the steps taken (probeMojo). */ + lineHits(): Map<number, number> { return new Map(this.hits); } + stepCount(): number { return this.count; } + + callOne(name: string, args: V[]): CallOutcome { + const outer = charging; + charging = this; + try { + if (this.mod.unread) throw new Refuse('code Programming+ cannot read yet', null); + this.declare(); + const def = this.functions.get(name); + if (!def) throw new Refuse(`a function named ${name}`, null); + return { kind: 'returned', value: this.callDef(def, [], null, def.pos, args.map(deepCopy)) }; + } catch (e) { + if (e instanceof Raised) return { kind: 'raised', message: show(e.value), line: e.pos?.line ?? null }; + if (e instanceof OutOfBounds) return { kind: 'aborted', message: e.message.replace(/^At: [^ ]+ /, ''), line: e.pos.line }; + if (e instanceof Limit) return { kind: 'limit' }; + if (e instanceof Refuse) return { kind: 'declined', reason: e.reason, line: e.pos?.line ?? null }; + if (e instanceof Unprintable) return { kind: 'declined', reason: `printing a value of this kind (${e.kind})`, line: null }; + if (e instanceof Halt) return { kind: 'declined', reason: 'a call that exits the program', line: null }; + const stop = engineStop(e); + return stop.limited ? { kind: 'limit' } : { kind: 'declined', reason: stop.message, line: null }; + } finally { charging = outer; } + } + + private declare(): void { + for (const s of this.mod.body) { + switch (s.k) { + case 'def': { + // an overloaded name: every def is kept, and a call picks the one its arguments fit + const prev = this.functions.get(s.name); + if (prev) { this.overloads.set(s.name, [...(this.overloads.get(s.name) ?? [prev]), s]); } + else this.functions.set(s.name, s); + break; + } + case 'struct': this.structs.set(s.name, s); break; + case 'trait': this.traits.set(s.name, s); break; + case 'from': for (const n of s.names) this.imported.add(n.alias ?? n.name); break; + case 'import': this.imported.add(s.alias ?? s.module); break; + case 'var': break; + case 'expr': if (s.e.k === 'str') break; throw new Refuse('code at file scope', s.pos); + default: throw new Refuse(`a ${s.k} statement at file scope`, s.pos); + } + } + this.findTracked(); + // comptime constants, in order + this.frames.push({ id: -1, name: 'globals', cells: new Map(), aliases: new Map() }); + for (const s of this.mod.body) if (s.k === 'var' && s.comptime && s.targets.k === 'name' && s.value && !s.typeParams?.length) this.globals.set(s.targets.id, this.coerce(this.eval(s.value), s.type)!); + else if (s.k === 'var') throw new Refuse('a variable at file scope', s.pos); + this.frames.pop(); + } + + /* ---------------- lifecycles (Stage 4) ---------------- */ + + /** Structs whose copies, moves or destruction can be seen: their own lifecycle methods do something, or a field's type does. */ + private tracked = new Set<string>(); + private created = new WeakMap<object, number>(); + private creation = 0; + private buckets: Doomed[][] = []; + + private findTracked(): void { + for (const st of this.structs.values()) { + const own = st.body.some((b) => b.k === 'def' && (b.name === '__deinit__' || b.name === '__del__' || this.ctorKind(b) !== null) && containsCall(b.body)); + if (own) this.tracked.add(st.name); + } + for (let changed = true; changed;) { + changed = false; + for (const st of this.structs.values()) { + if (this.tracked.has(st.name)) continue; + const fieldTypes = st.body.filter((b): b is Extract<Stmt, { k: 'var' }> => b.k === 'var' && !b.comptime && !!b.type).map((b) => typeText(b.type!)); + if (fieldTypes.some((t) => this.isTrackedType(t))) { this.tracked.add(st.name); changed = true; } + } + } + for (const name of this.tracked) { + const st = this.structs.get(name)!; + if (st.bases.some((b) => b.k === 'name' && b.id === 'ImplicitlyCopyable')) throw new Refuse(`${name}'s implicit copies (it is ImplicitlyCopyable and its lifecycle can be seen)`, st.pos); + if (st.decorators.some((d) => !(d.k === 'name' && d.id === 'fieldwise_init'))) throw new Refuse(`${name}'s decorators, with a lifecycle that can be seen`, st.pos); + } + } + + private isTrackedType(t: string): boolean { + if (this.tracked.has(t)) return true; + const m = /^(?:List|Optional)\[(.+)\]$/.exec(t); + return !!m && this.isTrackedType(m[1]); + } + + /** Whether a value's lifecycle can be seen: a tracked struct, or a list holding one. */ + private isTracked(v: V | undefined): boolean { + if (!v) return false; + if (v.k === 'struct') return this.tracked.has(v.name); + if (v.k === 'list') return this.isTrackedType(v.elem) || v.items.some((x) => this.isTracked(x)); + if (v.k === 'opt') return !!v.v && this.isTracked(v.v); + if (v.k === 'dict' || v.k === 'set' || v.k === 'tuple') { if ((v.k === 'tuple' ? v.items : [...(v.k === 'dict' ? [...v.map.values()].flat() : v.map.values())]).some((x) => this.isTracked(x))) throw new Refuse('a value with a visible lifecycle inside a Dict, Set or Tuple', null); } + return false; + } + + /** `__init__(out self, *, copy: Self)` and `__init__(out self, *, deinit move: Self)`. */ + private ctorKind(b: Stmt): 'copy' | 'move' | null { + if (b.k !== 'def' || b.name !== '__init__' || !b.params.some((p) => p.star === 'marker')) return null; + const kw = b.params.slice(b.params.findIndex((p) => p.star === 'marker') + 1); + if (kw.length === 1 && kw[0].name === 'copy') return 'copy'; + if (kw.length === 1 && kw[0].name === 'move') return 'move'; + return null; + } + + private born(v: V): V { if (typeof v === 'object' && !this.created.has(v)) this.created.set(v, ++this.creation); return v; } + + private lifeEvents: LifeEvent[] = []; + private event(kind: LifeEvent['kind'], v: V, extra: { from?: V; name?: string } = {}): void { + const id = this.created.get(v); + if (id === undefined) return; + this.lifeEvents.push({ kind, id, label: display(v), ...(extra.from && this.created.has(extra.from) ? { from: this.created.get(extra.from) } : {}), ...(extra.name ? { name: extra.name } : {}) }); + } + + /** A value whose last use is happening now: it dies when the innermost call (or statement) that uses it ends. */ + private doom(d: Doomed): void { + const b = this.buckets[this.buckets.length - 1]; + if (!b) throw new Refuse('destroying a value outside any statement', null); + if (!b.some((x) => x.v === d.v)) b.push(d); + } + + /** A temporary that something took ownership of: no longer the bucket's to destroy. */ + private claim(v: V | undefined): void { + if (!v) return; + for (const b of this.buckets) { const i = b.findIndex((x) => x.v === v); if (i >= 0) b.splice(i, 1); } + } + + private openBucket(): void { this.buckets.push([]); } + /** Destroy what died in this bucket, newest first (the life-asap-expression probe's order). */ + private closeBucket(line: number | null): void { + const doomed = this.buckets.pop() ?? []; + doomed.sort((a, b) => (this.created.get(b.v) ?? 0) - (this.created.get(a.v) ?? 0)); + for (const d of doomed) { + this.destroy(d.v, line, d.label); + if (d.frame && d.name) { const c = d.frame.cells.get(d.name); if (c && c.v === d.v) { c.v = undefined; c.destroyed = true; } } + } + } + + /** Run a value's destruction: its __deinit__, then its fields in declaration order (life-field-order); a list's elements in order. */ + private destroy(v: V, line: number | null, label: string): void { + if (v.k === 'list') { for (const x of v.items) if (this.isTracked(x)) this.destroy(x, line, `${label}'s element`); return; } + if (v.k === 'opt') { if (v.v && this.isTracked(v.v)) this.destroy(v.v, line, label); return; } + if (v.k !== 'struct' || !this.tracked.has(v.name)) return; + this.event('destroyed', v); + this.step(line, `${label} is destroyed`, 'destroy'); + const st = this.structs.get(v.name)!; + const deinit = st.body.find((b): b is DefStmt => b.k === 'def' && (b.name === '__deinit__' || b.name === '__del__')); + if (deinit) this.callDef(deinit, [], { loc: constLoc(v), value: v, struct: st }, deinit.pos); + for (const [name, f] of v.fields) if (f && this.isTracked(f)) this.destroy(f, line, `${label}.${name}`); + } + + /** A value moved into new storage: the move constructor runs (`var c = a^`, `self.f = x^`, into a List). */ + private moveInto(v: V, at: Pos): V { + if (v.k === 'list') return v; // a List moves its buffer, not its elements + if (v.k !== 'struct' || !this.tracked.has(v.name)) return v; + const st = this.structs.get(v.name)!; + const ctor = st.body.find((b): b is DefStmt => this.ctorKind(b) === 'move'); + if (!ctor) { + // the compiler's move constructor: each field moves + const fields = new Map<string, V | undefined>(); + for (const [n, f] of v.fields) fields.set(n, f && this.isTracked(f) ? this.moveInto(f, at) : f); + const moved = this.born({ k: 'struct', name: v.name, fields }); + this.event('moved', moved, { from: v }); + return moved; + } + const moved = this.born(this.runCtor(st, ctor, v, at)); + this.event('moved', moved, { from: v }); + return moved; + } + + /** `.copy()`: the copy constructor, the struct's own or the compiler's (each field copied). */ + private copyOf(v: V, at: Pos): V { + if (v.k === 'list') return { ...v, items: v.items.map((x) => (this.isTracked(x) ? this.copyOf(x, at) : deepCopy(x))) }; + if (v.k !== 'struct' || !this.tracked.has(v.name)) return deepCopy(v); + const st = this.structs.get(v.name)!; + const ctor = st.body.find((b): b is DefStmt => this.ctorKind(b) === 'copy'); + if (!ctor) { + const fields = new Map<string, V | undefined>(); + for (const [n, f] of v.fields) fields.set(n, f && this.isTracked(f) ? this.copyOf(f, at) : f && deepCopy(f)); + const copy = this.born({ k: 'struct', name: v.name, fields }); + this.event('copied', copy, { from: v }); + return copy; + } + const copy = this.born(this.runCtor(st, ctor, v, at)); + this.event('copied', copy, { from: v }); + return copy; + } + + private runCtor(st: StructStmt, ctor: DefStmt, from: V, at: Pos): V { + const fields = new Map<string, V | undefined>(); + for (const b of st.body) if (b.k === 'var' && !b.comptime && b.targets.k === 'name') fields.set(b.targets.id, undefined); + const value: V = { k: 'struct', name: st.name, fields }; + const cell: Cell = { v: value, type: st.name }; + this.callDef(ctor, [], { loc: { get: () => cell.v, set: (x) => { cell.v = x; }, name: 'self', frame: -1 }, value, struct: st }, at, [from]); + return cell.v!; + } + + /** The label a destruction step uses for a value. */ + private labelOf(v: V): string { return display(v); } + + /* ---------------- recording ---------------- */ + + private get frame(): Frame { return this.frames[this.frames.length - 1]; } + + private step(line: number | null, text: string, kind = 'stmt'): void { + this.count++; + if (this.count > this.maxSteps) throw new Limit(`Stopped after ${this.maxSteps.toLocaleString('en-US')} steps.${this.loops.length ? ` The loop on line ${this.loops[this.loops.length - 1].line} was on pass ${this.loops[this.loops.length - 1].pass}.` : ''}`); + if (line != null) this.hits.set(line, (this.hits.get(line) ?? 0) + 1); + if (this.opts.record !== false) { + this.steps.push({ line, kind, text, reads: this.reads, writes: this.writes, depth: this.frames.length - 1, sub: this.frame?.name ?? 'main', ...(this.lifeEvents.length ? { life: this.lifeEvents } : {}) }); + this.lifeEvents = []; + this.states.push(this.snapshot()); + } + this.reads = []; this.writes = []; + } + + /* ---------------- the budget (RUN_LIMITS) ---------------- */ + + private madeCount = 0; + private workCount = 0; + + /** `n` values made or copied into a collection. */ + made(n: number, pos: Pos | null): void { + this.madeCount += n; + if (this.madeCount > this.limits.made) throw new Limit(`Stopped after making or copying ${count(this.limits.made)} values in Lists, Dicts and Sets, more than Programming+ follows in one run.`, pos); + this.work(n, pos); + } + + /** `n` characters or values built, copied or searched through. */ + private work(n: number, pos: Pos | null): void { + this.workCount += n; + if (this.workCount > this.limits.work) throw new Limit(`Stopped after building, copying and searching through ${count(this.limits.work)} characters and values, more than Programming+ follows in one run.`, pos); + } + + /** A String of `length` characters about to be made, `cost` of them new work. */ + private grow(length: number, cost: number, pos: Pos | null): void { + if (length > this.limits.text) throw new Limit(`Stopped: this makes a String of ${count(length)} characters, more than Programming+ holds (${count(this.limits.text)}).`, pos); + this.work(cost, pos); + } + + /** A collection about to hold `size` values. */ + private room(size: number, what: string, pos: Pos | null): void { + if (size > this.limits.values) throw new Limit(`Stopped: this makes a ${what} of ${count(size)} values, more than Programming+ holds (${count(this.limits.values)}).`, pos); + } + + private recorded = 0; + private snapshot(): MojoState { + for (const f of this.frames) this.recorded += f.cells.size; + if (this.recorded > this.limits.recorded) throw new Limit(`Stopped after recording ${count(this.limits.recorded)} values of variables, one per variable at each step: too many variables, stepped too often, for the trace to keep.`, null); + const frames = this.frames.filter((f) => f.id >= 0).map((f) => { + const vars: MojoState['frames'][number]['vars'] = {}; + for (const [name, c] of f.cells) vars[name] = { name, value: c.v, type: c.type || typeName(c.v), text: c.destroyed ? '(destroyed)' : display(c.v, (s) => this.describe(s)) }; + return { id: f.id, name: f.name, vars }; + }); + return { frames, loops: this.loops.map((l) => ({ ...l })), outputLength: this.newlines + (this.stdout && !this.stdout.endsWith('\n') ? 1 : 0) }; + } + + private describe(s: V): string { return display(s); } + private textOf(p: Pos): string { return (this.lines[p.line - 1] ?? '').trim(); } + /** An if's or while's condition, as written: the line without its keyword and colon. */ + private condition(p: Pos): string { return this.textOf(p).replace(/^(el)?if\s+|^while\s+/, '').replace(/:\s*(#.*)?$/, ''); } + private newlines = 0; + private write(s: string): void { + if (this.stdout.length + s.length > this.limits.output) throw new Limit(`Stopped after printing ${count(this.limits.output)} characters.`, null); + for (let k = s.indexOf('\n'); k >= 0; k = s.indexOf('\n', k + 1)) this.newlines++; + this.stdout += s; + } + + /* ---------------- names ---------------- */ + + private lookup(name: string, pos: Pos): Loc | null { + const f = this.frame; + const alias = f.aliases.get(name); + if (alias) return alias; + const c = f.cells.get(name); + if (c) return { get: () => c.v, set: (v) => { c.v = v; if (v) c.destroyed = false; }, name, frame: f.id, type: c.type || undefined }; + void pos; + return null; + } + + private read(name: string, pos: Pos, node?: Expr): V { + const loc = this.lookup(name, pos); + if (loc) { + const v = loc.get(); + if (v === undefined) throw new Refuse(`reading ${name} after it was moved out (the compiler rejects this)`, pos); + this.reads.push({ name: loc.name, frame: loc.frame }); + const c = this.frame.cells.get(name); + if (node && c?.owned && !this.frame.aliases.has(name) && this.frame.life?.lastReads.has(node) && this.isTracked(v)) this.doom({ v, label: name, frame: this.frame, name }); + return v; + } + if (this.globals.has(name)) return this.globals.get(name)!; + if (name === 'Self' && this.frame.selfType) return { k: 'fn', name: this.frame.selfType }; + if (name === 'True') return bool(true); + if (name === 'False') return bool(false); + if (name === 'None') return NONE; + if (name === 'pi' && this.imported.has('pi')) return float(Math.PI); // std.math's pi, as a Float64 (probes/math-pi-literal) + if (this.functions.has(name) || this.structs.has(name) || BUILTINS.has(name) || SCALAR_TYPES.has(name) || ['List', 'Dict', 'Set', 'Optional', 'Error', 'SIMD', 'DType'].includes(name)) return { k: 'fn', name }; + throw new Refuse(`the name ${name}`, pos); + } + + private declareVar(name: string, v: V | undefined, type: string): void { + // the first var of a name in this block: remember what it hides, to restore at the block's end + const scope = this.shadowed[this.shadowed.length - 1]; + if (scope && this.frame.def && this.frame.def.body !== undefined && !scope.has(name) && this.blockDepth() > 0) { + const prev = this.frame.cells.get(name); + if (prev && v && this.isTracked(v)) throw new Refuse(`${name} declared again in an inner block, with a lifecycle that can be seen`, null); + scope.set(name, prev); + } + if (v && this.isTracked(v) && this.frame.life?.unmodeled.has(name)) throw new Refuse(`when ${name} is destroyed: it is used in a try, with, nested function, comprehension, t-string or ref, whose flow Programming+ does not follow yet`, null); + this.frame.cells.set(name, { v, type, owned: true }); + if (v && v.k === 'struct' && this.isTracked(v)) this.event('named', v, { name }); + this.frame.aliases.delete(name); + this.writes.push({ name, frame: this.frame.id }); + } + + /* ---------------- statements ---------------- */ + + /** + * A block's `var`s are its own (manual/variables: `var num = 2` inside an if is a new num that shadows + * the outer one, which is untouched): what a block declares is undone when it ends. + */ + private shadowed: Array<Map<string, Cell | undefined>> = []; + private block(list: Stmt[]): void { + const scope = new Map<string, Cell | undefined>(); + this.shadowed.push(scope); + try { for (const s of list) this.exec(s); } + finally { + this.shadowed.pop(); + const f = this.frame; + if (f) for (const [name, prev] of scope) { if (prev) f.cells.set(name, prev); else f.cells.delete(name); } + } + } + + private static SIMPLE = new Set(['expr', 'var', 'assign', 'aug', 'return', 'raise', 'pass', 'break', 'continue', 'assert']); + + /** A simple statement runs inside a bucket: what dies in it is destroyed when it ends, after its step. */ + private exec(s: Stmt): void { + if (!Interp.SIMPLE.has(s.k)) { this.execInner(s); return; } + const depth = this.buckets.length; + this.openBucket(); + try { + this.execInner(s); + } catch (e) { + if (e instanceof ReturnSignal || e instanceof BreakSignal || e instanceof ContinueSignal) { this.afterStatement(s); this.closeBucket(s.pos.line); throw e; } + if (this.buckets.slice(depth).some((b) => b.length)) throw new Refuse('values destroyed while an error is raised (unwinding is not modeled)', s.pos); + this.buckets.length = depth; + throw e; + } + this.afterStatement(s); + this.closeBucket(s.pos.line); + } + + /** + * Destroy every value this frame owns whose variable is not in `live`: what the path ahead cannot read. + * At a branch's start (life-branch-one-side, life-early-return, life-moved-on-one-path), at a pass's start + * (life-reassign-in-loop), when a loop ends (life-loop-outer-read-inside), at a return. + */ + private destroyDead(live: Set<string> | undefined, line: number | null): void { + const f = this.frame; + if (!f?.life || !live) return; + const dead = [...f.cells].filter(([name, c]) => c.owned && !c.destroyed && c.v && !f.aliases.has(name) && !live.has(name) && this.isTracked(c.v)); + if (!dead.length) return; + this.openBucket(); + for (const [name, c] of dead) this.doom({ v: c.v!, label: name, frame: f, name }); + this.closeBucket(line); + } + + /** A variable's new value that is never read dies as soon as its statement ends (life-last-use, life-reassign). */ + private afterStatement(s: Stmt): void { + const life = this.frame?.life; + for (const name of life?.unreadAfter.get(s) ?? []) { + const c = this.frame.cells.get(name); + if (c?.owned && c.v && this.isTracked(c.v)) this.doom({ v: c.v, label: name, frame: this.frame, name }); + } + } + + private execInner(s: Stmt): void { + switch (s.k) { + case 'expr': this.eval(s.e); this.step(s.pos.line, this.textOf(s.pos)); return; + case 'pass': this.step(s.pos.line, 'pass'); return; + case 'var': { + if (s.targets.k !== 'name') { + if (s.targets.k === 'tuple' && s.value) { + const v = this.eval(s.value); + if (v.k !== 'tuple' || v.items.length !== s.targets.items.length) throw new Refuse('unpacking this value', s.pos); + s.targets.items.forEach((t, i) => { if (t.k !== 'name') throw new Refuse('this unpacking target', s.pos); this.declareVar(t.id, deepCopy(v.items[i]), typeName(v.items[i])); }); + this.step(s.pos.line, this.textOf(s.pos)); + return; + } + throw new Refuse('this declaration', s.pos); + } + if (s.ref) { + if (!s.value) throw new Refuse('ref without a value', s.pos); + const loc = this.locate(s.value); + this.frame.aliases.set(s.targets.id, { ...loc, name: s.targets.id, frame: this.frame.id, get: loc.get, set: loc.set }); + this.step(s.pos.line, this.textOf(s.pos)); + return; + } + if (s.comptime && (this.frame.cells.has(s.targets.id) || this.globals.has(s.targets.id))) throw new Refuse(`comptime ${s.targets.id} declared again in an inner scope (comptime names are block-scoped)`, s.pos); + const v = s.value ? this.coerce(this.take(s.value, true), s.type) : undefined; + this.declareVar(s.targets.id, v, s.type ? typeText(s.type) : typeName(v)); + this.step(s.pos.line, this.textOf(s.pos)); + return; + } + case 'assign': { + const v = this.take(s.value, true); + for (const t of s.targets) this.assign(t, v); + this.step(s.pos.line, this.textOf(s.pos)); + return; + } + case 'aug': { + const loc = this.locate(s.target); + const cur = loc.get(); + if (cur === undefined) throw new Refuse('updating a moved-out value', s.pos); + const r = this.eval(s.value); + const op = s.op.slice(0, -1); + const next = cur.k === 'struct' ? this.callDunder(cur, inplace(op), [r], s.pos, loc) ?? this.binary(op, cur, r, s.pos) : this.binary(op, cur, r, s.pos); + if (cur.k !== 'struct' || next !== undefined) loc.set(next); + this.writes.push({ name: loc.name, frame: loc.frame, path: loc.path }); + this.step(s.pos.line, this.textOf(s.pos)); + return; + } + case 'if': { + const t = truthy(this.eval(s.test)); + this.step(s.pos.line, `${this.condition(s.pos)} → ${t ? 'yes' : 'no'}`, 'test'); + const arm = t ? s.body : s.orelse; + this.destroyDead(arm.length ? this.frame.life?.liveIn.get(arm) : this.frame.life?.liveOut.get(s), s.pos.line); + this.block(arm); + return; + } + case 'while': { + const loop = { kind: 'While', line: s.pos.line, pass: 0 }; + this.loops.push(loop); + let broke = false; + try { + for (;;) { + const t = truthy(this.eval(s.test)); + this.step(s.pos.line, `${this.condition(s.pos)} → ${t ? 'yes' : 'no'}`, 'test'); + if (!t) break; + loop.pass++; + this.destroyDead(this.frame.life?.liveIn.get(s.body), s.pos.line); + try { this.block(s.body); } catch (e) { if (e instanceof BreakSignal) { broke = true; break; } if (!(e instanceof ContinueSignal)) throw e; } + } + } finally { this.loops.pop(); } + this.destroyDead(broke || !s.orelse.length ? this.frame.life?.liveOut.get(s) : this.frame.life?.liveIn.get(s.orelse), s.pos.line); + if (!broke) this.block(s.orelse); // while ... else: runs when the loop ends without break + return; + } + case 'for': { + const items = this.iterate(s.iter, s.pos, s.convention); + const loop = { kind: 'For', line: s.pos.line, pass: 0 }; + this.loops.push(loop); + let broke = false; + try { + for (const item of items) { + loop.pass++; + this.destroyDead(this.frame.life?.liveIn.get(s.body), s.pos.line); + this.bindTarget(s.target, item, s.pos, s.convention); + this.step(s.pos.line, `${targetText(s.target)} = ${display(item.get())}`, 'loop'); + try { this.block(s.body); } catch (e) { if (e instanceof BreakSignal) { broke = true; break; } if (!(e instanceof ContinueSignal)) throw e; } + } + // the loop runs out of items: a step at its header, as a while loop's final test is one + if (!broke) this.step(s.pos.line, 'no more items', 'loop'); + } finally { this.loops.pop(); } + this.destroyDead(broke || !s.orelse.length ? this.frame.life?.liveOut.get(s) : this.frame.life?.liveIn.get(s.orelse), s.pos.line); + if (!broke) this.block(s.orelse); + return; + } + case 'break': this.step(s.pos.line, 'break'); throw new BreakSignal(); + case 'continue': this.step(s.pos.line, 'continue'); throw new ContinueSignal(); + case 'return': { + if (s.value?.k === 'name' && this.isTracked(this.frame.cells.get(s.value.id)?.v)) throw new Refuse('returning a variable whose lifecycle can be seen (when it moves is not modeled)', s.pos); + const v = s.value ? this.take(s.value, false) : null; + if (v) this.claim(v); + // the result is the caller's now; everything else this frame owns dies here + const returned = v; + this.destroyDead(new Set([...this.frame.cells.keys()].filter((n) => this.frame.cells.get(n)?.v === returned)), s.pos.line); + this.step(s.pos.line, this.textOf(s.pos), 'return'); + throw new ReturnSignal(v); + } + case 'raise': { + const v = s.value ? this.eval(s.value) : null; + this.step(s.pos.line, this.textOf(s.pos), 'raise'); + if (!v) throw new Refuse('re-raising', s.pos); + throw new Raised(v.k === 'str' ? { k: 'error', msg: v.v } : v, s.pos); + } + case 'try': { + try { + try { this.block(s.body); } + catch (e) { + if (!(e instanceof Raised) || !s.handlers.length) throw e; + const h = s.handlers[0]; + if (h.type) throw new Refuse('an except clause with a type', h.pos); + if (h.name) this.declareVar(h.name, e.value, 'Error'); + this.step(h.pos.line, this.textOf(h.pos), 'except'); + this.block(h.body); + return; + } + this.block(s.orelse); + } finally { + if (s.finalbody.length) this.block(s.finalbody); + } + return; + } + case 'assert': this.step(s.pos.line, `${this.textOf(s.pos)} (not checked in this build)`); return; // probes: assert and debug_assert are off + case 'with': { this.withBlock(s); return; } + case 'def': { + // a closure: what it captures is decided now, by its capture list (the manual's closures chapter) + this.declareVar(s.name, this.closure(s, s.pos), 'closure'); + this.step(s.pos.line, `${s.name}: a closure`, 'stmt'); + return; + } + case 'struct': case 'trait': throw new Refuse('a struct inside a function', s.pos); + case 'from': case 'import': for (const n of s.k === 'from' ? s.names : []) this.imported.add(n.alias ?? n.name); this.step(s.pos.line, this.textOf(s.pos)); return; + default: throw new Refuse(`a ${s.k} statement`, s.pos); + } + } + + /** + * `with m as v:` over the student's context manager: __enter__ (its result is v), the body, then + * __exit__(self) when the body ends, or __exit__(self, err) -> Bool when it raises (True swallows + * the error). std.testing's assert_raises: the body must raise (with contains=, a message holding + * that text); how it reports when that fails is not modeled, so that is declined. + */ + private withBlock(s: Extract<Stmt, { k: 'with' }>): void { + if (s.items.length !== 1) throw new Refuse('a with of several managers', s.pos); + const it = s.items[0]; + if (it.e.k === 'call' && it.e.fn.k === 'name' && it.e.fn.id === 'assert_raises' && this.imported.has('assert_raises')) { + const contains = it.e.args.find((a) => a.name === 'contains'); + const want = contains ? show(this.eval(contains.value)) : null; + this.step(s.pos.line, this.textOf(s.pos)); + try { this.block(s.body); } + catch (e) { + if (!(e instanceof Raised)) throw e; + if (want !== null && !show(e.value).includes(want)) throw new Refuse('an assert_raises whose error does not contain the text (how it reports that is not modeled)', s.pos); + this.step(s.pos.line, `raised as expected: ${show(e.value)}`, 'except'); + return; + } + throw new Refuse('an assert_raises whose body does not raise (how it reports that is not modeled)', s.pos); + } + const mgr = this.eval(it.e); + if (mgr.k !== 'struct' || !this.structs.has(mgr.name)) throw new Refuse(`a with over ${typeName(mgr)}`, s.pos); + if (this.isTracked(mgr)) throw new Refuse('a context manager whose lifecycle can be seen', s.pos); + const st = this.structs.get(mgr.name)!; + const exits = st.body.filter((b): b is DefStmt => b.k === 'def' && b.name === '__exit__'); + const plain = exits.find((d) => d.params.filter((p) => !p.star).length === 1); + const onError = exits.find((d) => d.params.filter((p) => !p.star).length === 2); + const cell: Cell = { v: mgr, type: mgr.name }; + const loc: Loc = { get: () => cell.v, set: (v) => { cell.v = v; }, name: '', frame: -1 }; + const enter = this.methodDef(st, '__enter__'); + const entered = enter ? this.callDef(enter, [], { loc, value: mgr, struct: st }, s.pos) : null; + if (it.as) { if (!entered) throw new Refuse('with … as, where __enter__ returns nothing', s.pos); this.bindTarget(it.as, constLoc(entered), s.pos, null); } + this.step(s.pos.line, this.textOf(s.pos)); + try { this.block(s.body); } + catch (e) { + if (e instanceof Raised) { + if (!onError) throw new Refuse(`${mgr.name} has no __exit__ that takes the error`, s.pos); + if (onError.typeParams.length && containsReflect(onError.body)) throw new Refuse(`${mgr.name}'s __exit__ reflects on the error's type`, s.pos); + const swallow = this.callDef(onError, [], { loc, value: cell.v!, struct: st }, s.pos, [e.value]); + if (swallow && truthy(swallow)) return; + throw e; + } + if (e instanceof ReturnSignal || e instanceof BreakSignal || e instanceof ContinueSignal) { if (plain) this.callDef(plain, [], { loc, value: cell.v!, struct: st }, s.pos); throw e; } + throw e; + } + if (!plain) throw new Refuse(`${mgr.name} has no __exit__(self)`, s.pos); + this.callDef(plain, [], { loc, value: cell.v!, struct: st }, s.pos); + } + + /** + * A closure over the current frame. imm, mut, ref and a bare name hold a reference to the outer + * variable (imm sees later updates); var takes a copy now, or the value itself with x^, kept by the + * closure across calls. A default convention alone applies to every outer name the body uses. + */ + private closure(d: DefStmt, at: Pos): V { + const caps = d.captures; + const own = new Set<string>([...d.params.map((p) => p.name), ...d.typeParams.map((p) => p.name)]); + const visitDecls = (x: any) => { if (!x || typeof x !== 'object') return; if (Array.isArray(x)) { x.forEach(visitDecls); return; } if (x.k === 'var' && x.targets?.k === 'name') own.add(x.targets.id); if (x.k === 'for' && x.target?.k === 'name') own.add(x.target.id); if (x.k === 'def') own.add(x.name); for (const v of Object.values(x)) if (v && typeof v === 'object') visitDecls(v); }; + visitDecls(d.body); + const captures = new Map<string, { loc?: Loc; cell?: Cell }>(); + const used = new Set<string>(); + const visitNames = (x: any) => { if (!x || typeof x !== 'object') return; if (Array.isArray(x)) { x.forEach(visitNames); return; } if (x.k === 'name') used.add(x.id); if (x.k === 'str' && x.t) for (const n of x.value.match(/[A-Za-z_]\w*/g) ?? []) used.add(n); for (const v of Object.values(x)) if (v && typeof v === 'object') visitNames(v); }; + visitNames(d.body); + // an outer name bound to a function or closure is called, not captured: visible without a capture list + const isFunction = (n: string) => { const v = this.frame.cells.get(n)?.v; return v?.k === 'closure' || v?.k === 'fn'; }; + const outer = [...used].filter((n) => !own.has(n) && (this.frame.cells.has(n) || this.frame.aliases.has(n)) && !isFunction(n)); + for (const n of used) if (!own.has(n) && isFunction(n)) captures.set(n, { loc: this.lookup(n, at)! }); + if (!caps) { + if (outer.length) throw new Refuse(`${d.name} uses ${outer[0]} from outside without a capture list (the default is not modeled)`, at); + return { k: 'closure', name: d.name, def: d, captures }; + } + // {var^}: copies, and makes the closure copyable (assignable to another variable), which function values here always are + const defaultConv = caps.defaultConv === 'var^' ? 'var' : caps.defaultConv; + if (defaultConv?.endsWith('^')) throw new Refuse(`the capture list {${caps.defaultConv}}`, at); + const take = (name: string, conv: string, move: boolean, pos: Pos) => { + const loc = this.lookup(name, pos); + if (!loc) throw new Refuse(`capturing ${name}, which is not a variable here`, pos); + if (conv === 'var') { + const v0 = loc.get(); + if (v0 === undefined) throw new Refuse(`capturing ${name} after it was moved out`, pos); + if (move) { loc.set(undefined); this.writes.push({ name: loc.name, frame: loc.frame }); } + captures.set(name, { cell: { v: move ? v0 : deepCopy(v0), type: typeName(v0) } }); + } else captures.set(name, { loc }); + }; + for (const it of caps.items) take(it.name, it.conv === 'read' ? 'imm' : it.conv, it.move, it.pos); + for (const n of outer) { + if (captures.has(n)) continue; + if (!defaultConv) throw new Refuse(`${d.name} uses ${n}, which its capture list does not name`, at); + take(n, defaultConv, false, at); + } + return { k: 'closure', name: d.name, def: d, captures }; + } + + /** Call a function value: a closure, or a named function passed along. */ + private callValue(v: V, args: Arg[], at: Pos): V { + if (v.k === 'closure') return this.callDef(v.def, args, null, at, undefined, [], v.captures as Map<string, { loc?: Loc; cell?: Cell }>) ?? NONE; + if (v.k === 'fn' && this.functions.has(v.name) && !this.overloads.has(v.name)) return this.callDef(this.functions.get(v.name)!, args, null, at) ?? NONE; + throw new Refuse(`calling a ${typeName(v)}`, at); + } + + private assign(t: Expr, v: V): void { + if (t.k === 'name') { + if (t.id === '_') return; + const loc = this.lookup(t.id, t.pos); + if (!loc) { this.declareVar(t.id, v, typeName(v)); return; } + if (this.isTracked(v) && this.frame.life?.unmodeled.has(t.id)) throw new Refuse(`when ${t.id} is destroyed: it is used where Programming+ does not follow the flow yet`, t.pos); + loc.set(loc.type ? this.coerceTo(v, loc.type) : v); + this.writes.push({ name: loc.name, frame: loc.frame, path: loc.path }); + return; + } + if (t.k === 'tuple') { + if (v.k !== 'tuple' || v.items.length !== t.items.length) throw new Refuse('unpacking this value', t.pos); + t.items.forEach((x, i) => this.assign(x, v.items[i])); + return; + } + if (t.k === 'index') { + const obj = this.eval(t.obj); + if (obj.k === 'struct') { this.callMethod(obj, '__setitem__', [...t.args.map((a) => this.eval(a.value)), v], t.pos, this.locate(t.obj)); return; } + } + const loc = this.locate(t); + loc.set(loc.type ? this.coerceTo(v, loc.type) : v); + this.writes.push({ name: loc.name, frame: loc.frame, path: loc.path }); + } + + /** The type a struct's field was declared with, as written. */ + private fieldType(struct: string, field: string): string | undefined { + const decl = this.structs.get(struct)?.body.find((b): b is Extract<Stmt, { k: 'var' }> => b.k === 'var' && !b.comptime && b.targets.k === 'name' && b.targets.id === field); + return decl?.type ? typeText(decl.type) : undefined; + } + + /** + * A value going to a new home. `x^` moves it: into new storage (a variable, a field, a list) the move + * constructor runs; into a `var` parameter it does not (life-var-arg). A temporary is used as it is; + * a variable or field without ^ is copied, which for a value whose lifecycle can be seen the compiler + * only allows explicitly (.copy()). + */ + private take(e: Expr, intoStorage = false): V { + if (e.k === 'transfer') { + const loc = this.locate(e.x); + const v = loc.get(); + if (v === undefined) throw new Refuse('moving a value that was already moved', e.pos); + loc.set(undefined); + this.writes.push({ name: loc.name, frame: loc.frame, path: loc.path }); + return intoStorage && this.isTracked(v) ? this.moveInto(v, e.pos) : v; + } + const v = this.eval(e); + if (e.k === 'name' || e.k === 'attr' || e.k === 'index') { + if (this.isTracked(v)) throw new Refuse('an implicit copy of a value whose lifecycle can be seen', e.pos); + return deepCopy(v); + } + this.claim(v); + return v; + } + + private bindTarget(t: Expr, item: Loc, pos: Pos, convention: string | null): void { + if (t.k === 'name') { + if (t.id === '_') return; + if (convention === 'ref') { this.frame.aliases.set(t.id, { ...item, name: t.id, frame: this.frame.id }); this.frame.cells.set(t.id, { v: undefined, type: '' }); this.writes.push({ name: t.id, frame: this.frame.id }); return; } + const v = item.get()!; + this.declareVar(t.id, convention === 'var' ? deepCopy(v) : v, typeName(v)); + return; + } + if (t.k === 'tuple') { + const v = item.get(); + if (v?.k !== 'tuple' || v.items.length !== t.items.length) throw new Refuse('unpacking this loop value', pos); + t.items.forEach((x, i) => this.bindTarget(x, constLoc(v.items[i]), pos, convention)); + return; + } + throw new Refuse('this loop target', pos); + } + + private iterate(e: Expr, pos: Pos, convention: string | null): Loc[] { + const v = this.eval(e); + switch (v.k) { + case 'range': { + const out: Loc[] = []; + if (v.step === 0n) throw new Refuse('a range with step 0', pos); + for (let i = v.start; v.step > 0n ? i < v.stop : i > v.stop; i += v.step) { out.push(constLoc(int(i))); if (out.length > this.maxSteps) break; } + return out; + } + // a loop takes a step per item, so the items past the step limit are never reached and not listed + case 'list': return v.items.slice(0, this.maxSteps + 1).map((_, i) => ({ get: () => v.items[i], set: (x) => { v.items[i] = x!; }, name: '', frame: -1 })); + case 'str': return [...firstItems(v.v, this.maxSteps + 1)].map((c) => constLoc(str(c))); + case 'dict': return [...firstItems(v.map.values(), this.maxSteps + 1)].map(([k]) => constLoc(k)); + case 'set': return [...firstItems(v.map.values(), this.maxSteps + 1)].map((x) => constLoc(x)); + case 'tuple': return v.items.map((x) => constLoc(x)); + default: void convention; throw new Refuse(`a for loop over ${typeName(v)}`, pos); + } + } + + /* ---------------- locations ---------------- */ + + private locate(e: Expr): Loc { + if (e.k === 'name') { + const loc = this.lookup(e.id, e.pos); + if (!loc) throw new Refuse(`the name ${e.id}`, e.pos); + return loc; + } + if (e.k === 'attr') { + const base = this.locate(e.obj); + const obj = base.get(); + if (obj?.k !== 'struct') throw new Refuse(`a field of ${typeName(obj)}`, e.pos); + if (!obj.fields.has(e.name)) throw new Refuse(`the field ${e.name}`, e.pos); + return { get: () => obj.fields.get(e.name), set: (v) => { obj.fields.set(e.name, v); }, name: base.name, frame: base.frame, path: [...(base.path ?? []), e.name], type: this.fieldType(obj.name, e.name) }; + } + if (e.k === 'index') { + const base = this.locate(e.obj); + const obj = base.get(); + if (obj?.k === 'list') { + const i = this.index(obj, e); + return { get: () => obj.items[i], set: (v) => { obj.items[i] = v!; }, name: base.name, frame: base.frame, path: [...(base.path ?? []), String(i)], type: obj.fixed ? undefined : obj.elem }; + } + if (obj?.k === 'dict') { + const k = this.coerceTo(this.eval(e.args[0].value), obj.key); + return { get: () => { const x = obj.map.get(keyOf(k)); if (!x) throw new Raised({ k: 'error', msg: '', kind: `DictKeyError[${obj.key}]` }, e.pos); return x[1]; }, set: (v) => { const key = keyOf(k); if (!obj.map.has(key)) { this.room(obj.map.size + 1, 'Dict', e.pos); this.made(1, e.pos); } this.work(key.length, e.pos); obj.map.set(key, [k, v!]); }, name: base.name, frame: base.frame, path: [...(base.path ?? []), show(k)], type: obj.val }; + } + throw new Refuse(`indexing ${typeName(obj)}`, e.pos); + } + // anything else is a temporary + const v = this.eval(e); + return constLoc(v); + } + + private index(list: Extract<V, { k: 'list' }>, e: Extract<Expr, { k: 'index' }>): number { + if (e.args.length !== 1 || e.args[0].name) throw new Refuse('this subscript', e.pos); + const iv = this.eval(e.args[0].value); + if (iv.k !== 'int') throw new Refuse(`indexing with ${typeName(iv)}`, e.pos); + const i = Number(iv.v); + if (i < 0 || i >= list.items.length) throw new OutOfBounds(`At: /app/example.mojo:${e.lbracket.line}:${e.lbracket.col}: Assert Error: index ${i} is out of bounds, valid range is 0 to ${list.items.length - 1}`, e.lbracket); + return i; + } + + /* ---------------- expressions ---------------- */ + + eval(e: Expr): V { + switch (e.k) { + case 'num': { + if (e.float) return float(Number(e.text.replace(/_/g, ''))); + const t = e.text.replace(/_/g, ''); + return int(BigInt(/^0[oO]/.test(t) ? '0o' + t.slice(2) : t)); + } + case 'str': return str(e.t ? this.tstring(e.value, e.pos) : e.value); + case 'name': return this.read(e.id, e.pos, e); + case 'list': { + // each item is evaluated once: it may call a function that prints, and evaluating the first again for + // its type cost twice the work per level of [[...]] + const items = e.items.map((x) => deepCopy(this.eval(x))); + return { k: 'list', items, elem: items.length ? typeName(items[0]) : 'Int', fixed: true }; + } + case 'tuple': return { k: 'tuple', items: e.items.map((x) => deepCopy(this.eval(x))) }; + case 'dict': { + const map = new Map<string, [V, V]>(); + let key = 'String', val = 'Int'; + for (const [kx, vx] of e.entries) { const k = this.eval(kx), v = deepCopy(this.eval(vx)); key = typeName(k); val = typeName(v); map.set(keyOf(k), [k, v]); } + return { k: 'dict', map, key, val }; + } + case 'set': { + const map = new Map<string, V>(); let elem = 'Int'; + for (const x of e.items) { const v = this.eval(x); elem = typeName(v); map.set(keyOf(v), v); } + return { k: 'set', map, elem }; + } + case 'attr': { + const obj = this.eval(e.obj); + if (obj.k === 'struct') { + if (obj.fields.has(e.name)) { + const v = obj.fields.get(e.name); + if (v === undefined) throw new Refuse(`reading ${e.name} after it was moved out`, e.pos); + return v; + } + const loc = this.tryLocate(e.obj); + return { k: 'fn', name: e.name, self: loc ?? constLoc(obj) }; + } + if (obj.k === 'fn' && this.structs.has(obj.name)) return this.member(this.structs.get(obj.name)!, e.name, e.pos); + if (obj.k === 'type' && this.structs.has(obj.name)) return this.member(this.structs.get(obj.name)!, e.name, e.pos); + throw new Refuse(`${typeName(obj)}.${e.name}`, e.pos); + } + case 'index': { + const obj = this.eval(e.obj); + if (obj.k === 'list') return obj.items[this.index(obj, e)]; + if (obj.k === 'dict') { const k = this.coerceTo(this.eval(e.args[0].value), obj.key); const x = obj.map.get(keyOf(k)); if (!x) throw new Raised({ k: 'error', msg: '', kind: `DictKeyError[${obj.key}]` }, e.pos); return x[1]; } + if (obj.k === 'str') return this.stringIndex(obj.v, e); + if (obj.k === 'simd' && e.args.length === 1 && !e.args[0].name) { const i = Number(this.intOf(this.eval(e.args[0].value), e.pos)); if (i < 0 || i >= obj.items.length) throw new Refuse('a SIMD lane out of range', e.pos); return obj.items[i]; } + if (obj.k === 'tuple' && e.args.length === 1 && !e.args[0].name) { const i = Number(this.intOf(this.eval(e.args[0].value), e.pos)); if (i < 0 || i >= obj.items.length) throw new Refuse('a tuple index out of range', e.pos); return obj.items[i]; } + if (obj.k === 'struct') return this.callMethod(obj, '__getitem__', e.args.map((a) => this.eval(a.value)), e.pos, this.tryLocate(e.obj)) ?? NONE; + if (obj.k === 'fn' && ['List', 'Dict', 'Set', 'Optional', ...SCALAR_TYPES].includes(obj.name)) return { k: 'type', name: obj.name, args: e.args.map((a) => typeText(a.value)) }; + if (e.obj.k === 'name' && e.obj.id === 'SIMD' && !this.lookup('SIMD', e.pos)) return { k: 'type', name: 'SIMD', args: e.args.map((a) => typeText(a.value)) }; + throw new Refuse(`indexing ${typeName(obj)}`, e.pos); + } + case 'call': return this.inBucket(e.lparen.line, () => this.call(e)); + case 'unary': { + const x = this.eval(e.x); + if (x.k === 'struct') return this.callMethod(x, ({ '-': '__neg__', '+': '__pos__', '~': '__invert__' } as Record<string, string>)[e.op], [], e.pos, null) ?? NONE; + if (e.op === '-' && x.k === 'int') return int(-x.v, x.type); + if (e.op === '-' && x.k === 'float') return float(-x.v, x.bits); + if (e.op === '+') return x; + if (e.op === '~' && x.k === 'int') return int(~x.v, x.type); + throw new Refuse(`${e.op} on ${typeName(x)}`, e.pos); + } + case 'not': return bool(!truthy(this.eval(e.x))); + case 'bool': { + const l = this.eval(e.l); + if (e.op === 'and') return truthy(l) ? this.eval(e.r) : l; + return truthy(l) ? l : this.eval(e.r); + } + case 'binary': { + // literals only, a float among them: Mojo computes it exactly at compile time and rounds once + // (manual/types/numeric_literals_and_conversions: 3.0 * (4.0 / 3.0 - 1.0) is 1.0, not 0.9999999999999998) + const exact = foldLiteral(e, this.literalName); + if (exact && exact.float) return float(ratioToDouble(exact.n, exact.d)); + return this.binaryExpr(e); + } + default: return this.evalRest(e); + } + } + + private binaryExpr(e: Extract<Expr, { k: 'binary' }>): V { + return this.inBucket(e.pos.line, () => { + const [l, r] = this.meet(e.l, this.eval(e.l), e.r, this.eval(e.r)); + if (l.k === 'struct') return this.callDunder(l, dunder(e.op), [r], e.pos, null) ?? (() => { throw new Refuse(`${e.op} on ${l.name}`, e.pos); })(); + return this.binary(e.op, l, r, e.pos); + }); + } + + /** + * A numeric literal takes the type of the value it meets, so beside a Float32 it is a Float32: for a Float32 x + * holding 0.1, `x == 0.1` is True and `x + 0.1` adds two singles (the compiler's answer for the Mojo made from + * library/pseudo/programs/50-float32, found by `pixi run mojo-local`). Two values of different float types + * do not meet at all in Mojo, so only a literal is converted here. + */ + private meet(le: Expr, l: V, re: Expr, r: V): [V, V] { + const single = (v: V): boolean => v.k === 'float' && v.bits === 32; + const literal = (e: Expr, v: V): boolean => (v.k === 'float' || v.k === 'int') && !single(v) && foldLiteral(e, this.literalName) !== null; + if (single(r) && literal(le, l)) return [float(Number((l as any).v), 32), r]; + if (single(l) && literal(re, r)) return [l, float(Number((r as any).v), 32)]; + return [l, r]; + } + + /** + * A name that is a literal of std.math: `pi` is a FloatLiteral of 64 digits, so beside other literals it is folded + * exactly at compile time (pi - 3.141592653589793 prints 2.384626433832795e-16, not 0.0: probes/math-pi-literal). + */ + private literalName = (name: string, pos: Pos): string | null => + name === 'pi' && this.imported.has('pi') && !this.lookup(name, pos) && !this.globals.has(name) ? PI_DIGITS : null; + + private evalRest(e: Expr): V { + switch (e.k) { + case 'compare': { + let left = this.eval(e.operands[0]); + for (let i = 0; i < e.ops.length; i++) { + const right = this.eval(e.operands[i + 1]); + // two literals are compared exactly, at compile time: pi == 3.141592653589793 is False (probes/math-pi-literal) + const a = foldLiteral(e.operands[i], this.literalName), b = a && foldLiteral(e.operands[i + 1], this.literalName); + if (a && b && (a.float || b.float) && ['==', '!=', '<', '<=', '>', '>='].includes(e.ops[i])) { + const c = a.n * b.d - b.n * a.d; // denominators are positive + const holds = ({ '==': c === 0n, '!=': c !== 0n, '<': c < 0n, '<=': c <= 0n, '>': c > 0n, '>=': c >= 0n } as Record<string, boolean>)[e.ops[i]]; + if (!holds) return bool(false); + left = right; + continue; + } + const [l, r] = this.meet(e.operands[i], left, e.operands[i + 1], right); + if (!this.compare(e.ops[i], l, r, e.opPos[i])) return bool(false); + left = right; + } + return bool(true); + } + case 'ifexp': return truthy(this.eval(e.cond)) ? this.eval(e.a) : this.eval(e.b); + case 'transfer': return this.take(e, false); + case 'lambda': { + // a lambda captures what it uses as imm: a reference, so a later call sees new values (manual: functions/lambda) + const params: Param[] = e.typed ?? e.params.map((n) => ({ name: n, convention: null, type: null, default: null, pos: e.pos })); + const d: DefStmt = { k: 'def', name: 'lambda', params, ret: null, raises: false, typeParams: [], decorators: [], body: [{ k: 'return', value: e.body, pos: e.body.pos }], pos: e.pos, colon: e.pos, captures: e.captures ? { defaultConv: e.captures.defaultConv ?? 'imm', items: e.captures.items } : { defaultConv: 'imm', items: [] } }; + return this.closure(d, e.pos); + } + default: throw new Refuse(`a ${e.k} expression`, e.pos); + } + } + + /** A compile-time parameter's value: a number, a type, a string. */ + private paramValue(e: Expr): V { + if (e.k === 'name' && (this.structs.has(e.id) || SCALAR_TYPES.has(e.id) || ['List', 'Dict', 'Set', 'Optional'].includes(e.id))) return { k: 'type', name: e.id, args: [] }; + return this.eval(e); + } + + /** Evaluate inside a bucket: what dies there is destroyed when it is done; a tracked result is a temporary of the enclosing one. */ + private inBucket(line: number, f: () => V): V { + this.openBucket(); + let v: V; + try { v = f(); } catch (e) { const b = this.buckets.pop(); if (b?.length && !(e instanceof Refuse)) throw new Refuse('values destroyed while an error is raised (unwinding is not modeled)', null); throw e; } + if (this.isTracked(v)) this.claim(v); + this.closeBucket(line); + if (this.isTracked(v)) { + const fresh = !this.created.has(v); + this.born(v); + if (fresh && v.k === 'struct') this.event('made', v); + this.doom({ v, label: this.labelOf(v) }); + } + return v; + } + + private tryLocate(e: Expr): Loc | null { + try { return e.k === 'name' || e.k === 'attr' || e.k === 'index' ? this.locate(e) : null; } catch (x) { if (x instanceof Refuse) return null; throw x; } + } + + private tstring(raw: string, pos: Pos): string { + let out = '', i = 0; + while (i < raw.length) { + if (raw.startsWith('{{', i)) { out += '{'; i += 2; continue; } + if (raw.startsWith('}}', i)) { out += '}'; i += 2; continue; } + if (raw[i] === '{') { + let d = 0, j = i; + for (; j < raw.length; j++) { if (raw[j] === '{') d++; else if (raw[j] === '}' && --d === 0) break; } + const inner = raw.slice(i + 1, j); + if (/[^=!<>]=[^=]|![rsa]$|:[^\]]*$/.test(inner) && !/^\s*[\w.()\[\] +\-*/%]+\s*$/.test(inner)) throw new Refuse('a t-string with a format specification', pos); + const m = parseModule(`def __t():\n return (${inner})\n`); + const d0 = m.body[0]; + if (m.unread || d0?.k !== 'def' || d0.body[0]?.k !== 'return' || !d0.body[0].value) throw new Refuse('this t-string', pos); + out += show(this.eval(d0.body[0].value), (s) => this.writeTo(s)); + i = j + 1; + continue; + } + out += raw[i]; i++; + } + this.grow(out.length, out.length, pos); + return cookT(out); + } + + private stringIndex(s: string, e: Extract<Expr, { k: 'index' }>): V { + this.work(s.length, e.pos); // an index or a slice reads the String from its start + const a = e.args[0]; + if (e.args.length !== 1 || (a.name !== 'byte' && a.name !== 'codepoint')) throw new Refuse('indexing a String this way (use byte= or codepoint=)', e.pos); + if (a.value.k === 'slice') { + if (a.name !== 'codepoint' || a.value.step) throw new Refuse('this String slice', e.pos); + const cps = codepoints(s); + const lo = a.value.lo ? Number(this.intOf(this.eval(a.value.lo), e.pos)) : 0, hi = a.value.hi ? Number(this.intOf(this.eval(a.value.hi), e.pos)) : cps.length; + return str(cps.slice(lo, hi).join('')); + } + const i = Number(this.intOf(this.eval(a.value), e.pos)); + // past either end the program aborts, in the standard library's words (probes/string-codepoint-oob, string-codepoint-negative) + if (a.name === 'codepoint') { const cps = codepoints(s); if (i < 0 || i >= cps.length) throw new OutOfBounds(`ABORT: Mojo/stdlib/std/collections/string/string_span.mojo:438:18: codepoint index is out of bounds: ${i}`, e.pos); return str(cps[i]); } + const bytes = utf8(s); + if (i < 0 || i >= bytes.length || bytes[i] >= 0x80) throw new Refuse('a byte that is not ASCII, or out of range', e.pos); + return str(String.fromCharCode(bytes[i])); + } + + /** A struct that conforms to Equatable without its own __eq__ gets the trait's default: every field equal. */ + private reflectiveEq(v: Extract<V, { k: 'struct' }>): boolean { + const st = this.structs.get(v.name); + return !!st && st.bases.some((b) => b.k === 'name' && (b.id === 'Equatable' || b.id === 'Comparable')); + } + + private equals(a: V, b: V, pos: Pos): boolean { + if (a.k === 'struct' && b.k === 'struct' && a.name === b.name) { + const st = this.structs.get(a.name); + if (st && this.methodDef(st, '__eq__')) return truthy(this.callMethod(a, '__eq__', [b], pos, null)!); + if (this.reflectiveEq(a)) { for (const [n, x] of a.fields) { const y = b.fields.get(n); if (!x || !y || !this.equals(x, y, pos)) return false; } return true; } + } + const e = equal(a, b); + if (e === null) throw new Refuse(`comparing ${typeName(a)} with ${typeName(b)}`, pos); + return e; + } + + /** The scalar type of a dtype's lanes; Float16 and BFloat16 print in ways not verified, so they are declined. */ + private lane(dtype: string, pos: Pos): (x: V) => V { + const ints: Record<string, string> = { int8: 'Int8', int16: 'Int16', int32: 'Int32', int64: 'Int64', uint8: 'UInt8', uint16: 'UInt16', uint32: 'UInt32', uint64: 'UInt64', int: 'Int', index: 'Int' }; + if (ints[dtype]) { const t = ints[dtype]; return (x) => (x.k === 'int' ? int(x.v, t) : x.k === 'float' && Number.isFinite(x.v) ? int(BigInt(Math.trunc(x.v)), t) : (() => { throw new Refuse(x.k === 'float' ? `converting ${show(x)} to an integer lane` : `a ${typeName(x)} lane of an integer SIMD`, pos); })()); } + if (dtype === 'float32' || dtype === 'float64') { const bits = dtype === 'float32' ? 32 : 64; return (x) => (x.k === 'int' || x.k === 'float' ? float(Number(x.v), bits) : (() => { throw new Refuse(`a ${typeName(x)} lane of a float SIMD`, pos); })()); } + if (dtype === 'bool') return (x) => bool(truthy(x)); + throw new Refuse(`SIMD of DType.${dtype}`, pos); + } + + private simd(typeArgs: string[], args: V[], at: Pos, scalar: boolean): V { + const [dtype, w] = typeArgs; + const width = Number(w); + if (!dtype || !Number.isInteger(width) || width < 1) throw new Refuse('this SIMD type', at); + const lane = this.lane(dtype, at); + if (scalar || width === 1) { if (args.length > 1) throw new Refuse('a Scalar of several values', at); return lane(args[0] ?? int(0n)); } + const items = args.length === 1 ? Array.from({ length: width }, () => lane(args[0])) : args.length === width ? args.map(lane) : null; + if (!items) throw new Refuse(`a SIMD of width ${width} built from ${args.length} values`, at); + return { k: 'simd', dtype, items }; + } + + private intOf(v: V, pos: Pos): bigint { if (v.k !== 'int') throw new Refuse(`using ${typeName(v)} as an Int`, pos); return v.v; } + + private binary(op: string, l: V, r: V, pos: Pos): V { + if (l.k === 'simd' || r.k === 'simd') { + if (!['+', '-', '*', '/', '//', '%'].includes(op)) throw new Refuse(`${op} on a SIMD`, pos); + const s = (l.k === 'simd' ? l : r) as Extract<V, { k: 'simd' }>; + const lane = this.lane(s.dtype, pos); + const at = (x: V, i: number) => (x.k === 'simd' ? x.items[i] : lane(x)); + if (l.k === 'simd' && r.k === 'simd' && (l.items.length !== r.items.length || l.dtype !== r.dtype)) throw new Refuse('SIMD values of different types', pos); + return { k: 'simd', dtype: s.dtype, items: s.items.map((_, i) => lane(this.binary(op, at(l, i), at(r, i), pos))) }; + } + if (l.k === 'int' && r.k === 'int') { + if (l.type !== r.type && l.type !== 'Int' && r.type !== 'Int') throw new Refuse(`${l.type} ${op} ${r.type}`, pos); + const t = l.type !== 'Int' ? l.type : r.type; + switch (op) { + case '+': return int(l.v + r.v, t); case '-': return int(l.v - r.v, t); case '*': return int(l.v * r.v, t); + case '/': return int(truncdiv(l.v, r.v), t); case '//': return int(floordiv(l.v, r.v), t); case '%': return int(mod(l.v, r.v), t); + case '**': if (r.v < 0n) throw new Refuse('a negative power of an Int', pos); return int(BigInt.asIntN(64, l.v ** (r.v > 200n ? 200n : r.v)), t); + case '&': return int(l.v & r.v, t); case '|': return int(l.v | r.v, t); case '^': return int(l.v ^ r.v, t); + // a shift past the width would ask the engine for a number of that many bits (1 << 10**18) + case '<<': if (r.v > 64n) throw new Refuse('a shift by more than 64 bits', pos); return int(l.v << r.v, t); + case '>>': return int(l.v >> r.v, t); + } + } + if (l.k === 'float' || r.k === 'float') { + const a = num(l), b = num(r); + if (a === null || b === null) throw new Refuse(`${typeName(l)} ${op} ${typeName(r)}`, pos); + const bits: 32 | 64 = l.k === 'float' ? l.bits : (r as any).bits; + switch (op) { + case '+': return float(a + b, bits); case '-': return float(a - b, bits); case '*': return float(a * b, bits); case '/': return float(a / b, bits); + case '//': return float(Math.floor(a / b), bits); case '%': return float(a - b * Math.floor(a / b), bits); + case '**': { + // Mojo raises a float to a whole power by squaring (probes/float-pow-whole: 1.1 ** 20 is 6.727499949325609, where the + // correctly rounded power is ...611), and to any other power with a routine of its own that is good to about 13 + // digits (probes/float-pow-accuracy), which is not reproduced here + if (!Number.isInteger(b) || Math.abs(b) >= 2 ** 31) throw new Refuse('a float raised to a power that is not a whole number (Mojo computes it with its own routine, to about 13 digits)', pos); + let r = 1, x = bits === 32 ? Math.fround(a) : a; + for (let n = Math.abs(b); n > 0; n = Math.floor(n / 2)) { if (n % 2 === 1) r = bits === 32 ? Math.fround(r * x) : r * x; x = bits === 32 ? Math.fround(x * x) : x * x; } + return float(b < 0 ? 1 / r : r, bits); + } + } + } + // the whole length counts: every step's String is kept by the trace, and reading one makes it whole + if (l.k === 'str' && r.k === 'str' && op === '+') { this.grow(l.v.length + r.v.length, l.v.length + r.v.length, pos); return str(l.v + r.v); } + if (l.k === 'str' && r.k === 'int' && op === '*') { const n = r.v > 0n && l.v ? Number(r.v) : 0; this.grow(l.v.length * n, l.v.length * n, pos); return str(l.v.repeat(n)); } + if (l.k === 'list' && r.k === 'list' && op === '+') { this.room(l.items.length + r.items.length, 'List', pos); this.made(l.items.length + r.items.length, pos); return { ...l, items: l.items.concat(r.items).map(deepCopy), fixed: false }; } + throw new Refuse(`${typeName(l)} ${op} ${typeName(r)}`, pos); + } + + private compare(op: string, l: V, r: V, pos: Pos): boolean { + if (op === 'in' || op === 'not in') { + let has: boolean; + if (r.k === 'list' || r.k === 'tuple') { this.work(r.items.length, pos); has = r.items.some((x) => this.equals(x, l, pos)); } + else if (r.k === 'dict' || r.k === 'set') has = r.map.has(keyOf(l)); + else if (r.k === 'str' && l.k === 'str') { this.work(r.v.length, pos); has = r.v.includes(l.v); } + else if (r.k === 'struct') has = truthy(this.callMethod(r, '__contains__', [l], pos, null) ?? bool(false)); + else throw new Refuse(`in on ${typeName(r)}`, pos); + return op === 'in' ? has : !has; + } + if (op === 'is' || op === 'is not') { + const same = (l.k === 'none' || (l.k === 'opt' && !l.v)) && (r.k === 'none' || (r.k === 'opt' && !r.v)); + if (r.k !== 'none' && l.k !== 'none') throw new Refuse('is on these values', pos); + return op === 'is' ? same : !same; + } + if (l.k === 'struct' && (op === '==' || op === '!=') && !this.methodDef(this.structs.get(l.name)!, '__eq__') && this.reflectiveEq(l)) { + const eq = this.equals(l, r, pos); return op === '==' ? eq : !eq; + } + if (l.k === 'struct') { + const name = ({ '==': '__eq__', '!=': '__ne__', '<': '__lt__', '<=': '__le__', '>': '__gt__', '>=': '__ge__' } as Record<string, string>)[op]; + const res = this.callDunder(l, name, [r], pos, null); + if (res) return truthy(res); + if (op === '!=') { const eq = this.callDunder(l, '__eq__', [r], pos, null); if (eq) return !truthy(eq); } + throw new Refuse(`${op} on ${l.name}`, pos); + } + if (op === '==' || op === '!=') { const eq = this.equals(l, r, pos); return op === '==' ? eq : !eq; } + const a = l.k === 'str' ? l.v : num(l), b = r.k === 'str' ? r.v : num(r); + if (a === null || b === null || typeof a !== typeof b) throw new Refuse(`${op} on ${typeName(l)} and ${typeName(r)}`, pos); + switch (op) { case '<': return a < b; case '<=': return a <= b; case '>': return a > b; case '>=': return a >= b; } + throw new Refuse(`the comparison ${op}`, pos); + } + + /* ---------------- calls ---------------- */ + + private call(e: Extract<Expr, { k: 'call' }>): V { + let fn: Expr = e.fn; + let typeArgs: string[] = []; + if (fn.k === 'index' && fn.obj.k === 'name' && (this.structs.has(fn.obj.id) || ['List', 'Dict', 'Set', 'Optional', ...SCALAR_TYPES].includes(fn.obj.id) || this.functions.has(fn.obj.id))) { + typeArgs = fn.args.map((a) => typeText(a.value)); + if (this.functions.has(fn.obj.id) && fn.args.length) { + const def = this.functions.get(fn.obj.id)!; + if (fn.args.some((a) => a.name) || fn.args.length > def.typeParams.filter((p) => !p.star).length) throw new Refuse('these compile-time parameters', e.pos); + const params = fn.args.map((a, i) => [def.typeParams[i].name, this.paramValue(a.value)] as [string, V]); + return this.callDef(def, e.args, null, e.lparen, undefined, params) ?? NONE; + } + fn = fn.obj; + } + // std.math's inf[DType.float64]() and nan[...](): every NaN prints as nan (probes/math-inf-nan) + if (fn.k === 'index' && fn.obj.k === 'name' && (fn.obj.id === 'inf' || fn.obj.id === 'nan') && this.imported.has(fn.obj.id) && !this.lookup(fn.obj.id, fn.pos) && fn.args.length === 1 && !e.args.length) { + const dt = typeText(fn.args[0].value); + if (dt === 'float64' || dt === 'float32') return float(fn.obj.id === 'inf' ? Infinity : NaN, dt === 'float32' ? 32 : 64); + } + // SIMD[dtype, width](...) and Scalar[dtype](...), directly or through a comptime alias of the type + if (fn.k === 'index' && fn.obj.k === 'name' && (fn.obj.id === 'SIMD' || fn.obj.id === 'Scalar') && !this.lookup(fn.obj.id, fn.pos)) + return this.simd(fn.obj.id === 'Scalar' ? [typeText(fn.args[0].value), '1'] : fn.args.map((a) => typeText(a.value)), this.args(e.args, e.lparen, 'SIMD'), e.lparen, fn.obj.id === 'Scalar'); + if (fn.k === 'name' && this.globals.get(fn.id)?.k === 'type' && (this.globals.get(fn.id) as any).name === 'SIMD') + return this.simd((this.globals.get(fn.id) as any).args, this.args(e.args, e.lparen, fn.id), e.lparen, false); + if (fn.k === 'name' && fn.id === 'Self' && this.frame.selfType) return this.construct(this.structs.get(this.frame.selfType)!, e.args, e.lparen); + if (fn.k === 'name' && !this.lookup(fn.id, fn.pos)) { + const name = fn.id; + const st = this.structs.get(name); + if (st) return this.construct(st, e.args, e.pos); + const set = this.overloads.get(name); + if (set) return this.callDef(this.pick(set, e.args, null, e.lparen, name), e.args, null, e.lparen) ?? NONE; + const def = this.functions.get(name); + if (def) return this.callDef(def, e.args, null, e.lparen) ?? NONE; + return this.builtin(name, e.args, typeArgs, e); + } + // a function value in a variable or parameter: a closure, or a named function passed along + if (fn.k === 'name' && this.lookup(fn.id, fn.pos)) return this.callValue(this.read(fn.id, fn.pos), e.args, e.lparen); + if (fn.k === 'attr') { + const objLoc = this.tryLocate(fn.obj); + const obj = objLoc ? objLoc.get() : this.eval(fn.obj); + if (obj === undefined) throw new Refuse(`calling a method on ${targetText(fn.obj)} after it was moved out`, e.pos); + if (obj.k === 'struct') { + const args = e.args; + return this.methodCall(obj, fn.name, args, e.pos, objLoc); + } + if (obj.k === 'fn' && this.structs.has(obj.name)) { + const st = this.structs.get(obj.name)!; + const def = this.methodDef(st, fn.name); + if (!def || !def.decorators.some((d) => d.k === 'name' && d.id === 'staticmethod')) throw new Refuse(`${obj.name}.${fn.name}(...)`, e.pos); + return this.callDef(def, e.args, null, e.lparen) ?? NONE; + } + return this.method(obj, fn.name, e.args.map((a) => { if (a.name) throw new Refuse(`keyword arguments to ${fn.k === 'attr' ? (fn as any).name : ''}`, e.pos); return this.eval(a.value); }), e, objLoc); + } + throw new Refuse('calling this', e.pos); + } + + /** Bind arguments to a function's parameters by convention, and run it. */ + private callDef(def: DefStmt, args: Arg[], self: { loc: Loc | null; value: V; struct: StructStmt } | null, at: Pos, prebound?: V[], params0: Array<[string, V]> = [], captures?: Map<string, { loc?: Loc; cell?: Cell }>): V | null { + const frame: Frame = { id: this.nextFrameId++, name: self ? `${self.struct.name}.${def.name}` : def.name, cells: new Map(), aliases: new Map() }; + // a closure's captures: a reference to the outer variable, or the closure's own copy (kept across calls) + for (const [name, c] of captures ?? []) { + if (c.cell) frame.cells.set(name, c.cell); + else if (c.loc) { frame.aliases.set(name, { ...c.loc, name, frame: frame.id }); frame.cells.set(name, { v: undefined, type: '' }); } + } + for (const [n, v] of params0) frame.cells.set(n, { v, type: typeName(v) }); + if (self) frame.selfType = self.struct.name; + const params = def.params.filter((p) => p.star !== 'marker' && p.star !== 'positional-marker'); + const positional = args.filter((a) => !a.name); + const keywords = new Map(args.filter((a) => a.name).map((a) => [a.name!, a])); + let pi = 0; + const pending: Array<() => void> = []; + params.forEach((p, idx) => { + if (self && idx === 0) { + // self: mut/ref alias the caller's value, out starts uninitialized, otherwise it is read in place + if (p.convention === 'mut' || p.convention === 'ref' || p.convention === 'inout') { if (!self.loc) throw new Refuse(`calling a mut method on a temporary`, at); frame.aliases.set(p.name, { ...self.loc, name: p.name, frame: frame.id }); frame.cells.set(p.name, { v: undefined, type: self.struct.name }); } + else frame.cells.set(p.name, { v: p.convention === 'out' ? self.value : self.value, type: self.struct.name }); + return; + } + if (p.convention === 'out') throw new Refuse('an out parameter other than self', p.pos); + if (p.star === '**') throw new Refuse('keyword variadic parameters', p.pos); + if (p.star === '*') { + // *args: every remaining positional argument, as a list the function reads + if (prebound || p.convention) throw new Refuse('this variadic parameter', p.pos); + const rest = positional.slice(pi).map((a) => deepCopy(this.eval(a.value))); + pi = positional.length; + frame.cells.set(p.name, { v: { k: 'list', items: rest, elem: p.type ? typeText(p.type) : (rest.length ? typeName(rest[0]) : 'Int'), fixed: true }, type: `*${p.type ? typeText(p.type) : ''}` }); + return; + } + const arg = prebound ? undefined : (pi < positional.length ? positional[pi++] : keywords.get(p.name)); + if (prebound) { + const v = prebound[pi++]; + if (v === undefined) { if (!p.default) throw new Refuse(`a call missing ${p.name}`, at); frame.cells.set(p.name, { v: this.coerce(this.eval(p.default), p.type), type: p.type ? typeText(p.type) : '' }); return; } + frame.cells.set(p.name, { v: p.convention === 'var' || p.convention === 'owned' ? deepCopy(v) : v, type: p.type ? typeText(p.type) : typeName(v) }); + return; + } + if (!arg) { + if (!p.default) throw new Refuse(`a call missing ${p.name}`, at); + frame.cells.set(p.name, { v: this.coerce(this.eval(p.default), p.type), type: p.type ? typeText(p.type) : '' }); + return; + } + if (p.convention === 'mut' || p.convention === 'ref' || p.convention === 'inout') { + const loc = this.locate(arg.value); + frame.aliases.set(p.name, { ...loc }); + frame.cells.set(p.name, { v: undefined, type: p.type ? typeText(p.type) : '' }); + pending.push(() => { frame.aliases.set(p.name, { ...loc, name: p.name, frame: frame.id }); }); + return; + } + const owned = p.convention === 'var' || p.convention === 'owned'; + let v = owned ? this.take(arg.value, false) : this.eval(arg.value); + v = this.implicitly(v, p.type, at); + if (owned) this.claim(v); + frame.cells.set(p.name, { v: this.coerce(v, p.type), type: p.type ? typeText(p.type) : typeName(v), owned }); + }); + if (pi < positional.length && !prebound) throw new Refuse(`a call with more arguments than ${def.name} takes`, at); + for (const f of pending) f(); + // a mut parameter's cell shows the caller's value, through the alias + for (const [name, loc] of frame.aliases) { const c = frame.cells.get(name); if (c) Object.defineProperty(c, 'v', { get: () => loc.get(), set: (x) => loc.set(x), enumerable: true, configurable: true }); } + frame.def = def; + frame.life = this.lifeOf(def); + for (const [name, c] of frame.cells) { + if (!c.owned || !this.isTracked(c.v)) continue; + if (frame.life.unmodeled.has(name)) throw new Refuse(`when ${name} is destroyed: it is used where Programming+ does not follow the flow yet`, def.pos); + if (frame.life.unusedParams.has(name)) throw new Refuse(`an owned argument (${name}) that is never used: when it is destroyed is not modeled`, def.pos); + } + // Each Mojo call is a dozen JavaScript frames here, and a browser gives about ten thousand. A function that + // recurses without end (or to a huge n, as generated inputs try) must stop with the step limit's kind of + // answer, not with a JavaScript RangeError. + if (this.frames.length > MAX_CALL_DEPTH) throw new Limit(`Stopped at ${MAX_CALL_DEPTH} calls deep: ${def.name} kept calling itself (a compiled program would run out of stack somewhere past this).`); + this.frames.push(frame); + this.frameBase.set(frame, this.shadowed.length); + this.step(def.pos.line, `${frame.name}(${[...frame.cells].map(([n, c]) => `${n}=${display(c.v)}`).join(', ')})`, 'call'); + try { + this.block(def.body); + const left = [...frame.cells].filter(([n, c]) => c.owned && !c.destroyed && c.v && !frame.aliases.has(n) && this.isTracked(c.v)); + if (left.length) throw new Refuse(`${left[0][0]} still alive at the end of ${def.name} (a gap in Programming+'s lifetime analysis)`, def.pos); + return null; + } catch (e) { + // the value becomes the declared return type: an Int returned as Optional[Int] is an Optional, + // so `if r:` is true for a found 0 (stress/optional-lookup) + if (e instanceof ReturnSignal) return e.value && def.ret ? this.coerce(e.value, def.ret) ?? e.value : e.value; + throw e; + } finally { + this.frames.pop(); + } + } + + /** + * Of several defs, the one the arguments fit: by count (a variadic takes the rest), by keyword + * names, then by type. An argument's type is read from its form (a literal, a constructor call) + * or a variable's current value, never by evaluating it twice; one it cannot tell fits any + * parameter. Two that still fit: declined, rather than guess the compiler's ranking. + */ + private pick(defs: DefStmt[], args: Arg[], owner: StructStmt | null, at: Pos, what: string): DefStmt { + const kindOf = (e: Expr): string | null => { + if (e.k === 'num') return e.float ? 'float' : 'int'; + if (e.k === 'str') return 'String'; + if (e.k === 'name' && (e.id === 'True' || e.id === 'False')) return 'Bool'; + if (e.k === 'call' && e.fn.k === 'name' && this.structs.has(e.fn.id)) return e.fn.id; + if (e.k === 'name') { const v = this.lookup(e.id, e.pos)?.get(); if (!v) return null; return v.k === 'int' ? 'int' : v.k === 'float' ? 'float' : v.k === 'str' ? 'String' : v.k === 'bool' ? 'Bool' : v.k === 'struct' ? v.name : v.k === 'list' ? 'List' : null; } + return null; + }; + const fitsType = (kind: string | null, type: Expr | null): 0 | 1 | 2 => { + if (!kind || !type) return 1; + let t = typeText(type); + if (t === 'Self' && owner) t = owner.name; + if (INT_TYPES[t]) return kind === 'int' ? 2 : 0; + if (t === 'Float64' || t === 'Float32') return kind === 'float' ? 2 : kind === 'int' ? 1 : 0; + if (t === 'String' || t === 'StringSlice' || t === 'StringLiteral') return kind === 'String' ? 2 : 0; + if (t === 'Bool') return kind === 'Bool' ? 2 : 0; + if (/^List\[/.test(t)) return kind === 'List' ? 2 : 0; + if (this.structs.has(t)) return kind === t ? 2 : kind === 'int' || kind === 'float' || kind === 'String' || kind === 'Bool' ? 0 : 1; + return 1; + }; + const scored: Array<[DefStmt, number]> = []; + for (const d of defs) { + const ps = d.params.filter((p) => !(p.convention === 'out' || p.name === 'self') && p.star !== 'marker' && p.star !== 'positional-marker'); + const variadic = ps.find((p) => p.star === '*'); + const fixed = ps.filter((p) => !p.star); + const positional = args.filter((a) => !a.name); + if (!variadic && positional.length > fixed.length) continue; + if (args.some((a) => a.name && !fixed.some((p) => p.name === a.name))) continue; + const given = new Set([...fixed.slice(0, positional.length).map((p) => p.name), ...args.filter((a) => a.name).map((a) => a.name!)]); + if (fixed.some((p) => !p.default && !given.has(p.name))) continue; + let score = 0, ok = true; + positional.forEach((a, i) => { const p = i < fixed.length ? fixed[i] : variadic!; const f = fitsType(kindOf(a.value), p.type); if (!f) ok = false; score += f; }); + for (const a of args.filter((x) => x.name)) { const f = fitsType(kindOf(a.value), fixed.find((p) => p.name === a.name)!.type); if (!f) ok = false; score += f; } + if (ok) scored.push([d, score]); + } + scored.sort((a, b) => b[1] - a[1]); + if (!scored.length || (scored.length > 1 && scored[0][1] === scored[1][1])) throw new Refuse(`choosing among ${what}`, at); + return scored[0][0]; + } + + /** How deep in nested blocks of the current function execution is (0: its body). */ + private blockDepth(): number { return this.shadowed.length - (this.frameBase.get(this.frame) ?? 0) - 1; } + private frameBase = new WeakMap<Frame, number>(); + + private lifeCache = new WeakMap<DefStmt, Lifetimes>(); + private lifeOf(def: DefStmt): Lifetimes { + let l = this.lifeCache.get(def); + if (!l) { l = lifetimes(def); this.lifeCache.set(def, l); } + return l; + } + + private construct(st: StructStmt, args: Arg[], at: Pos): V { + if (st.k !== 'struct') throw new Refuse('constructing a trait', at); + const fields = new Map<string, V | undefined>(); + for (const b of st.body) if (b.k === 'var' && !b.comptime && b.targets.k === 'name') fields.set(b.targets.id, b.value ? this.eval(b.value) : undefined); + const value: V = { k: 'struct', name: st.name, fields }; + const inits = st.body.filter((b): b is DefStmt => b.k === 'def' && b.name === '__init__' && !b.params.some((p) => p.name === 'copy' || p.name === 'move')); + const fieldwise = st.decorators.some((d) => d.k === 'name' && d.id === 'fieldwise_init'); + if (!inits.length && fieldwise) { + const names = [...fields.keys()]; + if (args.length !== names.length || args.some((a) => a.name && !names.includes(a.name))) throw new Refuse(`this call to ${st.name}`, at); + args.forEach((a, i) => { + let v = this.take(a.value, false); + if (this.isTracked(v)) { this.claim(v); v = this.moveInto(v, at); } // @fieldwise_init takes var arguments and moves them into the fields + const ft = this.fieldType(st.name, a.name ?? names[i]); + fields.set(a.name ?? names[i], ft ? this.coerceTo(v, ft) : v); + }); + return value; + } + if (!inits.length) throw new Refuse(`constructing ${st.name}`, at); + const init = inits.length === 1 ? inits[0] : this.pick(inits, args, st, at, `${st.name}'s constructors`); + const cell: Cell = { v: value, type: st.name }; + const loc: Loc = { get: () => cell.v, set: (v) => { cell.v = v; }, name: 'self', frame: this.nextFrameId }; + this.callDef(init, args, { loc, value, struct: st }, at); + return cell.v!; + } + + private memberCache = new Map<string, V>(); + /** A struct's comptime member, `FileError.not_found`: evaluated once. */ + private member(st: StructStmt, name: string, pos: Pos): V { + const key = `${st.name}.${name}`; + if (this.memberCache.has(key)) return this.memberCache.get(key)!; + const decl = st.body.find((b): b is Extract<Stmt, { k: 'var' }> => b.k === 'var' && b.comptime && b.targets.k === 'name' && b.targets.id === name); + if (!decl?.value || decl.typeParams?.length) throw new Refuse(`${st.name}.${name}`, pos); + const v = this.coerce(this.eval(decl.value), decl.type)!; + this.memberCache.set(key, v); + return v; + } + + private methodDef(st: StructStmt, name: string): DefStmt | null { + const own = st.body.find((b): b is DefStmt => b.k === 'def' && b.name === name); + if (own) return own; + for (const base of st.bases) { + if (base.k !== 'name') continue; + const tr = this.traits.get(base.id); + const d = tr?.body.find((b): b is DefStmt => b.k === 'def' && b.name === name && b.body.some((x) => !(x.k === 'expr' && (x.e.k === 'ellipsis' || x.e.k === 'str')))); + if (d) return d; + } + return null; + } + + private methodCall(obj: Extract<V, { k: 'struct' }>, name: string, args: Arg[], at: Pos, loc: Loc | null): V { + const st = this.structs.get(obj.name)!; + if (name === 'copy' && !this.methodDef(st, 'copy')) return this.isTracked(obj) ? this.copyOf(obj, at) : deepCopy(obj); + const def = this.methodDef(st, name); + if (!def) throw new Refuse(`the method ${obj.name}.${name}`, at); + if (def.decorators.some((d) => d.k === 'name' && d.id === 'staticmethod')) return this.callDef(def, args, null, at) ?? NONE; + return this.callDef(def, args, { loc, value: obj, struct: st }, at) ?? NONE; + } + + /** A method called with values already evaluated (operators, print's write_to). */ + private callMethod(obj: Extract<V, { k: 'struct' }>, name: string, args: V[], at: Pos, loc: Loc | null): V | null { + const st = this.structs.get(obj.name); + if (!st) throw new Refuse(`a ${obj.name} value`, at); + const def = this.methodDef(st, name); + if (!def) return null; + return this.callDef(def, [], { loc, value: obj, struct: st }, at, args) ?? NONE; + } + + private callDunder(obj: Extract<V, { k: 'struct' }>, name: string, args: V[], at: Pos, loc: Loc | null): V | undefined { + const r = this.callMethod(obj, name, args, at, loc); + return r === null ? undefined : r; + } + + /** print() of a struct: its write_to, with a writer that collects what it writes. */ + private writeTo(s: V): string { + if (s.k !== 'struct') return show(s); + const st = this.structs.get(s.name)!; + const def = this.methodDef(st, 'write_to'); + if (!def) { + // Writable's default write_to, by reflection: Number(value=-3) (probes/life-asap-expression). Only numeric + // and Bool fields are verified; how a String field is written is not, so that is declined + if (!st.bases.some((b) => b.k === 'name' && b.id === 'Writable')) throw new Refuse(`printing ${s.name}, which has no write_to`, null); + if (st.typeParams.length) throw new Refuse(`printing the parametric ${s.name} through Writable's default write_to (it shows its parameters)`, null); + const parts: string[] = []; + for (const [n, f] of s.fields) { + if (!f || !(f.k === 'int' || f.k === 'float' || f.k === 'bool')) throw new Refuse(`printing ${s.name} through Writable's default write_to with a field of type ${typeName(f)}`, null); + parts.push(`${n}=${show(f)}`); + } + return `${s.name}(${parts.join(', ')})`; + } + const w: V = { k: 'writer', out: [] }; + const cell: Cell = { v: w, type: 'Writer' }; + const wloc: Loc = { get: () => cell.v, set: (v) => { cell.v = v; }, name: 'writer', frame: -1 }; + const frameArgs: V[] = [w]; + void wloc; + this.callDef(def, [], { loc: null, value: s, struct: st }, def.pos, frameArgs); + return (w as Extract<V, { k: 'writer' }>).out.join(''); + } + + /** A value passed where the student's struct is wanted, through that struct's @implicit constructor. */ + private implicitly(v: V, type: Expr | null, at: Pos): V { + if (!type || type.k !== 'name') return v; + const st = this.structs.get(type.id); + if (!st || (v.k === 'struct' && v.name === st.name)) return v; + const init = st.body.find((b): b is DefStmt => b.k === 'def' && b.name === '__init__' && b.decorators.some((d) => d.k === 'name' && d.id === 'implicit')); + if (!init) throw new Refuse(`passing ${typeName(v)} where ${st.name} is wanted`, at); + const fields = new Map<string, V | undefined>(); + for (const b of st.body) if (b.k === 'var' && !b.comptime && b.targets.k === 'name') fields.set(b.targets.id, undefined); + const value: V = { k: 'struct', name: st.name, fields }; + const cell: Cell = { v: value, type: st.name }; + this.callDef(init, [], { loc: { get: () => cell.v, set: (x) => { cell.v = x; }, name: 'self', frame: -1 }, value, struct: st }, at, [v]); + return cell.v!; + } + + private coerce(v: V | undefined, type: Expr | null): V | undefined { + if (!v || !type) return v; + return this.coerceTo(v, typeText(type)); + } + + /** A value landing in a place of the type written `t`. */ + private coerceTo(v: V, t: string): V { + // the same value back when it already has the type, so a value is never rebuilt for nothing + if (INT_TYPES[t] && v.k === 'int') return v.type === t ? v : int(v.v, t); + if ((t === 'Float64' || t === 'Float32') && (v.k === 'int' || v.k === 'float')) { const bits = t === 'Float32' ? 32 : 64; return v.k === 'float' && v.bits === bits ? v : float(Number(v.v), bits); } + const m = /^List\[(.+)\]$/.exec(t); + // a list in brackets takes the place's type, and each of its items the element type: [1, 2.5] landing in a + // List[Float64] prints [1.0, 2.5] (probes/list-literal-typed). A List that already is one is left as it is. + if (m && v.k === 'list') return !v.fixed && v.elem === m[1] ? v : { ...v, fixed: false, elem: m[1], items: v.fixed ? v.items.map((x) => this.coerceTo(x, m[1])) : v.items }; + const d = /^Dict\[(.+), (.+)\]$/.exec(t); + if (d && v.k === 'dict') { + // a Dict in braces: its keys and values take the place's types too ({"x": [1, 2]} landing in a + // Dict[String, List[Float64]]: probes/list-literal-into-place), in the order they were written + this.work(v.map.size, null); + const was = [...v.map.values()]; + const entries = was.map(([k, x]): [V, V] => [this.coerceTo(k, d[1]), this.coerceTo(x, d[2])]); + const same = entries.every(([k, x], i) => k === was[i][0] && x === was[i][1]); + if (same && v.key === d[1] && v.val === d[2]) return v; + return { ...v, key: d[1], val: d[2], map: same ? v.map : new Map(entries.map(([k, x]) => [keyOf(k), [k, x]])) }; + } + const o = /^Optional\[(.+)\]$/.exec(t); + if (o) return v.k === 'none' ? { k: 'opt', v: null, elem: o[1] } : v.k === 'opt' ? v : { k: 'opt', v, elem: o[1] }; + return v; + } + + /* ---------------- the standard library's core ---------------- */ + + private args(args: Arg[], at: Pos, name: string): V[] { + return args.map((a) => { if (a.name) throw new Refuse(`the keyword argument ${a.name}= to ${name}`, at); return this.eval(a.value); }); + } + + private builtin(name: string, args: Arg[], typeArgs: string[], e: Extract<Expr, { k: 'call' }>): V { + const at = e.lparen; + switch (name) { + case 'print': { + let sep = ' ', end = '\n'; + const vals: string[] = []; + for (const a of args) { + if (a.name === 'sep') { sep = show(this.eval(a.value)); continue; } + if (a.name === 'end') { end = show(this.eval(a.value)); continue; } + if (a.name === 'flush') continue; + if (a.name) throw new Refuse(`print(${a.name}=)`, at); + vals.push(show(this.eval(a.value), (s) => this.writeTo(s))); + } + this.write(vals.join(sep) + end); + return NONE; + } + case 'len': { + const [v] = this.args(args, at, name); + if (v.k === 'list' || v.k === 'tuple') return int(BigInt(v.items.length)); + if (v.k === 'dict' || v.k === 'set') return int(BigInt(v.map.size)); + if (v.k === 'struct') return this.callMethod(v, '__len__', [], at, null) ?? (() => { throw new Refuse(`len of ${v.name}`, at); })(); + throw new Refuse(`len of ${typeName(v)}`, at); + } + case 'range': { + const vs = this.args(args, at, name).map((v) => this.intOf(v, at)); + const [start, stop, step] = vs.length === 1 ? [0n, vs[0], 1n] : vs.length === 2 ? [vs[0], vs[1], 1n] : [vs[0], vs[1], vs[2]]; + return { k: 'range', start, stop, step }; + } + case 'input': { + if (!this.input.length) throw new Raised({ k: 'error', msg: 'EOF' }, at); + return str(this.input.shift()!); + } + // a decimal number in text, read to the nearest Float64 (probes/string-atof-decimal); the other texts atof reads + // (spaces around it, inf, nan) and how it reports the ones it refuses are not modeled + case 'atof': { + const [v] = this.args(args, at, name); + if (args.length !== 1 || v?.k !== 'str') break; + if (!/^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/.test(v.v)) throw new Refuse(`atof of text that is not a plain decimal number (${JSON.stringify(v.v.slice(0, 40))})`, at); + this.work(v.v.length, at); + return float(Number(v.v)); + } + case 'atol': { + const [v] = this.args(args, at, name); + const s = show(v).trim(); + if (!/^[+-]?\d+$/.test(s.replace(/_/g, ''))) throw new Raised({ k: 'error', msg: `String is not convertible to integer with base 10: '${show(v)}'` }, at); + return int(BigInt(s.replace(/_/g, ''))); + } + case 'abs': { const [v] = this.args(args, at, name); if (v.k === 'int') return int(v.v < 0n ? -v.v : v.v, v.type); if (v.k === 'float') return float(Math.abs(v.v), v.bits); break; } + case 'min': case 'max': { + const vs = this.args(args, at, name); + const list = vs.length === 1 && vs[0].k === 'list' ? vs[0].items : vs; + this.work(list.length, at); + if (!list.length) throw new Refuse(`${name} of nothing`, at); + return list.reduce((a, b) => (this.compare(name === 'min' ? '<' : '>', b, a, at) ? b : a)); + } + case 'round': { const vs = this.args(args, at, name); if (vs.length === 1 && vs[0].k === 'float') return float(roundHalfEven(vs[0].v), vs[0].bits); if (vs.length === 1 && vs[0].k === 'int') return vs[0]; break; } + case 'sort': { + if (!args.length) break; + const loc = this.locate(args[0].value); + const v = loc.get(); + if (v?.k !== 'list') break; + this.work(v.items.length * Math.ceil(Math.log2(v.items.length + 1)), at); + v.items.sort((a, b) => (this.compare('<', a, b, at) ? -1 : this.compare('<', b, a, at) ? 1 : 0)); + return NONE; + } + case 'String': { + const vs = this.args(args, at, name); + if (vs.length === 0) return str(''); + if (vs.length === 1) { const t = show(vs[0], (s) => this.writeTo(s)); this.grow(t.length, t.length, at); return str(t); } + break; + } + case 'Bool': { const vs = this.args(args, at, name); return bool(vs.length ? truthy(vs[0]) : false); } + case 'Float64': case 'Float32': { + const vs = this.args(args, at, name); + const bits = name === 'Float32' ? 32 : 64; + if (!vs.length) return float(0, bits); + const v = vs[0]; + if (v.k === 'int' || v.k === 'float') return float(Number(v.v), bits); + break; + } + case 'Error': { const vs = this.args(args, at, name); return { k: 'error', msg: vs.length ? show(vs[0]) : '' }; } + case 'List': { + const piece = args.length === 1 && !args[0].name ? this.listSlice(args[0].value, typeArgs[0], at) : null; + if (piece) return piece; + const vs = this.args(args, at, name); + if (!typeArgs.length && !vs.length) throw new Refuse('List() with no element type', at); + this.made(vs.length, at); + return { k: 'list', items: vs.map(deepCopy), elem: typeArgs[0] ?? typeName(vs[0]), fixed: false, cap: vs.length }; + } + case 'Dict': if (!args.length) return { k: 'dict', map: new Map(), key: typeArgs[0] ?? 'String', val: typeArgs[1] ?? 'Int' }; break; + case 'Set': { + const vs = this.args(args, at, name); + const map = new Map<string, V>(); for (const v of vs) map.set(keyOf(v), v); + return { k: 'set', map, elem: typeArgs[0] ?? (vs.length ? typeName(vs[0]) : 'Int') }; + } + case 'Optional': { const vs = this.args(args, at, name); return { k: 'opt', v: vs.length && vs[0].k !== 'none' ? vs[0] : null, elem: typeArgs[0] ?? (vs.length ? typeName(vs[0]) : 'Int') }; } + case 'assert_equal': case 'assert_not_equal': case 'assert_true': case 'assert_false': { + if (!this.imported.has(name)) break; + const vs = args.filter((a) => !a.name).map((a) => this.eval(a.value)); + const ok = name === 'assert_true' ? truthy(vs[0]) : name === 'assert_false' ? !truthy(vs[0]) : name === 'assert_equal' ? this.equals(vs[0], vs[1], at) : !this.equals(vs[0], vs[1], at); + if (ok) return NONE; // a passing assert prints nothing + // a failing assert_equal raises, in the compiler's words (quest/MQ-950/starter); the others are not modeled + const plainValue = (v: V) => v.k === 'int' || v.k === 'float' || v.k === 'bool'; + if (name === 'assert_equal' && plainValue(vs[0]) && plainValue(vs[1]) && !args.some((a) => a.name)) + throw new Raised({ k: 'error', msg: `At /app/example.mojo:${at.line}:${at.col}: AssertionError: \`left == right\` comparison failed:\n left: ${show(vs[0])}\n right: ${show(vs[1])}` }, at); + throw new Refuse(`a failing ${name} (how it reports itself is not modeled)`, at); + } + case 'assert_almost_equal': { + if (!this.imported.has(name)) break; + const vs = args.filter((a) => !a.name).map((a) => this.eval(a.value)); + const a = vs[0], b = vs[1]; + if (!a || !b || (a.k !== 'float' && a.k !== 'int') || (b.k !== 'float' && b.k !== 'int') || args.some((x) => x.name)) break; + const x = Number(a.v), y = Number(b.v); + if (Math.abs(x - y) <= 1e-8 + 1e-5 * Math.abs(y)) return NONE; // passing: nothing printed + throw new Refuse('a failing assert_almost_equal (how it reports itself is not modeled)', at); + } + case 'sqrt': case 'floor': case 'ceil': case 'trunc': { + if (!this.imported.has(name)) break; + const [v] = this.args(args, at, name); + if (v?.k !== 'float') break; + const f = ({ sqrt: Math.sqrt, floor: Math.floor, ceil: Math.ceil, trunc: Math.trunc } as Record<string, (x: number) => number>)[name]; + return float(f(v.v), v.bits); + } + // a String of one character and its code point (probes/string-ord-chr); anything else is not modeled + case 'ord': { + const [v] = this.args(args, at, name); + if (args.length !== 1 || v?.k !== 'str') break; + const cps = codepoints(v.v); + if (cps.length !== 1) throw new Refuse(`ord of a String of ${cps.length} characters`, at); + return int(BigInt(cps[0].codePointAt(0)!)); + } + case 'chr': { + const [v] = this.args(args, at, name); + if (args.length !== 1 || v?.k !== 'int') break; + const c = Number(v.v); + if (c < 0 || c > 0x10ffff || (c >= 0xd800 && c <= 0xdfff)) throw new Refuse(`chr of ${c}, which is not a code point`, at); + return str(String.fromCodePoint(c)); + } + // std.math's gcd, lcm and comb, step for step as Mojo 1.2.0 computes them, wrapping at 64 bits where its Int + // does (probes/math-gcd-lcm-comb) + case 'gcd': case 'lcm': case 'comb': { + if (!this.imported.has(name)) break; + const vs = this.args(args, at, name); + if (vs.length !== 2 || vs.some((v) => v.k !== 'int' || v.type !== 'Int')) break; + const [m, n] = vs.map((v) => (v as Extract<V, { k: 'int' }>).v); + if (name === 'gcd') return int(this.gcd(m, n, at)); + if (name === 'lcm') { const d = this.gcd(m, n, at); if (d === 0n) return int(0n); const p = BigInt.asIntN(64, m > n ? floordiv(m, d) * n : floordiv(n, d) * m); return int(p < 0n ? -p : p); } + // comb asserts that both are not negative; assert statements are not compiled in (probes/error-assert-statement) + if (m < 0n || n < 0n) throw new Refuse('comb of a negative number', at); + if (n > m) return int(0n); + const k2 = n <= m - n ? n : m - n; + if (k2 > BigInt(MATH_STEPS)) throw new Limit(`Stopped: comb(${m}, ${n}) takes ${count(Number(k2))} steps in Mojo's std.math, more than Programming+ follows in one call (${count(MATH_STEPS)}).`, at); + this.work(Number(k2), at); + let result = 1n; + for (let i = 0n; i < k2; i++) { + let num = m - i, den = i + 1n; + const g = this.gcd(num, den, at); + num = floordiv(num, g); den = floordiv(den, g); + result = BigInt.asIntN(64, floordiv(result, den) * num); + } + return int(result); + } + // std.math's copysign, nextafter, ulp, fma and hypot of Float64 values: each is exact, or rounds once, or (hypot) + // is Mojo's own sequence of exact steps, so each is computed here as Mojo does (probes/math-copysign-nextafter-ulp, + // math-fma-hypot) + case 'copysign': case 'nextafter': case 'ulp': case 'fma': case 'hypot': { + if (!this.imported.has(name)) break; + const vs = this.args(args, at, name); + const arity = name === 'ulp' ? 1 : name === 'fma' ? 3 : 2; + if (vs.length !== arity || vs.some((v) => v.k !== 'float' || v.bits !== 64)) break; + const [x, y, z] = vs.map((v) => (v as Extract<V, { k: 'float' }>).v); + if (name === 'copysign') { if (Number.isNaN(x) || Number.isNaN(y)) throw new Refuse('copysign of a NaN (which sign a NaN carries is not modeled)', at); return float((Object.is(y, -0) || y < 0) ? -Math.abs(x) : Math.abs(x)); } + if (name === 'nextafter') return float(nextAfter(x, y)); + if (name === 'ulp') { if (Number.isNaN(x)) return float(x); const ax = Math.abs(x); if (ax === Infinity) return float(ax); const up = nextAfter(ax, Infinity); return float(up === Infinity ? ax - nextAfter(ax, -Infinity) : up - ax); } + if (name === 'fma') return float(fma(x, y, z)); + return float(hypot(x, y)); + } + // std.math's isinf, isnan and isfinite of a Float64 or Float32 (probes/math-float-classify) + case 'isinf': case 'isnan': case 'isfinite': { + if (!this.imported.has(name)) break; + const vs = this.args(args, at, name); + if (vs.length !== 1 || vs[0].k !== 'float') break; + const x = vs[0].v; + return bool(name === 'isinf' ? x === Infinity || x === -Infinity : name === 'isnan' ? Number.isNaN(x) : Number.isFinite(x)); + } + case 'exit': { + if (!this.imported.has('exit')) break; + const vs = this.args(args, at, name); + throw new Halt(vs.length && vs[0].k === 'int' ? Number(vs[0].v) : 0); + } + default: { + if (INT_TYPES[name]) { + const vs = this.args(args, at, name); + if (!vs.length) return int(0n, name); + const v = vs[0]; + if (v.k === 'int') return int(v.v, name); + if (v.k === 'float' && !Number.isFinite(v.v)) throw new Refuse(`converting ${show(v)} to ${name}`, at); + if (v.k === 'float') return int(BigInt(Math.trunc(v.v)), name); // float-to-int: truncates toward zero + if (v.k === 'bool') return int(v.v ? 1n : 0n, name); + } + } + } + // std.math's transcendental functions: Mojo computes each with a routine of its own or the C library's, good to the + // last digit or a few short of it (exp and ** to about 13 digits: probes/float-pow-accuracy, float-cosh-accuracy), + // and a browser's Math computes them with others, so the last digits would be a guess + if (/^random_/.test(name) && this.imported.has(name)) throw new Refuse(`${name}(...), whose numbers come from Mojo's own generator, which Programming+ cannot know`, at); + if (MATH_OWN.has(name) && this.imported.has(name)) throw new Refuse(`${name}(...), which Mojo computes with a routine of its own that Programming+ does not reproduce digit for digit`, at); + throw new Refuse(`${name}(...) as written here`, at); + } + + /** + * std.math's gcd as Mojo 1.2.0 writes it: a binary gcd that shifts the second number by its trailing zeros and + * subtracts. When the first is even and the second odd it subtracts one from the other many times over (gcd(6, + * 4503599627370597) takes about 10^15 passes; the built program does not come back), so past MATH_STEPS passes the + * run stops as at its step limit. The result is the greatest common divisor, never negative; |Int.MIN| does not + * fit an Int, and what Mojo makes of it is not modeled. + */ + private gcd(m: bigint, n: bigint, at: Pos): bigint { + const MIN = -(2n ** 63n); + if (m === MIN || n === MIN) throw new Refuse('gcd of the smallest Int', at); + const tz = (x: bigint): bigint => { let k = 0n; while ((x & 1n) === 0n) { x >>= 1n; k++; } return k; }; + let u = m < 0n ? -m : m, v = n < 0n ? -n : n; + if (u === 0n) return v; + if (v === 0n) return u; + let vz = tz(v); + const shift = tz(u) < vz ? tz(u) : vz; + u >>= shift; + let passes = 0; + for (; ; passes++) { + if (passes > MATH_STEPS) throw new Limit(`Stopped: gcd(${m}, ${n}) takes more than ${count(MATH_STEPS)} steps in Mojo's std.math (it subtracts when one number is even and the other odd), more than Programming+ follows in one call.`, at); + v >>= vz; + const diff = v - u; + if (diff === 0n) break; + [u, v] = [u < v ? u : v, diff < 0n ? -diff : diff]; + vz = tz(diff < 0n ? -diff : diff); + } + this.work(passes + 1, at); + return u << shift; + } + + /** + * `List[T](l[lo:hi])`: l[lo:hi] is a Span that borrows l, and the List made from it holds copies of its items, so + * neither changes with the other afterwards (probes/list-slice-copy). A start or end past the List's ends, a negative + * one, or a start past the end aborts the program at the bracket (probes/list-slice-end-oob, list-slice-negative, + * list-slice-reversed). A Span anywhere else, and a slice with a step (a new List with Python's clamped bounds), are + * not modeled; null for anything that is not a slice of a List, which the caller reads as an ordinary argument. + */ + private listSlice(arg: Expr, elem: string | undefined, at: Pos): V | null { + if (arg.k !== 'index' || arg.args.length !== 1 || arg.args[0].name || arg.args[0].value.k !== 'slice') return null; + const sl = arg.args[0].value; + const obj = this.eval(arg.obj); + if (obj.k !== 'list' || obj.fixed) throw new Refuse(`a slice of ${typeName(obj)}`, arg.pos); + if (sl.step) throw new Refuse('a List slice with a step', arg.pos); + if (!elem) throw new Refuse('List() of a slice with no element type', at); + const n = obj.items.length; + const lo = sl.lo ? Number(this.intOf(this.eval(sl.lo), arg.pos)) : 0; + const hi = sl.hi ? Number(this.intOf(this.eval(sl.hi), arg.pos)) : n; + const where = `At: /app/example.mojo:${arg.lbracket.line}:${arg.lbracket.col}: Assert Error:`; + if (lo < 0 || lo > n) throw new OutOfBounds(`${where} slice start index ${lo} is out of bounds, valid range is 0 to ${n}`, arg.lbracket); + if (hi < 0 || hi > n) throw new OutOfBounds(`${where} slice end index ${hi} is out of bounds, valid range is 0 to ${n}`, arg.lbracket); + if (lo > hi) throw new OutOfBounds(`${where} slice start index ${lo} is greater than slice end index ${hi}`, arg.lbracket); + this.room(hi - lo, 'List', at); this.made(hi - lo, at); + const items = obj.items.slice(lo, hi).map((x) => (this.isTracked(x) ? this.copyOf(x, at) : deepCopy(x))); + return { k: 'list', items, elem, fixed: false, cap: items.length }; + } + + private method(obj: V, name: string, args: V[], e: Extract<Expr, { k: 'call' }>, loc: Loc | null): V { + const at = e.lparen; + if (obj.k === 'list') { + switch (name) { + case 'append': { + if (obj.fixed) throw new Refuse('append on a fixed-size Array (annotate it as a List)', at); + let v = args[0]; + if (this.isTracked(v)) { + this.claim(v); + // a full buffer doubles, moving every element across (life-list-elements) + const cap = obj.cap ?? obj.items.length; + if (obj.items.length >= cap) { obj.items = obj.items.map((x) => this.moveInto(x, at)); obj.cap = Math.max(1, cap * 2); } + v = this.moveInto(v, at); + } else v = deepCopy(v); + // an item takes the list's element type: 4 appended to a List[Float64] is 4.0 + if (obj.elem) v = this.coerceTo(v, obj.elem); + this.room(obj.items.length + 1, 'List', at); this.made(1, at); + obj.items.push(v); + if (obj.cap !== undefined && obj.items.length > obj.cap) obj.cap = obj.items.length; + this.noteWrite(loc); + return NONE; + } + case 'pop': { + if (obj.fixed) break; + if (!obj.items.length) throw new Refuse('pop from an empty List', at); + const i = args.length ? Number(this.intOf(args[0], at)) : obj.items.length - 1; + this.work(obj.items.length - (i < 0 ? obj.items.length + i : i), at); + const v = obj.items.splice(i < 0 ? obj.items.length + i : i, 1)[0]; + this.noteWrite(loc); return v; + } + case 'insert': if (obj.fixed) break; this.room(obj.items.length + 1, 'List', at); this.made(1, at); this.work(obj.items.length, at); obj.items.splice(Number(this.intOf(args[0], at)), 0, obj.elem ? this.coerceTo(deepCopy(args[1]), obj.elem) : deepCopy(args[1])); this.noteWrite(loc); return NONE; + case 'extend': { + if (obj.fixed || args[0]?.k !== 'list') break; + const add = args[0].items; + this.room(obj.items.length + add.length, 'List', at); this.made(add.length, at); + for (const x of add.slice()) obj.items.push(deepCopy(x)); // not push(...add): a long list is past the engine's argument limit + this.noteWrite(loc); return NONE; + } + // capacity is not something the stepper shows: asking for room changes no value + case 'reserve': if (obj.fixed || args.length !== 1 || args[0]?.k !== 'int') break; return NONE; + case 'clear': if (obj.fixed) break; obj.items.length = 0; this.noteWrite(loc); return NONE; + case 'copy': return this.isTracked(obj) ? this.copyOf(obj, at) : deepCopy(obj); + case 'reverse': obj.items.reverse(); this.noteWrite(loc); return NONE; + case '__len__': return int(BigInt(obj.items.length)); + } + } + if (obj.k === 'dict') { + switch (name) { + case 'keys': case 'values': case 'items': this.made(obj.map.size, at); break; + } + switch (name) { + case 'keys': return { k: 'list', items: [...obj.map.values()].map(([k]) => k), elem: obj.key, fixed: false }; + case 'values': return { k: 'list', items: [...obj.map.values()].map(([, v]) => v), elem: obj.val, fixed: false }; + case 'items': return { k: 'list', items: [...obj.map.values()].map(([k, v]) => ({ k: 'struct', name: 'DictEntry', fields: new Map([['key', k], ['value', v]]) } as V)), elem: 'DictEntry', fixed: false }; + case 'get': { const x = obj.map.get(keyOf(args[0])); return args.length > 1 ? (x ? x[1] : args[1]) : { k: 'opt', v: x ? x[1] : null, elem: obj.val }; } + case 'pop': { + const key = keyOf(args[0]); const x = obj.map.get(key); + if (!x) { if (args.length > 1) return args[1]; throw new Raised({ k: 'error', msg: '', kind: `DictKeyError[${obj.key}]` }, at); } + obj.map.delete(key); this.noteWrite(loc); return x[1]; + } + case 'copy': return deepCopy(obj); + case 'clear': obj.map.clear(); this.noteWrite(loc); return NONE; + } + } + if (obj.k === 'set') { + switch (name) { + case 'add': { + this.keyable(args[0], at); + const key = keyOf(args[0]); + if (!obj.map.has(key)) { this.room(obj.map.size + 1, 'Set', at); this.made(1, at); } + this.work(key.length, at); + obj.map.set(key, args[0]); this.noteWrite(loc); return NONE; + } + case 'remove': if (!obj.map.has(keyOf(args[0]))) throw new Refuse('remove of a missing element', at); obj.map.delete(keyOf(args[0])); this.noteWrite(loc); return NONE; + case 'discard': obj.map.delete(keyOf(args[0])); this.noteWrite(loc); return NONE; + case 'copy': return deepCopy(obj); + } + } + if (obj.k === 'opt') { + if (name === 'value') { if (!obj.v) throw new Refuse('value() of an empty Optional (it aborts)', at); return obj.v; } + if (name === 'or_else') return obj.v ?? args[0]; + } + if (obj.k === 'str') { + const s = obj.v; + this.work(s.length, at); // each of these reads the whole String + switch (name) { + case 'upper': return str(s.toUpperCase()); + case 'lower': return str(s.toLowerCase()); + case 'strip': return str(s.trim()); + case 'lstrip': return str(s.trimStart()); + case 'rstrip': return str(s.trimEnd()); + case 'startswith': return bool(s.startsWith(show(args[0]))); + case 'endswith': return bool(s.endsWith(show(args[0]))); + case 'find': return int(BigInt(utf8(s.slice(0, Math.max(0, s.indexOf(show(args[0]))))).length * (s.includes(show(args[0])) ? 1 : 0) + (s.includes(show(args[0])) ? 0 : -1))); + case 'replace': { + const parts = s.split(show(args[0])), by = show(args[1]); + const length = s.length + (parts.length - 1) * (by.length - show(args[0]).length); + this.grow(length, length, at); + return str(parts.join(by)); + } + case 'byte_length': return int(BigInt(utf8(s).length)); + case 'count_codepoints': return int(BigInt(codepoints(s).length)); + case 'codepoint_slices': { const cps = codepoints(s); this.room(cps.length, 'List', at); this.made(cps.length, at); return { k: 'list', items: cps.map(str), elem: 'StringSlice', fixed: false }; } + case 'split': { + const parts = args.length ? s.split(show(args[0])) : s.split(/\s+/).filter(Boolean); + this.room(parts.length, 'List', at); this.made(parts.length, at); + return { k: 'list', items: parts.map(str), elem: 'String', fixed: false }; + } + case 'join': { + if (args[0]?.k !== 'list') break; + const parts = args[0].items.map((x) => show(x)); + const length = parts.reduce((n, p) => n + p.length, 0) + Math.max(0, parts.length - 1) * s.length; + this.grow(length, length, at); + return str(parts.join(s)); + } + } + } + if (obj.k === 'simd') { + const lane = this.lane(obj.dtype, at); + const fold = (f: (a: V, b: V) => V) => obj.items.slice(1).reduce(f, obj.items[0]); + switch (name) { + case 'reduce_add': return lane(fold((a, b) => this.binary('+', a, b, at))); + case 'reduce_mul': return lane(fold((a, b) => this.binary('*', a, b, at))); + case 'reduce_max': return fold((a, b) => (this.compare('>', b, a, at) ? b : a)); + case 'reduce_min': return fold((a, b) => (this.compare('<', b, a, at) ? b : a)); + } + } + if (obj.k === 'error' && name === '__str__') return str(obj.msg); + if (obj.k === 'writer' && name === 'write') { obj.out.push(...args.map((a) => show(a, (x) => this.writeTo(x)))); return NONE; } + if (obj.k === 'struct' && obj.name === 'DictEntry') throw new Refuse(`the method ${name} of a dict entry`, at); + throw new Refuse(`${typeName(obj)}.${name}(...)`, at); + } + + /** A struct as a Set or Dict key: only one whose equality and hash are the traits' defaults (by its fields). */ + private keyable(v: V, at: Pos): void { + if (v.k !== 'struct') return; + const st = this.structs.get(v.name); + if (!st || this.methodDef(st, '__eq__') || this.methodDef(st, '__hash__')) throw new Refuse(`${v.name} as a key (its own __eq__ or __hash__ decides equality)`, at); + } + + private noteWrite(loc: Loc | null): void { if (loc) this.writes.push({ name: loc.name, frame: loc.frame, path: loc.path }); } +} + +class OutOfBounds extends Error { constructor(message: string, readonly pos: Pos) { super(message); } } +class Limit extends Error { constructor(message: string, readonly pos: Pos | null = null) { super(message); } } +const MAX_CALL_DEPTH = 200; +/** The passes one call of std.math's gcd or comb may take here; Mojo's own can take far more (Interp.gcd). */ +const MATH_STEPS = 1_000_000; +/** std.math's functions whose digits are Mojo's own routine's (or the C library's), not reproduced here. */ +const MATH_OWN = new Set(['sin', 'cos', 'tan', 'asin', 'acos', 'atan', 'atan2', 'sinh', 'cosh', 'tanh', 'asinh', 'acosh', 'atanh', 'exp', 'exp2', 'expm1', 'log', 'log2', 'log10', 'log1p', 'cbrt', 'erf', 'erfc', 'gamma', 'lgamma']); + +const BUILTINS = new Set(['print', 'len', 'ord', 'chr', 'range', 'input', 'atol', 'atof', 'abs', 'min', 'max', 'round', 'sort', 'exit', 'assert_equal', 'assert_not_equal', 'assert_true', 'assert_false', 'assert_almost_equal', 'sqrt', 'floor', 'ceil', 'trunc']); + +const constLoc = (v: V): Loc => ({ get: () => v, set: () => { throw new Refuse('assigning to a temporary', null); }, name: '', frame: -1 }); +const dunder = (op: string): string => ({ '+': '__add__', '-': '__sub__', '*': '__mul__', '/': '__truediv__', '//': '__floordiv__', '%': '__mod__', '**': '__pow__', '&': '__and__', '|': '__or__', '^': '__xor__', '<<': '__lshift__', '>>': '__rshift__', '@': '__matmul__' } as Record<string, string>)[op]; +const inplace = (op: string): string => dunder(op).replace('__', '__i'); +const num = (v: V): number | null => (v.k === 'int' ? Number(v.v) : v.k === 'float' ? v.v : null); + +function typeText(e: Expr): string { + if (e.k === 'name') return e.id; + if (e.k === 'member') return e.name; // .float32 + if (e.k === 'attr' && e.obj.k === 'name' && e.obj.id === 'DType') return e.name; // DType.float32 + if (e.k === 'index' && e.obj.k === 'name') return `${e.obj.id}[${e.args.map((a) => typeText(a.value)).join(', ')}]`; + if (e.k === 'num') return e.text; + return '?'; +} + +function targetText(e: Expr): string { + if (e.k === 'name') return e.id; + if (e.k === 'tuple') return e.items.map(targetText).join(', '); + return '…'; +} + +/** A t-string's literal parts still carry their escapes. */ +function cookT(s: string): string { return s.replace(/\\n/g, '\n').replace(/\\t/g, '\t').replace(/\\"/g, '"').replace(/\\\\/g, '\\'); } + +/* ---------------- Float64 bits: std.math's exact functions ---------------- */ + +const bitsOf = (x: number): bigint => { const d = new DataView(new ArrayBuffer(8)); d.setFloat64(0, x); return d.getBigUint64(0); }; +const fromBits = (b: bigint): number => { const d = new DataView(new ArrayBuffer(8)); d.setBigUint64(0, BigInt.asUintN(64, b)); return d.getFloat64(0); }; + +/** std.utils.numerics.nextafter: the neighbor of x toward y, by its bits; y itself when they are equal, NaN for a NaN. */ +function nextAfter(x: number, y: number): number { + if (Number.isNaN(x) || Number.isNaN(y)) return NaN; + if (x === y) return y; // +0 and -0 too: y + if (x === 0) return y < 0 ? -fromBits(1n) : fromBits(1n); + const b = bitsOf(x); + return fromBits((y > x) === (x > 0) ? b + 1n : b - 1n); +} + +/** A finite double as an exact fraction m * 2^e (m a BigInt). */ +function exactOf(x: number): [bigint, number] { + const b = bitsOf(x); + const exp = Number((b >> 52n) & 0x7ffn), frac = b & ((1n << 52n) - 1n); + const m = exp === 0 ? frac : frac | (1n << 52n); + return [x < 0 || Object.is(x, -0) ? -m : m, (exp === 0 ? 1 : exp) - 1075]; +} + +/** a * b + c rounded once, as the fused instruction does. */ +function fma(a: number, b: number, c: number): number { + if (!Number.isFinite(a) || !Number.isFinite(b) || !Number.isFinite(c)) return a * b + c; // the IEEE special cases agree + const [ma, ea] = exactOf(a), [mb, eb] = exactOf(b), [mc, ec] = exactOf(c); + const mp = ma * mb, ep = ea + eb; + const e = Math.min(ep, ec); + const sum = (mp << BigInt(ep - e)) + (mc << BigInt(ec - e)); + if (sum === 0n) { + // an exact zero: -0 only when the product and c are both -0, or both negative zeros of opposite form + const prodNeg = (a < 0 || Object.is(a, -0)) !== (b < 0 || Object.is(b, -0)); + return mp === 0n && mc === 0n && prodNeg && Object.is(c, -0) ? -0 : 0; + } + return e >= 0 ? ratioToDouble(sum << BigInt(e), 1n) : ratioToDouble(sum, 1n << BigInt(-e)); +} + +/** std.math's hypot as Mojo 1.2.0 computes it (_hypot_impl): scaled by the larger operand's exponent, an fma, an IEEE square root. */ +function hypot(x: number, y: number): number { + if (x === Infinity || x === -Infinity || y === Infinity || y === -Infinity) return Infinity; + if (Number.isNaN(x) || Number.isNaN(y)) return NaN; + const MAG = (1n << 63n) - 1n, MANT = (1n << 52n) - 1n; + const ax = bitsOf(x) & MAG, ay = bitsOf(y) & MAG; + const big = ax > ay ? ax : ay, small = ax > ay ? ay : ax; + const eBig = (big >> 52n) & 0x7ffn, eSmall = (small >> 52n) & 0x7ffn; + const clamp = (v: bigint, lo: bigint, hi: bigint) => (v < lo ? lo : v > hi ? hi : v); + const e = clamp(eBig - 1023n, -1022n, 1022n); + const down = fromBits((1023n - e) << 52n), up = fromBits((1023n + e) << 52n); + let a = fromBits(eBig === 0n ? 0n : big) * down; + let b = fromBits(eSmall === 0n ? 0n : small) * down; + const unit = fromBits((clamp(-1074n - e, -1022n, 1023n) + 1023n) << 52n); + if (eBig === 0n) a = Number(big & MANT) * unit; + if (eSmall === 0n && e <= -52n) b = Number(small & MANT) * unit; + return Math.sqrt(fma(a, a, b * b)) * up; +} + +/** std.math's pi as Mojo 1.2.0 writes it (std/math/constants.mojo). */ +const PI_DIGITS = '3.1415926535897932384626433832795028841971693993751058209749445923'; + +/** n / d rounded once to the nearest double, ties to even, as the compiler materializes a FloatLiteral. */ +function ratioToDouble(n: bigint, d: bigint): number { + if (n === 0n) return 0; + const neg = (n < 0n) !== (d < 0n); + if (n < 0n) n = -n; + if (d < 0n) d = -d; + const bits = (x: bigint): number => x.toString(2).length; + // q * 2^e with q of 53 bits (fewer for a subnormal), and the remainder to round by + const at = (e: number): [bigint, bigint, bigint] => { const N = e < 0 ? n << BigInt(-e) : n, D = e > 0 ? d << BigInt(e) : d; return [N / D, N % D, D]; }; + let e = bits(n) - bits(d) - 53; + let [q, r, D] = at(e); + if (bits(q) > 53) { e++; [q, r, D] = at(e); } + if (e < -1074) { e = -1074; [q, r, D] = at(e); } + if (2n * r > D || (2n * r === D && (q & 1n) === 1n)) q += 1n; + const v = Number(q) * 2 ** e; + return neg ? -v : v; +} + +/** A literal-only arithmetic expression as an exact fraction, or null; `float` when a float literal is in it. */ +function foldLiteral(e: Expr, named?: (name: string, pos: Pos) => string | null): { n: bigint; d: bigint; float: boolean } | null { + const gcd = (a: bigint, b: bigint): bigint => { a = a < 0n ? -a : a; b = b < 0n ? -b : b; while (b) [a, b] = [b, a % b]; return a || 1n; }; + const norm = (n: bigint, d: bigint, float: boolean) => { if (d < 0n) { n = -n; d = -d; } const g = gcd(n, d); return { n: n / g, d: d / g, float }; }; + switch (e.k) { + case 'name': { const text = named?.(e.id, e.pos); return text ? foldLiteral({ k: 'num', text, float: true, pos: e.pos } as Expr) : null; } + case 'num': { + const t = e.text.replace(/_/g, ''); + if (!e.float) return /^\d+$/.test(t) ? { n: BigInt(t), d: 1n, float: false } : null; + const m = /^(\d*)(?:\.(\d*))?(?:[eE]([+-]?\d+))?$/.exec(t); + if (!m) return null; + const frac = m[2] ?? '', exp = Number(m[3] ?? 0) - frac.length; + // 1e999999 would ask for a number of millions of digits; past this the literal is not folded exactly + if (Math.abs(exp) > 10000 || frac.length + (m[1] ?? '').length > 10000) throw new Refuse('a float literal with more than 10,000 digits or an exponent past 10,000', e.pos); + const digits = BigInt((m[1] || '0') + frac); + return exp >= 0 ? norm(digits * 10n ** BigInt(exp), 1n, true) : norm(digits, 10n ** BigInt(-exp), true); + } + case 'unary': { if (e.op !== '-' && e.op !== '+') return null; const x = foldLiteral(e.x, named); return x && (e.op === '-' ? { ...x, n: -x.n } : x); } + case 'binary': { + const a = foldLiteral(e.l, named), b = foldLiteral(e.r, named); + if (!a || !b) return null; + const float = a.float || b.float; + switch (e.op) { + case '+': return norm(a.n * b.d + b.n * a.d, a.d * b.d, float); + case '-': return norm(a.n * b.d - b.n * a.d, a.d * b.d, float); + case '*': return norm(a.n * b.n, a.d * b.d, float); + case '/': return float && b.n !== 0n ? norm(a.n * b.d, a.d * b.n, true) : null; // an Int literal / is not modeled here + default: return null; + } + } + default: return null; + } +} + +function containsReflect(body: Stmt[]): boolean { return /"id":"reflect"/.test(JSON.stringify(body)); } + +function containsCall(body: Stmt[]): boolean { + return JSON.stringify(body).includes('"k":"call"'); +} diff --git a/src/mojo/lex.ts b/src/mojo/lex.ts new file mode 100644 index 0000000..a430a23 --- /dev/null +++ b/src/mojo/lex.ts @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The Mojo lexer for Programming+'s checker: source text to tokens, with the + * indentation made explicit (INDENT, DEDENT) and newlines inside brackets + * ignored, as in Python. Positions are 1-based lines and columns, the + * compiler's convention, so a finding lands where the compiler's would. + * + * Punctuation and keywords are the compiler's own (src/mojo/generated/tokens.ts). + * A character no token can start with is the compiler's "unexpected character". + */ +import { MOJO_PUNCTUATION } from './generated/tokens.js'; + +export type Kind = 'name' | 'number' | 'string' | 'op' | 'newline' | 'indent' | 'dedent' | 'eof'; + +export interface Tok { + kind: Kind; + text: string; + line: number; + col: number; + /** One past the last column, on the token's first line. */ + endCol: number; + /** A string's prefix letters (r, t) and whether it is triple-quoted. */ + prefix?: string; + triple?: boolean; + /** A float literal (has a point or an exponent). */ + float?: boolean; + /** A `backtick` name: any text, MLIR's included. */ + escaped?: boolean; +} + +export interface LexError { line: number; col: number; message: string } + +const PUNCT = [...MOJO_PUNCTUATION].sort((a, b) => b.length - a.length); +const OPEN = new Set(['(', '[', '{']); +const CLOSE = new Set([')', ']', '}']); + +export function lex(source: string): { tokens: Tok[]; errors: LexError[] } { + const tokens: Tok[] = []; + const errors: LexError[] = []; + const lines = source.split('\n'); + const indents = [0]; + let depth = 0; // bracket nesting: newlines inside brackets are not statement ends + let i = 0; // line index + let resume: number | null = null; // after a string that spans lines: the column to go on from + const push = (t: Tok) => tokens.push(t); + + while (i < lines.length) { + const text = lines[i]; + const lineNo = i + 1; + let c = 0; + if (resume !== null) { c = resume; resume = null; } + else if (depth === 0) { + // a blank or comment-only line says nothing about indentation + const stripped = text.replace(/^[ \t]*/, ''); + if (!stripped || stripped.startsWith('#')) { i++; continue; } + const width = text.length - stripped.length; + if (width > indents[indents.length - 1]) { indents.push(width); push({ kind: 'indent', text: '', line: lineNo, col: 1, endCol: width + 1 }); } + while (width < indents[indents.length - 1]) { indents.pop(); push({ kind: 'dedent', text: '', line: lineNo, col: width + 1, endCol: width + 1 }); } + c = width; + } + let continued = false; + while (c < text.length) { + const ch = text[c]; + const rest = text.slice(c); + if (ch === ' ' || ch === '\t') { c++; continue; } + if (ch === '#') break; + if (ch === '\\' && c === text.length - 1) { continued = true; break; } + const s = /^([rRtT]{0,2})("""|'''|"|')/.exec(rest); + if (s && (s[1] === '' || /^[rRtT]{1,2}$/.test(s[1]))) { + const quote = s[2], prefix = s[1].toLowerCase(), raw = prefix.includes('r'); + const start = { line: lineNo, col: c + 1 }; + let j = c + s[0].length, body = ''; + let ln = i, cur = text; + for (; ;) { + if (j >= cur.length) { + if (quote.length === 3 && ln + 1 < lines.length) { body += '\n'; ln++; cur = lines[ln]; j = 0; continue; } + break; + } + // a backslash escapes in a raw string too, as far as the quote goes: r"A\"B" is one string + if (cur[j] === '\\' && (!raw || cur[j + 1] === quote[0] || cur[j + 1] === '\\')) { body += cur.slice(j, j + 2); j += 2; continue; } + if (cur.startsWith(quote, j)) { j += quote.length; break; } + body += cur[j]; j++; + } + push({ kind: 'string', text: body, line: start.line, col: start.col, endCol: ln === i ? j + 1 : cur.length + 1, prefix, triple: quote.length === 3 }); + if (ln !== i) { i = ln; resume = j; break; } + c = j; + continue; + } + const num = /^(?:0[xX][0-9a-fA-F_]+|0[oO][0-7_]+|0[bB][01_]+|(?:\d[\d_]*)?\.\d[\d_]*(?:[eE][+-]?\d+)?|\d[\d_]*(?:\.(?!\.)[\d_]*)?(?:[eE][+-]?\d+)?)/.exec(rest); + if (num && num[0] && /^[\d.]/.test(ch) && !(ch === '.' && !/\d/.test(text[c + 1] ?? ''))) { + // the compiler's own lexer messages for malformed literals (its tests, lexer_errors.mojo) + if (/^0[0-9_]*[1-9]/.test(num[0]) && !/[.eE]/.test(num[0])) errors.push({ line: lineNo, col: c + 1, message: "decimal integer literals must not use leading zeros; add '0o' for octal literals" }); + if (/^0[oO]_*$/.test(num[0])) errors.push({ line: lineNo, col: c + 1, message: 'no digits specified for octal literal' }); + push({ kind: 'number', text: num[0], line: lineNo, col: c + 1, endCol: c + num[0].length + 1, float: /[.eE]/.test(num[0]) && !/^0[xX]/.test(num[0]) }); + c += num[0].length; + continue; + } + const id = /^[A-Za-z_][A-Za-z0-9_]*/.exec(rest); + if (id) { push({ kind: 'name', text: id[0], line: lineNo, col: c + 1, endCol: c + id[0].length + 1 }); c += id[0].length; continue; } + if (ch === '`') { + const end = text.indexOf('`', c + 1); + if (end < 0) { errors.push({ line: lineNo, col: c + 1, message: 'unterminated backtick identifier' }); c = text.length; continue; } + if (end > c) { push({ kind: 'name', text: text.slice(c + 1, end), line: lineNo, col: c + 1, endCol: end + 2, escaped: true }); c = end + 1; continue; } + } + const p = PUNCT.find((x) => rest.startsWith(x)); + if (p) { + if (OPEN.has(p)) depth++; + else if (CLOSE.has(p)) depth = Math.max(0, depth - 1); + push({ kind: 'op', text: p, line: lineNo, col: c + 1, endCol: c + p.length + 1 }); + c += p.length; + continue; + } + errors.push({ line: lineNo, col: c + 1, message: 'unexpected character' }); + c++; + } + if (resume !== null) continue; + if (!continued && depth === 0 && tokens.length && tokens[tokens.length - 1].kind !== 'newline' && tokens[tokens.length - 1].kind !== 'indent' && tokens[tokens.length - 1].kind !== 'dedent') { + push({ kind: 'newline', text: '', line: lineNo, col: text.length + 1, endCol: text.length + 1 }); + } + i++; + } + const last = lines.length; + if (tokens.length && tokens[tokens.length - 1].kind !== 'newline') push({ kind: 'newline', text: '', line: last, col: 1, endCol: 1 }); + while (indents.length > 1) { indents.pop(); push({ kind: 'dedent', text: '', line: last + 1, col: 1, endCol: 1 }); } + push({ kind: 'eof', text: '', line: last + 1, col: 1, endCol: 1 }); + return { tokens, errors }; +} diff --git a/src/mojo/lifetimes.ts b/src/mojo/lifetimes.ts new file mode 100644 index 0000000..de4575b --- /dev/null +++ b/src/mojo/lifetimes.ts @@ -0,0 +1,260 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * When each value dies: Mojo destroys a value at the first point where it can no + * longer be read, along the path the program takes, not at the end of its scope. + * Every rule here is a probe's answer (library/mojo/probes/life-*, godbolt's stored answers at the pin): + * + * straight-line code it dies at its last read (life-last-use, life-asap-expression) + * a branch that never reads it it dies as that branch begins (life-branch-one-side, + * life-early-return, life-moved-on-one-path) + * a loop that reads it each pass it dies when the loop ends (life-loop-outer-read-inside) + * overwritten before it is read it dies as the pass that overwrites it begins (life-reassign-in-loop) + * + * This is the static half: a backward liveness analysis over the function's + * structure. For every statement, the variables that may still be read after it + * (`liveOut`); for every block (a branch's body, a loop's body), those that may be + * read from its start (`liveIn`). The interpreter (src/mojo/interp.ts) destroys a + * value when its variable leaves the live set: at its last read inside a statement, + * at the start of a branch or pass that does not read it, when a loop ends, at a + * return. Code whose control flow this does not model (try, with, a nested function, + * a comprehension or lambda, a t-string's names, ref) marks the names it touches + * `unmodeled`, and the interpreter declines rather than guess. + */ +import type { Expr, Stmt } from './parse.js'; + +type DefStmt = Extract<Stmt, { k: 'def' }>; +type Live = Set<string>; + +export interface Lifetimes { + /** What may be read after each statement. */ + liveOut: Map<Stmt, Live>; + /** What may be read from the start of each block (a branch's body, a loop's body). */ + liveIn: Map<Stmt[], Live>; + /** Read nodes that are the last read, in their statement, of a value not live after it. */ + lastReads: Set<Expr>; + /** Statements after which a variable's new value, never read, dies: statement → variable names. */ + unreadAfter: Map<Stmt, string[]>; + /** Names used where control flow is not modeled: their destruction is declined. */ + unmodeled: Set<string>; + /** Parameters never read (nor moved): the compiler does not warn about these. */ + unusedParams: Set<string>; + /** The same set as `unmodeled`, under the name the warnings use. */ + complex: Set<string>; +} + +interface Ctx { breakOut: Live | null; continueOut: Live | null } + +/** + * How many statements one function's analysis may visit. A loop's body is visited until its live set settles + * (twice at least), so loops nested n deep visit their inner body 2^n times or more; past this budget the function + * is not analyzed, and every name in it is `unmodeled` (the interpreter then declines to say when a value with a + * visible lifecycle dies, and the warnings stay quiet). tests/mojo-robustness.test.mjs. + */ +const MAX_VISITS = 200_000; +class TooComplex extends Error {} +let visits = 0; + +export function lifetimes(def: DefStmt): Lifetimes { + const unmodeled = new Set<string>(); + const out: Lifetimes = { liveOut: new Map(), liveIn: new Map(), lastReads: new Set(), unreadAfter: new Map(), unmodeled, unusedParams: new Set(), complex: unmodeled }; + visits = 0; + try { + const bodyIn = block(def.body, new Set(), { breakOut: null, continueOut: null }, out); + for (const p of def.params) if (p.star !== 'marker' && p.star !== 'positional-marker' && !bodyIn.has(p.name) && !moves(def.body, p.name)) out.unusedParams.add(p.name); + return out; + } catch (e) { + if (!(e instanceof TooComplex)) throw e; + const all = usedNames(def.body); + for (const p of def.params) all.add(p.name); + return { liveOut: new Map(), liveIn: new Map(), lastReads: new Set(), unreadAfter: new Map(), unmodeled: all, unusedParams: new Set(), complex: all }; + } +} + +/** Results for a fixed-point iteration: thrown away, so only the converged pass records anything. */ +const scratch = (out: Lifetimes): Lifetimes => ({ liveOut: new Map(), liveIn: new Map(), lastReads: new Set(), unreadAfter: new Map(), unmodeled: out.unmodeled, unusedParams: new Set(), complex: out.unmodeled }); + +const union = (...sets: Live[]): Live => { const u = new Set<string>(); for (const s of sets) for (const x of s) u.add(x); return u; }; +const same = (a: Live, b: Live) => a.size === b.size && [...a].every((x) => b.has(x)); + +/** Live-in of a block, given what is live after it; records each statement's live-out on the way. */ +function block(body: Stmt[], after: Live, ctx: Ctx, out: Lifetimes): Live { + visits += body.length; + if (visits > MAX_VISITS) throw new TooComplex(); + let live = after; + for (let i = body.length - 1; i >= 0; i--) { + out.liveOut.set(body[i], live); + live = stmt(body[i], live, ctx, out); + } + out.liveIn.set(body, live); + return live; +} + +function stmt(s: Stmt, after: Live, ctx: Ctx, out: Lifetimes): Live { + switch (s.k) { + case 'if': { + const thenIn = block(s.body, after, ctx, out); + const elseIn = s.orelse.length ? block(s.orelse, after, ctx, out) : after; + return union(reads(s.test, out), thenIn, elseIn); + } + case 'while': { + // the test runs before every pass and once more at the end; iterate on scratch results + // to a fixed point, then record from the converged sets only (stale marks would destroy early) + // `while True` never ends through its test: only its breaks leave + const forever = s.test.k === 'name' && s.test.id === 'True'; + const exit = forever ? new Set<string>() : s.orelse.length ? block(s.orelse, after, ctx, out) : after; + let head = union(reads(s.test, out), exit); + for (let n = 0; n < 50; n++) { + const bodyIn = block(s.body, head, { breakOut: after, continueOut: head }, scratch(out)); + const next = union(reads(s.test, out), bodyIn, exit); + if (same(next, head)) break; + head = next; + } + block(s.body, head, { breakOut: after, continueOut: head }, out); + return head; + } + case 'for': { + // each pass binds the loop variable, then runs the body; after the last pass, what follows the loop + const target = names(s.target); + const exit = s.orelse.length ? block(s.orelse, after, ctx, out) : after; + let head = exit; + for (let n = 0; n < 50; n++) { + const bodyIn = block(s.body, head, { breakOut: after, continueOut: head }, scratch(out)); + const next = union(new Set([...bodyIn].filter((x) => !target.includes(x))), exit); + if (same(next, head)) break; + head = next; + } + block(s.body, head, { breakOut: after, continueOut: head }, out); + return union(reads(s.iter, out), head); + } + case 'return': return s.value ? reads(s.value, out) : new Set(); + case 'raise': return s.value ? reads(s.value, out) : new Set(); + case 'break': return ctx.breakOut ?? after; + case 'continue': return ctx.continueOut ?? after; + case 'try': case 'with': { + const used = usedNames(s); + for (const n of used) out.unmodeled.add(n); + return union(after, used); + } + case 'def': case 'struct': case 'trait': { + const used = usedNames(s.body); + for (const n of used) out.unmodeled.add(n); + return union(after, used); + } + default: return simple(s, after, out); + } +} + +/** A simple statement: what it reads, what it writes; and in it, each value's last read. */ +function simple(s: Stmt, after: Live, out: Lifetimes): Live { + const order: Array<{ name: string; node: Expr | null; kind: 'read' | 'move' | 'write' }> = []; + const push = (e: Expr) => walkReads(e, (node, moved) => order.push({ name: (node as Extract<Expr, { k: 'name' }>).id, node, kind: moved ? 'move' : 'read' }), (n) => n.forEach((x) => out.unmodeled.add(x))); + switch (s.k) { + case 'expr': push(s.e); break; + case 'var': + if (s.value) push(s.value); + if (s.ref) for (const n of names(s.targets)) out.unmodeled.add(n); + if (s.value) for (const n of names(s.targets)) order.push({ name: n, node: null, kind: 'write' }); + break; + case 'assign': + push(s.value); + for (const t of s.targets) { if (t.k === 'name' || t.k === 'tuple') for (const n of names(t)) order.push({ name: n, node: null, kind: 'write' }); else push(t); } + break; + case 'aug': push(s.value); push(s.target); break; + case 'assert': for (const n of namesIn(s.test)) out.unmodeled.add(n); break; + default: break; + } + // backwards through the statement's own events: the last read of a value not live after it + const live = new Set(after); + const unread: string[] = []; + for (let i = order.length - 1; i >= 0; i--) { + const ev = order[i]; + if (ev.kind === 'write') { + if (!live.has(ev.name)) unread.push(ev.name); + live.delete(ev.name); + } else if (ev.kind === 'read') { + if (!live.has(ev.name) && ev.node) out.lastReads.add(ev.node); + live.add(ev.name); + } else { + live.add(ev.name); // x^ needs the value (a use), but its leaving is not a destruction + } + } + if (unread.length) out.unreadAfter.set(s, unread); + return live; +} + +function reads(e: Expr, out: Lifetimes): Live { + const live = new Set<string>(); + walkReads(e, (node) => { live.add((node as Extract<Expr, { k: 'name' }>).id); }, (n) => n.forEach((x) => out.unmodeled.add(x))); // a move needs the value too + return live; +} + +function names(e: Expr): string[] { + if (e.k === 'name') return e.id === '_' ? [] : [e.id]; + if (e.k === 'tuple' || e.k === 'list') return e.items.flatMap(names); + return []; +} + +/** + * Reads in evaluation order; `moved` for `x^`. A comprehension, a lambda, or a t-string's {…} + * reads names this does not follow: they go to `captured` (unmodeled). + */ +function walkReads(e: Expr, f: (node: Expr, moved: boolean) => void, captured: (names: string[]) => void): void { + const w = (x: Expr) => walkReads(x, f, captured); + switch (e.k) { + case 'name': f(e, false); return; + case 'transfer': if (e.x.k === 'name') f(e.x, true); else w(e.x); return; + case 'call': w(e.fn); e.args.forEach((a) => w(a.value)); return; + case 'index': w(e.obj); e.args.forEach((a) => w(a.value)); return; + case 'attr': w(e.obj); return; + case 'list': case 'set': case 'tuple': e.items.forEach(w); return; + case 'dict': e.entries.forEach(([k, v]) => { w(k); w(v); }); return; + case 'unary': case 'not': w(e.x); return; + case 'binary': case 'bool': w(e.l); w(e.r); return; + case 'compare': e.operands.forEach(w); return; + case 'ifexp': w(e.cond); w(e.a); w(e.b); return; + case 'record': e.fields.forEach((x) => w(x.value)); return; + case 'walrus': w(e.value); return; + case 'slice': for (const x of [e.lo, e.hi, e.step]) if (x) w(x); return; + case 'comp': case 'lambda': captured(namesIn(e)); return; + case 'str': if (e.t) captured(e.value.match(/[A-Za-z_]\w*/g) ?? []); return; + default: return; + } +} + +function namesIn(e: unknown): string[] { + const out: string[] = []; + const visit = (x: any) => { + if (!x || typeof x !== 'object') return; + if (Array.isArray(x)) { x.forEach(visit); return; } + if (x.k === 'name' && typeof x.id === 'string') out.push(x.id); + if (x.k === 'str' && x.t && typeof x.value === 'string') out.push(...(x.value.match(/[A-Za-z_]\w*/g) ?? [])); + for (const v of Object.values(x)) if (v && typeof v === 'object') visit(v); + }; + visit(e); + return out; +} + +function usedNames(x: unknown): Live { + const out = new Set<string>(namesIn(x)); + const visit = (y: any) => { + if (!y || typeof y !== 'object') return; + if (Array.isArray(y)) { y.forEach(visit); return; } + if (y.k === 'var' && y.targets?.k === 'name') out.add(y.targets.id); + for (const v of Object.values(y)) if (v && typeof v === 'object') visit(v); + }; + visit(x); + return out; +} + +function moves(body: Stmt[], name: string): boolean { + let found = false; + const visit = (x: any) => { + if (found || !x || typeof x !== 'object') return; + if (Array.isArray(x)) { x.forEach(visit); return; } + if (x.k === 'transfer' && x.x?.k === 'name' && x.x.id === name) { found = true; return; } + for (const v of Object.values(x)) if (v && typeof v === 'object') visit(v); + }; + visit(body); + return found; +} diff --git a/src/mojo/parse.ts b/src/mojo/parse.ts new file mode 100644 index 0000000..dedd986 --- /dev/null +++ b/src/mojo/parse.ts @@ -0,0 +1,901 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Programming+'s Mojo parser: tokens (src/mojo/lex.ts) to a syntax tree for + * the checker (Stage 2) and, later, the interpreter (Stage 3). + * + * Two kinds of finding, never mixed up: + * errors mistakes the compiler certainly reports, in its words and at its + * positions (an empty body, a parameter without a type, code at + * file scope, an instance method without self). Scored against + * godbolt's verdicts on Mojo Quest's starters; never raised on a + * program the compiler accepts (tests/mojo-check.test.mjs). + * unread syntax this parser does not know. It says so (a note: "compile + * to check") and skips to the next statement, rather than guess, + * and the checker stays quiet about anything it could not read. + * + * Operator precedence is the reference's (docs/site/reference/operators.mdx). + */ +import { lex, type Tok } from './lex.js'; +import { MOJO_KEYWORDS } from './generated/tokens.js'; + +export interface Pos { line: number; col: number; endCol: number } + +export type Expr = + | { k: 'name'; id: string; pos: Pos; escaped?: boolean } + | { k: 'num'; text: string; float: boolean; pos: Pos } + /** `value` is the string's contents with escapes applied; a t-string keeps its source text, so its {…} can be found. */ + | { k: 'str'; value: string; t: boolean; pos: Pos } + | { k: 'list' | 'set' | 'tuple'; items: Expr[]; pos: Pos } + | { k: 'dict'; entries: Array<[Expr, Expr]>; pos: Pos } + | { k: 'comp'; kind: 'list' | 'set' | 'dict' | 'gen'; elt: Expr; vars: Expr; iter: Expr; cond: Expr | null; pos: Pos } + | { k: 'call'; fn: Expr; args: Arg[]; pos: Pos; lparen: Pos } + | { k: 'index'; obj: Expr; args: Arg[]; pos: Pos; lbracket: Pos } + | { k: 'attr'; obj: Expr; name: string; pos: Pos; dot: Pos } + | { k: 'unary'; op: string; x: Expr; pos: Pos } + | { k: 'binary'; op: string; l: Expr; r: Expr; pos: Pos } + | { k: 'compare'; ops: string[]; operands: Expr[]; pos: Pos; opPos: Pos[] } + | { k: 'bool'; op: 'and' | 'or'; l: Expr; r: Expr; pos: Pos } + | { k: 'not'; x: Expr; pos: Pos } + | { k: 'ifexp'; cond: Expr; a: Expr; b: Expr; pos: Pos } + | { k: 'transfer'; x: Expr; pos: Pos } + | { k: 'walrus'; name: string; value: Expr; pos: Pos } + | { k: 'slice'; lo: Expr | null; hi: Expr | null; step: Expr | null; pos: Pos } + | { k: 'ellipsis'; pos: Pos } + | { k: 'lambda'; params: string[]; body: Expr; pos: Pos; typed?: Param[]; captures?: Captures; ret?: Expr | null } + /** `.float32`: a member of the type the context expects (a DType, say). */ + | { k: 'member'; name: string; pos: Pos } + /** `{count = 2}`: fields given by name. */ + | { k: 'record'; fields: Array<{ name: string; value: Expr }>; pos: Pos } + /** A type the checker does not look inside: `def(T) -> Bool`, `ref[origin] T`. */ + | { k: 'opaque'; what: string; inner: Expr[]; pos: Pos; fn?: { params: Expr[]; ret: Expr | null; effects: string[] } }; + +export interface Arg { name: string | null; value: Expr; star?: '*' | '**' } + +/** + * A closure's capture list (the manual's closures chapter): `{imm x, mut y, var z^}`, a default + * convention alone (`{mut}`, applying to every outer name the body uses), or both (`{mut, var z}`). + * A bare name is `imm`. `{}` captures nothing. + */ +export interface Captures { defaultConv: string | null; items: Array<{ name: string; conv: string; move: boolean; pos: Pos }> } + +export interface Param { + name: string; + convention: string | null; + type: Expr | null; + default: Expr | null; + /** `*args`, `**kwargs`, or the bare `*` / `/` markers. */ + star?: '*' | '**' | 'marker' | 'positional-marker'; + pos: Pos; +} + +export type Stmt = + | { k: 'expr'; e: Expr; pos: Pos } + | { k: 'var'; targets: Expr; type: Expr | null; value: Expr | null; comptime: boolean; ref: boolean; pos: Pos; typeParams?: Param[] } + | { k: 'assign'; targets: Expr[]; value: Expr; pos: Pos } + | { k: 'aug'; target: Expr; op: string; value: Expr; pos: Pos } + | { k: 'if'; test: Expr; body: Stmt[]; orelse: Stmt[]; comptime: boolean; pos: Pos } + | { k: 'while'; test: Expr; body: Stmt[]; orelse: Stmt[]; pos: Pos } + | { k: 'for'; target: Expr; iter: Expr; body: Stmt[]; orelse: Stmt[]; comptime: boolean; pos: Pos; convention: string | null } + | { k: 'break' | 'continue' | 'pass'; pos: Pos } + | { k: 'return'; value: Expr | null; pos: Pos } + | { k: 'raise'; value: Expr | null; pos: Pos } + | { k: 'try'; body: Stmt[]; handlers: Array<{ name: string | null; type: Expr | null; body: Stmt[]; pos: Pos }>; orelse: Stmt[]; finalbody: Stmt[]; pos: Pos } + | { k: 'with'; items: Array<{ e: Expr; as: Expr | null }>; body: Stmt[]; pos: Pos } + | { k: 'assert'; test: Expr; msg: Expr | null; comptime: boolean; pos: Pos } + | { k: 'def'; name: string; params: Param[]; ret: Expr | null; raises: boolean; typeParams: Param[]; decorators: Expr[]; body: Stmt[]; pos: Pos; colon: Pos; raisesType?: Expr | null; captures?: Captures | null } + | { k: 'struct' | 'trait'; name: string; bases: Expr[]; typeParams: Param[]; decorators: Expr[]; body: Stmt[]; pos: Pos; colon: Pos } + | { k: 'import'; module: string; alias: string | null; pos: Pos } + | { k: 'from'; module: string; names: Array<{ name: string; alias: string | null; pos: Pos }>; star: boolean; pos: Pos } + | { k: 'global'; names: string[]; pos: Pos } + | { k: 'unread'; pos: Pos }; + +export interface Finding { severity: 'error' | 'note'; line: number; col: number; endCol: number; message: string; id: string; /** For an unread note: what the parser expected, and the token it met. */ why?: string } + +export interface Module { body: Stmt[]; findings: Finding[]; unread: number } + +class Unread extends Error { constructor(readonly tok: Tok, readonly why: string) { super(why); } } + +/** + * How deep a program may nest (brackets, blocks, calls inside calls, prefix operators, lambdas) and how tall its + * syntax tree may grow, a long chain such as a + b + c ... counting one level per operand. Every stage after the + * parser (the checks, the practices, the interpreter, the translators) walks the tree by calling itself, so a + * tree tens of thousands of levels deep would end in the engine's own stack error. The pinned compiler accepts + * 5,000 nested brackets, 1,000 nested blocks, 5,000 elif arms and a 10,000-term sum, and crashes (a stack + * overflow, exit 139) on 30,000 nested brackets or a 30,000-term sum, so neither limit is an error: past either, + * the statement is code Programming+ cannot read yet, with one note where the limit is passed + * (tests/mojo-robustness.test.mjs, docs/limits.md). + */ +export const MAX_NESTING = 200; +export const MAX_TREE = 1000; + +/** The one note a program nested past the limits gets, in the place of the usual unread note. */ +const tooDeep = (what: string): string => `${what}; Programming+ cannot read this yet, and Compile & run asks the real compiler`; + +const COMPARE = new Set(['==', '!=', '<', '<=', '>', '>=']); +const AUG = new Set(['+=', '-=', '*=', '/=', '//=', '%=', '**=', '@=', '&=', '|=', '^=', '<<=', '>>=']); +const SOFT_CONVENTIONS = new Set(['mut', 'out', 'deinit', 'imm', 'read', 'owned', 'inout', 'borrowed']); + +const at = (t: Tok): Pos => ({ line: t.line, col: t.col, endCol: t.endCol }); + +const ESCAPES: Record<string, string> = { n: '\n', t: '\t', r: '\r', '\\': '\\', '"': '"', "'": "'", '0': '\0', a: '\x07', b: '\b', f: '\f', v: '\v', '\n': '' }; + +/** A string literal's escapes applied: \n, \t, \\, \", \xNN, \uNNNN, \UNNNNNNNN. */ +export function cook(text: string): string { + return text.replace(/\\(x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8}|[\s\S])/g, (m, c: string) => { + if (c.length > 1) { const cp = parseInt(c.slice(1), 16); return cp <= 0x10ffff ? String.fromCodePoint(cp) : m; } // \xNN, \uNNNN, \UNNNNNNNN + return c in ESCAPES ? ESCAPES[c] : m; + }); +} + +export function parseModule(source: string): Module { + const { tokens, errors } = lex(source); + const findings: Finding[] = errors.map((e) => ({ severity: 'error' as const, line: e.line, col: e.col, endCol: e.col + 1, message: e.message, id: 'unexpected-character' })); + let i = 0; + let unread = 0; + const peek = (o = 0) => tokens[Math.min(i + o, tokens.length - 1)]; + const next = () => tokens[i++]; + const isOp = (t: Tok, ...ops: string[]) => t.kind === 'op' && ops.includes(t.text); + const isName = (t: Tok, ...ws: string[]) => t.kind === 'name' && (ws.length === 0 || ws.includes(t.text)); + const expectOp = (op: string) => { const t = peek(); if (!isOp(t, op)) throw new Unread(t, `expected '${op}'`); return next(); }; + const expectName = () => { const t = peek(); if (t.kind !== 'name') throw new Unread(t, 'expected a name'); return next(); }; + const finding = (f: Omit<Finding, 'severity'> & { severity?: Finding['severity'] }) => findings.push({ severity: 'error', ...f } as Finding); + let depth = 0; // brackets, blocks, prefix operators and calls open around the token being read + let deepSaid = false; // the too-deep note is said once for the program + /** Read one more level of nesting; past MAX_NESTING the statement is not read (statement() catches Unread). */ + function nest<T>(read: () => T): T { + if (depth >= MAX_NESTING) throw new Unread(peek(), `nested more than ${MAX_NESTING} levels deep`); + depth++; + try { return read(); } finally { depth--; } + } + + /* ---------------- expressions ---------------- */ + + function parseArgs(close: string): Arg[] { + const args: Arg[] = []; + while (!isOp(peek(), close)) { + let star: Arg['star']; + if (isOp(peek(), '*', '**')) star = next().text as '*' | '**'; + if (peek().kind === 'name' && isOp(peek(1), '=') && !star) { + const name = next().text; next(); + args.push({ name, value: expr() }); + } else { + const v = close === ']' ? subscriptItem() : expr(); + if (isName(peek(), 'where')) { next(); expr(); } + if (isName(peek(), 'for') && args.length === 0 && close === ')') { args.push({ name: null, value: comprehension('gen', v, at(peek())) }); } + else args.push({ name: null, value: v, star }); + } + if (!isOp(peek(), ',')) break; + next(); + } + expectOp(close); + return args; + } + + function subscriptItem(): Expr { + const t = peek(); + const part = (): Expr | null => (isOp(peek(), ':', ']', ',') ? null : expr()); + const lo = part(); + if (!isOp(peek(), ':')) return lo!; + next(); + const hi = part(); + let step: Expr | null = null; + if (isOp(peek(), ':')) { next(); step = part(); } + return { k: 'slice', lo, hi, step, pos: at(t) }; + } + + function comprehension(kind: 'list' | 'set' | 'dict' | 'gen', elt: Expr, pos: Pos): Expr { + next(); // for + const vars = targetList(); + if (!isName(peek(), 'in')) throw new Unread(peek(), "expected 'in'"); + next(); + const iter = orExpr(); + let cond: Expr | null = null; + if (isName(peek(), 'if')) { next(); cond = orExpr(); } + if (isName(peek(), 'for')) throw new Unread(peek(), 'nested comprehension'); + return { k: 'comp', kind, elt, vars, iter, cond, pos }; + } + + function atom(): Expr { + const t = peek(); + if (t.kind === 'number') { next(); return { k: 'num', text: t.text, float: !!t.float, pos: at(t) }; } + if (t.kind === 'string') { + next(); + const isT = !!t.prefix?.includes('t'); + const part = (x: Tok) => (x.prefix?.includes('r') || x.prefix?.includes('t') ? x.text : cook(x.text)); + let value = part(t); + while (peek().kind === 'string') value += part(next()); // adjacent literals concatenate + return { k: 'str', value, t: isT, pos: at(t) }; + } + if (t.kind === 'name') { + if (t.text === 'lambda') return nest(() => { + next(); + const params: string[] = []; + let typed: Param[] | undefined; + let ret: Expr | null = null; + if (isOp(peek(), '(')) { next(); typed = paramList(')'); for (const p of typed) params.push(p.name); if (isOp(peek(), '->')) { next(); ret = orExpr(); } } + else if (isOp(peek(), '->')) { next(); ret = orExpr(); typed = []; } // lambda -> Float64: ... + else while (!isOp(peek(), ':')) { params.push(expectName().text); if (isOp(peek(), ',')) next(); else break; } + const captures = isOp(peek(), '{') ? captureList() : undefined; + expectOp(':'); + return { k: 'lambda', params, body: expr(), pos: at(t), typed, captures, ret } as Expr; + }); + if (t.text === 'ref' && isOp(peek(1), '[')) return nest(() => { + next(); next(); + const origin = parseArgs(']').map((a) => a.value); + return { k: 'opaque', what: 'ref', inner: [...origin, postfix()], pos: at(t) } as Expr; + }); + if ((t.text === 'def' || t.text === 'fn') && isOp(peek(1), '(')) return nest(() => { + next(); next(); + const inner = parseArgs(')').map((a) => a.value); + const params = [...inner]; + const tail = signatureTail(inner); + // a function type: its parameter types, its effects (thin, capturing, raises), its result + return { k: 'opaque', what: 'def', inner, pos: at(t), fn: { params, ret: tail.ret, effects: tail.effects } } as Expr; + }); + next(); + return { k: 'name', id: t.text, pos: at(t), escaped: t.escaped }; + } + if (isOp(t, '...')) { next(); return { k: 'ellipsis', pos: at(t) }; } + if (isOp(t, '.') && peek(1).kind === 'name') { next(); const n = next(); return { k: 'member', name: n.text, pos: at(n) }; } + if (isOp(t, '(', '[', '{')) return nest(() => bracketed(t)); + throw new Unread(t, 'expected an expression'); + } + + /** A bracketed atom: a parenthesized expression or tuple, a list, a dict, a set, a record, a comprehension. */ + function bracketed(t: Tok): Expr { + if (isOp(t, '(')) { + next(); + if (isOp(peek(), ')')) { next(); return { k: 'tuple', items: [], pos: at(t) }; } + const first = starOr(expr); + if (isName(peek(), 'for')) { const c = comprehension('gen', first, at(t)); expectOp(')'); return c; } + if (isOp(peek(), ')')) { next(); return first; } + const items = [first]; + while (isOp(peek(), ',')) { next(); if (isOp(peek(), ')')) break; items.push(starOr(expr)); } + expectOp(')'); + return { k: 'tuple', items, pos: at(t) }; + } + if (isOp(t, '[')) { + next(); + if (isOp(peek(), ']')) { next(); return { k: 'list', items: [], pos: at(t) }; } + const first = starOr(expr); + if (isName(peek(), 'for')) { const c = comprehension('list', first, at(t)); expectOp(']'); return c; } + const items = [first]; + while (isOp(peek(), ',')) { next(); if (isOp(peek(), ']')) break; items.push(starOr(expr)); } + expectOp(']'); + return { k: 'list', items, pos: at(t) }; + } + if (isOp(t, '{')) { + next(); + if (isOp(peek(), '}')) { next(); return { k: 'dict', entries: [], pos: at(t) }; } + if (peek().kind === 'name' && isOp(peek(1), '=')) { + const fields: Array<{ name: string; value: Expr }> = []; + while (!isOp(peek(), '}')) { const n = expectName().text; expectOp('='); fields.push({ name: n, value: expr() }); if (!isOp(peek(), ',')) break; next(); } + expectOp('}'); + return { k: 'record', fields, pos: at(t) }; + } + const key = expr(); + if (isOp(peek(), ':')) { + next(); + const value = expr(); + if (isName(peek(), 'for')) { const c = comprehension('dict', { k: 'tuple', items: [key, value], pos: key.pos }, at(t)); expectOp('}'); return c; } + const entries: Array<[Expr, Expr]> = [[key, value]]; + while (isOp(peek(), ',')) { next(); if (isOp(peek(), '}')) break; const k2 = expr(); expectOp(':'); entries.push([k2, expr()]); } + expectOp('}'); + return { k: 'dict', entries, pos: at(t) }; + } + if (isName(peek(), 'for')) { const c = comprehension('set', key, at(t)); expectOp('}'); return c; } + const items = [key]; + while (isOp(peek(), ',')) { next(); if (isOp(peek(), '}')) break; items.push(expr()); } + expectOp('}'); + return { k: 'set', items, pos: at(t) }; + } + throw new Unread(t, 'expected an expression'); + } + + function starOr(f: () => Expr): Expr { + if (isOp(peek(), '*')) { const t = next(); return { k: 'unary', op: '*', x: f(), pos: at(t) }; } + return f(); + } + + function postfix(): Expr { + let e = atom(); + for (; ;) { + const t = peek(); + if (isOp(t, '(')) { next(); e = { k: 'call', fn: e, args: nest(() => parseArgs(')')), pos: e.pos, lparen: at(t) }; continue; } + if (isOp(t, '[')) { next(); e = { k: 'index', obj: e, args: nest(() => parseArgs(']')), pos: e.pos, lbracket: at(t) }; continue; } + if (isOp(t, '.')) { next(); const n = expectName(); e = { k: 'attr', obj: e, name: n.text, pos: at(n), dot: at(t) }; continue; } + // postfix ^ is a transfer when it ends the operand (the next token cannot start one) + if (isOp(t, '^') && t.col === (tokens[i - 1]?.endCol ?? -1) && endsOperand(peek(1))) { next(); e = { k: 'transfer', x: e, pos: at(t) }; continue; } + return e; + } + } + + function endsOperand(t: Tok): boolean { + return t.kind === 'newline' || t.kind === 'eof' || isOp(t, ')', ']', '}', ',', ':', '=') || isName(t, 'if', 'else', 'for', 'and', 'or'); + } + + function power(): Expr { + const base = postfix(); + if (isOp(peek(), '**')) { const t = next(); return { k: 'binary', op: '**', l: base, r: nest(unary), pos: at(t) }; } + return base; + } + function unary(): Expr { + const t = peek(); + if (isOp(t, '-', '+', '~')) { next(); return { k: 'unary', op: t.text, x: nest(unary), pos: at(t) }; } + return power(); + } + const binaryLevel = (ops: string[], sub: () => Expr) => (): Expr => { + let l = sub(); + while (isOp(peek(), ...ops)) { const t = next(); l = { k: 'binary', op: t.text, l, r: sub(), pos: at(t) }; } + return l; + }; + const mul = binaryLevel(['*', '@', '/', '//', '%'], unary); + const add = binaryLevel(['+', '-'], mul); + const shift = binaryLevel(['<<', '>>'], add); + const band = binaryLevel(['&'], shift); + const bxor = binaryLevel(['^'], band); + const bor = binaryLevel(['|'], bxor); + + function compare(): Expr { + const first = bor(); + const ops: string[] = []; + const opPos: Pos[] = []; + const operands = [first]; + for (; ;) { + const t = peek(); + opPos.push(at(t)); + if (t.kind === 'op' && COMPARE.has(t.text)) { next(); ops.push(t.text); } + else if (isName(t, 'in')) { next(); ops.push('in'); } + else if (isName(t, 'not') && isName(peek(1), 'in')) { next(); next(); ops.push('not in'); } + else if (isName(t, 'is')) { next(); if (isName(peek(), 'not')) { next(); ops.push('is not'); } else ops.push('is'); } + else { opPos.pop(); break; } + operands.push(bor()); + } + return ops.length ? { k: 'compare', ops, operands, pos: first.pos, opPos } : first; + } + function notExpr(): Expr { + const t = peek(); + if (isName(t, 'not')) { next(); return { k: 'not', x: nest(notExpr), pos: at(t) }; } + return compare(); + } + function andExpr(): Expr { let l = notExpr(); while (isName(peek(), 'and')) { const t = next(); l = { k: 'bool', op: 'and', l, r: notExpr(), pos: at(t) }; } return l; } + function orExpr(): Expr { let l = andExpr(); while (isName(peek(), 'or')) { const t = next(); l = { k: 'bool', op: 'or', l, r: andExpr(), pos: at(t) }; } return l; } + function expr(): Expr { + if (peek().kind === 'name' && isOp(peek(1), ':=')) { const n = next(); next(); return { k: 'walrus', name: n.text, value: nest(expr), pos: at(n) }; } + const a = orExpr(); + if (isName(peek(), 'if')) { + const t = next(); + const cond = orExpr(); + if (!isName(peek(), 'else')) throw new Unread(peek(), "expected 'else'"); + next(); + return { k: 'ifexp', cond, a, b: nest(expr), pos: at(t) }; + } + return a; + } + /** `a, b = ...` and `for a, b in ...`: a bare comma list is a tuple. */ + function exprList(): Expr { + const first = starOr(expr); + if (!isOp(peek(), ',')) return first; + const items = [first]; + while (isOp(peek(), ',')) { next(); if (peek().kind === 'newline' || isOp(peek(), '=', ':') || isName(peek(), 'in')) break; items.push(starOr(expr)); } + return { k: 'tuple', items, pos: first.pos }; + } + function targetList(): Expr { + const first = bor(); + if (!isOp(peek(), ',')) return first; + const items = [first]; + while (isOp(peek(), ',')) { next(); if (isName(peek(), 'in')) break; items.push(bor()); } + return { k: 'tuple', items, pos: first.pos }; + } + + /* ---------------- statements ---------------- */ + + function endOfStatement() { + const t = peek(); + if (isOp(t, ';')) { next(); return; } + if (t.kind === 'newline') { next(); return; } + if (t.kind === 'eof' || t.kind === 'dedent') return; + throw new Unread(t, 'expected the end of the statement'); + } + + /** The statements of an indented block, or the one simple statement after the colon. */ + function block(colon: Tok, owner: string): Stmt[] { + if (peek().kind !== 'newline') { + const stmts: Stmt[] = []; + do { stmts.push(simple()); if (isOp(peek(), ';')) next(); else break; } while (peek().kind !== 'newline' && peek().kind !== 'eof'); + if (peek().kind === 'newline') next(); + return stmts; + } + next(); // newline + if (peek().kind !== 'indent') { + // confirmed by godbolt for a def (MQ-102, MQ-111) and an if (MQ-322, a body that is only a + // comment); the compiler's own tests show an empty struct body is not reported this way + if (owner !== 'struct' && owner !== 'trait') finding({ line: colon.line, col: colon.endCol, endCol: colon.endCol + 1, message: "body must not be empty; use 'pass' or check that the lines below are indented", id: 'empty-body' }); + return []; + } + next(); + const body: Stmt[] = []; + while (peek().kind !== 'dedent' && peek().kind !== 'eof') body.push(statement()); + if (peek().kind === 'dedent') next(); + return body; + } + + function decorators(): Expr[] { + const ds: Expr[] = []; + while (isOp(peek(), '@')) { next(); ds.push(postfix()); if (peek().kind === 'newline') next(); } + return ds; + } + + function params(close: string, inDef: boolean): Param[] { + const ps: Param[] = []; + while (!isOp(peek(), close)) { + const t = peek(); + if (isOp(t, '*') && (isOp(peek(1), ',') || isOp(peek(1), close))) { next(); ps.push({ name: '*', convention: null, type: null, default: null, star: 'marker', pos: at(t) }); } + else if (isOp(t, '/') && (isOp(peek(1), ',') || isOp(peek(1), close))) { next(); ps.push({ name: '/', convention: null, type: null, default: null, star: 'positional-marker', pos: at(t) }); } + else if (isOp(t, '//') && (isOp(peek(1), ',') || isOp(peek(1), close))) { next(); ps.push({ name: '//', convention: null, type: null, default: null, star: 'positional-marker', pos: at(t) }); } + else { + let star: Param['star']; + let convention: string | null = null; + const conv = () => { + if ((peek().kind === 'name') && (SOFT_CONVENTIONS.has(peek().text) || peek().text === 'var' || peek().text === 'ref') && (peek(1).kind === 'name' || isOp(peek(1), '*', '**', '['))) { + convention = next().text; + if (convention === 'ref' && isOp(peek(), '[')) { next(); parseArgs(']'); } + } + }; + conv(); + if (isOp(peek(), '*', '**')) star = next().text as '*' | '**'; + if (!convention) conv(); + const n = expectName(); + let type: Expr | null = null; + if (isOp(peek(), ':')) { next(); type = expr(); } + let def: Expr | null = null; + if (isOp(peek(), '=')) { next(); def = expr(); } + ps.push({ name: n.text, convention, type, default: def, star, pos: at(n) }); + } + if (!isOp(peek(), ',')) break; + next(); + } + expectOp(close); + void inDef; + return ps; + } + + /** `{imm x, mut y, var z^}`, `{mut}`, `{mut, var z}`, `{var^}`, `{}`: a closure's or a lambda's captures. */ + function captureList(): Captures { + next(); // { + const captures: Captures = { defaultConv: null, items: [] }; + while (!isOp(peek(), '}') && peek().kind !== 'eof') { + let conv: string | null = null; + if (isName(peek(), 'imm', 'mut', 'var', 'ref', 'read')) conv = next().text; + if (peek().kind === 'name') { + const n = next(); + const move = isOp(peek(), '^'); if (move) next(); + captures.items.push({ name: n.text, conv: conv ?? 'imm', move, pos: at(n) }); + } else if (conv) { captures.defaultConv = conv; if (isOp(peek(), '^')) { next(); captures.defaultConv = `${conv}^`; } } + if (!isOp(peek(), ',')) break; + next(); + } + expectOp('}'); + return captures; + } + + /** After a parameter list: capture lists, effects, raises, the return type, where. */ + function signatureTail(sink: Expr[]): { raises: boolean; ret: Expr | null; raisesType: Expr | null; captures: Captures | null; effects: string[] } { + const effects: string[] = []; + let raises = false; + let ret: Expr | null = null; + let raisesType: Expr | null = null; + let captures: Captures | null = null; + for (; ;) { + if (isOp(peek(), '{')) { captures = captureList(); continue; } + if (isName(peek(), 'raises')) { next(); raises = true; effects.push('raises'); if (!isOp(peek(), '->', ':', ',', ')', ']') && !isName(peek(), 'where') && peek().kind !== 'newline') { raisesType = postfix(); sink.push(raisesType); } continue; } + if (isName(peek(), 'capturing', 'escaping', 'thin', 'register_passable')) { effects.push(next().text); if (isOp(peek(), '[')) { next(); parseArgs(']'); } continue; } + if (isName(peek(), 'abi')) { next(); expectOp('('); parseArgs(')'); continue; } + if (isOp(peek(), '->')) { next(); ret = orExpr(); sink.push(ret); continue; } + if (isName(peek(), 'where')) { next(); orExpr(); continue; } + return { raises, ret, raisesType, captures, effects }; + } + } + + function paramList(close: string): Param[] { return params(close, false); } + + function def(decos: Expr[], start: Tok): Stmt { + next(); // def + const n = expectName(); + let typeParams: Param[] = []; + if (isOp(peek(), '[')) { next(); typeParams = params(']', false); } + expectOp('('); + const ps = params(')', true); + const { raises, ret, raisesType, captures } = signatureTail([]); + if (peek().kind === 'newline' || peek().kind === 'eof') { + // a signature with no colon: an indented body below it makes the mistake certain + const end = tokens[i - 1]; + endOfStatement(); + // a signature may go on, indented, on the next line: ` where cond:`, ` raises -> T:` + if (peek().kind === 'indent' && (isName(peek(1), 'where', 'raises', 'capturing', 'escaping', 'thin') || isOp(peek(1), '->'))) { + next(); + const tail = signatureTail([]); + const colon = expectOp(':'); + // the body sits at the continuation's own indentation, or deeper + let body: Stmt[]; + if (peek().kind === 'newline' && peek(1).kind !== 'indent') { + next(); + body = []; + while (peek().kind !== 'dedent' && peek().kind !== 'eof') body.push(statement()); + } else body = block(colon, 'def'); + while (peek().kind === 'newline') next(); + if (peek().kind === 'dedent') next(); + return { k: 'def', name: n.text, params: ps, ret: tail.ret ?? ret, raises: raises || tail.raises, typeParams, decorators: decos, body, pos: at(n), colon: at(colon) }; + } + if (peek().kind === 'indent') { + // certainly a mistake, but the compiler words it two ways ("missing ':' at end of function + // signature", "expected ':' in function definition") depending on what Programming+ cannot + // see; say nothing rather than guess, and read the body so its code is not called stray + void end; + next(); + const body: Stmt[] = []; + while (peek().kind !== 'dedent' && peek().kind !== 'eof') body.push(statement()); + if (peek().kind === 'dedent') next(); + return { k: 'def', name: n.text, params: ps, ret, raises, typeParams, decorators: decos, body, pos: at(n), colon: at(end) }; + } + return { k: 'def', name: n.text, params: ps, ret, raises, typeParams, decorators: decos, body: [], pos: at(n), colon: at(n) }; + } + const colon = expectOp(':'); + const body = block(colon, 'def'); + void start; + return { k: 'def', name: n.text, params: ps, ret, raises, typeParams, decorators: decos, body, pos: at(n), colon: at(colon), raisesType, captures }; + } + + function structOrTrait(decos: Expr[]): Stmt { + const kw = next(); + const n = expectName(); + let typeParams: Param[] = []; + if (isOp(peek(), '[')) { next(); typeParams = params(']', false); } + let bases: Expr[] = []; + if (isOp(peek(), '(')) { next(); bases = parseArgs(')').map((a) => a.value); } + if (isName(peek(), 'where')) { next(); expr(); } + const colon = expectOp(':'); + const body = block(colon, kw.text); + return { k: kw.text as 'struct' | 'trait', name: n.text, bases, typeParams, decorators: decos, body, pos: at(n), colon: at(colon) }; + } + + function simple(): Stmt { + const t = peek(); + if (isName(t, 'pass')) { next(); return { k: 'pass', pos: at(t) }; } + if (isName(t, 'break')) { next(); return { k: 'break', pos: at(t) }; } + if (isName(t, 'continue')) { next(); return { k: 'continue', pos: at(t) }; } + if (isName(t, 'return')) { next(); const v = peek().kind === 'newline' || isOp(peek(), ';') || peek().kind === 'eof' ? null : exprList(); return { k: 'return', value: v, pos: at(t) }; } + if (isName(t, 'raise')) { next(); const v = peek().kind === 'newline' || peek().kind === 'eof' ? null : expr(); if (isName(peek(), 'from')) { next(); expr(); } return { k: 'raise', value: v, pos: at(t) }; } + if (isName(t, 'global', 'nonlocal')) { next(); const names = [expectName().text]; while (isOp(peek(), ',')) { next(); names.push(expectName().text); } return { k: 'global', names, pos: at(t) }; } + if (isName(t, 'assert') || (isName(t, 'comptime') && isName(peek(1), 'assert'))) { + const comptime = t.text === 'comptime'; if (comptime) next(); next(); + const test = expr(); let msg: Expr | null = null; + if (isOp(peek(), ',')) { next(); msg = expr(); } + return { k: 'assert', test, msg, comptime, pos: at(t) }; + } + if (isName(t, 'import')) { + next(); + let mod = expectName().text; while (isOp(peek(), '.')) { next(); mod += '.' + expectName().text; } + let alias: string | null = null; if (isName(peek(), 'as')) { next(); alias = expectName().text; } + return { k: 'import', module: mod, alias, pos: at(t) }; + } + if (isName(t, 'from')) { + next(); + let mod = ''; + while (isOp(peek(), '.')) { next(); mod += '.'; } + if (peek().kind === 'name' && !isName(peek(), 'import')) { mod += expectName().text; while (isOp(peek(), '.')) { next(); mod += '.' + expectName().text; } } + if (!isName(peek(), 'import')) throw new Unread(peek(), "expected 'import'"); + next(); + if (isOp(peek(), '*')) { next(); return { k: 'from', module: mod, names: [], star: true, pos: at(t) }; } + const paren = isOp(peek(), '('); if (paren) next(); + const names: Array<{ name: string; alias: string | null; pos: Pos }> = []; + for (; ;) { + if (paren && isOp(peek(), ')')) break; + const n = expectName(); let alias: string | null = null; + if (isName(peek(), 'as')) { next(); alias = expectName().text; } + names.push({ name: n.text, alias, pos: at(n) }); + if (!isOp(peek(), ',')) break; next(); + } + if (paren) expectOp(')'); + return { k: 'from', module: mod, names, star: false, pos: at(t) }; + } + if (isName(t, 'var', 'ref') || (isName(t, 'comptime') && !isName(peek(1), 'if', 'for', 'assert'))) { + next(); + // `comptime Name[size: Int] = ...`: a parametric alias, whose brackets declare parameters + let typeParams: Param[] | undefined; + let targets: Expr; + if (t.text === 'comptime' && peek().kind === 'name' && isOp(peek(1), '[')) { const n = next(); next(); typeParams = params(']', false); targets = { k: 'name', id: n.text, pos: at(n) }; } + else targets = isOp(peek(), '(') ? atom() : targetList(); + let type: Expr | null = null; let value: Expr | null = null; + if (isOp(peek(), ':')) { next(); type = expr(); } + if (isOp(peek(), '=')) { next(); value = exprList(); } + return { k: 'var', targets, type, value, comptime: t.text === 'comptime', ref: t.text === 'ref', pos: at(t), typeParams }; + } + const e = exprList(); + if (isOp(peek(), '=')) { + const targets = [e]; + let value: Expr = e; + while (isOp(peek(), '=')) { next(); value = exprList(); targets.push(value); } + targets.pop(); + return { k: 'assign', targets, value, pos: e.pos }; + } + if (peek().kind === 'op' && AUG.has(peek().text)) { const op = next().text; return { k: 'aug', target: e, op, value: exprList(), pos: e.pos }; } + if (isOp(peek(), ':') && !isOp(peek(1), '=')) throw new Unread(peek(), 'annotated assignment without var'); + return { k: 'expr', e, pos: e.pos }; + } + + function compound(): Stmt | null { + const t = peek(); + const comptime = isName(t, 'comptime') && isName(peek(1), 'if', 'for'); + if (comptime) next(); + const h = peek(); + if (isName(h, 'if')) { + next(); const test = expr(); const colon = expectOp(':'); + const body = block(colon, 'if'); + let orelse: Stmt[] = []; + if (isName(peek(), 'elif')) { const e = compoundElif(comptime); orelse = [e]; } + else if (isName(peek(), 'else')) { next(); orelse = block(expectOp(':'), 'else'); } + return { k: 'if', test, body, orelse, comptime, pos: at(t) }; + } + if (isName(h, 'while')) { + next(); const test = expr(); const body = block(expectOp(':'), 'while'); + let orelse: Stmt[] = []; if (isName(peek(), 'else')) { next(); orelse = block(expectOp(':'), 'else'); } + return { k: 'while', test, body, orelse, pos: at(t) }; + } + if (isName(h, 'for')) { + next(); + let convention: string | null = null; + if (isName(peek(), 'ref', 'var') && peek(1).kind === 'name' && !isName(peek(1), 'in')) convention = next().text; // for ref r in ..., for var x in ... + const target = targetList(); + if (!isName(peek(), 'in')) throw new Unread(peek(), "expected 'in'"); + next(); const iter = exprList(); const body = block(expectOp(':'), 'for'); + let orelse: Stmt[] = []; if (isName(peek(), 'else')) { next(); orelse = block(expectOp(':'), 'else'); } + return { k: 'for', target, iter, body, orelse, comptime, pos: at(t), convention }; + } + if (comptime) throw new Unread(h, 'comptime'); + if (isName(h, 'match') && peek(1).kind === 'name' && isOp(peek(2), ':')) return matchStatement(); + if (isName(h, 'try')) { + next(); const body = block(expectOp(':'), 'try'); + const handlers: Array<{ name: string | null; type: Expr | null; body: Stmt[]; pos: Pos }> = []; + let orelse: Stmt[] = [], finalbody: Stmt[] = []; + while (isName(peek(), 'except')) { + const ex = next(); let type: Expr | null = null; let name: string | null = null; + if (!isOp(peek(), ':')) { + if (peek().kind === 'name' && isOp(peek(1), ':')) name = next().text; + else { type = expr(); if (isName(peek(), 'as')) { next(); name = expectName().text; } } + } + handlers.push({ name, type, body: block(expectOp(':'), 'except'), pos: at(ex) }); + } + if (isName(peek(), 'else')) { next(); orelse = block(expectOp(':'), 'else'); } + if (isName(peek(), 'finally')) { next(); finalbody = block(expectOp(':'), 'finally'); } + return { k: 'try', body, handlers, orelse, finalbody, pos: at(t) }; + } + if (isName(h, 'with')) { + next(); const items: Array<{ e: Expr; as: Expr | null }> = []; + for (; ;) { const e = expr(); let as: Expr | null = null; if (isName(peek(), 'as')) { next(); as = bor(); } items.push({ e, as }); if (!isOp(peek(), ',')) break; next(); } + return { k: 'with', items, body: block(expectOp(':'), 'with'), pos: at(t) }; + } + if (isOp(h, '@') || isName(h, 'def', 'fn', 'struct', 'trait')) { + const decos = decorators(); + if (isName(peek(), 'def', 'fn')) return def(decos, peek()); + if (isName(peek(), 'struct', 'trait')) return structOrTrait(decos); + if (isName(peek(), 'comptime', 'var')) { const s = simple(); endOfStatement(); return s; } + throw new Unread(peek(), 'expected def or struct after a decorator'); + } + return null; + } + + /** + * `match name:` with `case` arms whose patterns are literals (a number or a text), alternatives of them joined by + * `|`, `_`, or a capture name, each with an optional `if` guard: read as the if/elif/else chain it runs as, so every + * check and the interpreter follow it as they follow an if (probes/match-statement, match-literal-capture-guard). + * The first arm whose pattern matches and whose guard holds runs, and none when nothing matches. An arm is an `if` + * at its `case` line: `subject == literal or ...`, or the guard; a capture is a `var` of that name holding the + * subject, its guard reading the subject in its place; an arm that always matches is the `else`, and the arms after + * it, which never run, are left out. Any other subject or pattern (an Optional's `.Some(x)`, a list, `-1`, which + * Mojo 1.2.0 rejects) is not read. + */ + function matchStatement(): Stmt { + const startAt = i; + try { return matchArms(); } + catch (e) { + // what is not read is the whole match, not the rest of the arm it stopped in + if (e instanceof Unread) i = startAt + 1; + throw e; + } + } + + function matchArms(): Stmt { + next(); + const subjTok = next(); + const subject = (): Expr => ({ k: 'name', id: subjTok.text, pos: at(subjTok) }); + expectOp(':'); + if (peek().kind !== 'newline') throw new Unread(peek(), 'a match on one line'); + next(); + // the arms may sit one level in, or at the match's own level (the compiler's own tests: mojo-parser/statements/match) + const indented = peek().kind === 'indent'; + if (indented) next(); + else if (!isName(peek(), 'case')) throw new Unread(peek(), 'a match with no case'); + const more = () => (indented ? peek().kind !== 'dedent' && peek().kind !== 'eof' : isName(peek(), 'case')); + type Arm = { test: Expr | null; prelude: Stmt[]; body: Stmt[]; pos: Pos }; + const arms: Arm[] = []; + while (more()) { + const ct = peek(); + if (!isName(ct, 'case')) throw new Unread(ct, "expected 'case' in a match"); + next(); + const literals: Expr[] = []; + let capture: Tok | null = null, wildcard = false; + for (; ;) { + const p = peek(); + if (p.kind === 'number' || p.kind === 'string') literals.push(atom()); + else if (p.kind === 'name' && p.text === '_') { next(); wildcard = true; } + else if (p.kind === 'name' && !MOJO_KEYWORDS.includes(p.text) && !['True', 'False', 'None'].includes(p.text)) { next(); capture = p; } + else throw new Unread(p, 'this match pattern'); + if (!isOp(peek(), '|')) break; + next(); + } + if ((wildcard || capture) && (literals.length || (wildcard && capture))) throw new Unread(ct, 'a match pattern that mixes a name or _ with others'); + let guard: Expr | null = null; + if (isName(peek(), 'if')) { next(); guard = expr(); } + const body = block(expectOp(':'), 'case'); + const pos = at(ct); + const prelude: Stmt[] = []; + if (capture) { + const cap = capture.text; + // only where the arm reads it: a binding the compiler never warns about is not a variable the checks can call unused + if (JSON.stringify(body).includes(`"k":"name","id":${JSON.stringify(cap)}`)) prelude.push({ k: 'var', targets: { k: 'name', id: cap, pos: at(capture) }, type: null, value: subject(), comptime: false, ref: false, pos }); + if (guard) guard = renamed(guard, cap, subjTok.text); + } + let test: Expr | null = guard; + if (literals.length) { + const eq = (lit: Expr): Expr => ({ k: 'compare', ops: ['=='], operands: [subject(), lit], pos, opPos: [lit.pos] }); + let any = eq(literals[0]); + for (const lit of literals.slice(1)) any = { k: 'bool', op: 'or', l: any, r: eq(lit), pos }; + test = guard ? { k: 'bool', op: 'and', l: any, r: guard, pos } : any; + } + arms.push({ test, prelude, body, pos }); + if (!test) break; + } + // the arms after one that always matches never run + while (more()) statementSkip(); + if (indented && peek().kind === 'dedent') next(); + let chain: Stmt[] = []; + for (let k = arms.length - 1; k >= 0; k--) { + const a = arms[k]; + const body = [...a.prelude, ...a.body]; + chain = a.test ? [{ k: 'if', test: a.test, body, orelse: chain, comptime: false, pos: a.pos }] : body; + } + return chain.length === 1 && chain[0].k === 'if' ? chain[0] : { k: 'if', test: { k: 'name', id: 'True', pos: at(subjTok) }, body: chain, orelse: [], comptime: false, pos: at(subjTok) }; + } + + /** A case arm after one that always matches: its tokens are passed over, block and all. */ + function statementSkip(): void { + let depth = 0; + for (; ;) { + const t = next(); + if (t.kind === 'eof') return; + if (t.kind === 'indent') depth++; + if (t.kind === 'dedent') { depth--; if (depth <= 0) return; } + if (t.kind === 'newline' && depth === 0 && peek().kind !== 'indent') return; + } + } + + /** The elif arms after an if, each the else of the one before; read in a loop, so a long chain needs no stack. */ + function compoundElif(comptime: boolean): Stmt { + let first: Stmt | null = null; + let last: Extract<Stmt, { k: 'if' }> | null = null; + for (; ;) { + const t = next(); // elif + const test = expr(); const body = block(expectOp(':'), 'elif'); + const arm: Extract<Stmt, { k: 'if' }> = { k: 'if', test, body, orelse: [], comptime, pos: at(t) }; + if (last) last.orelse = [arm]; else first = arm; + last = arm; + if (isName(peek(), 'elif')) continue; + if (isName(peek(), 'else')) { next(); last.orelse = block(expectOp(':'), 'else'); } + return first!; + } + } + + function statement(): Stmt { + const start = i; + try { + // a statement inside a block is one level deeper than the block's own line + return nest(() => { + const c = compound(); + if (c) return c; + const s = simple(); + endOfStatement(); + return s; + }); + } catch (caught) { + // a RangeError is the engine's stack running out on a shape the limits above do not foresee: the same refusal + const e = caught instanceof RangeError ? new Unread(tokens[start] ?? peek(), `nested more than ${MAX_NESTING} levels deep`) : caught; + if (!(e instanceof Unread)) throw e; + // skip to the end of this logical line, and past any block that hangs off it + unread++; + const tok = tokens[start]; + const deep = e.why.startsWith('nested more than'); + if (deep && !deepSaid) { deepSaid = true; findings.push({ severity: 'note', line: e.tok.line, col: e.tok.col, endCol: e.tok.endCol, message: tooDeep(`nested more than ${MAX_NESTING} levels deep`), id: 'unread', why: `${e.why} at ${JSON.stringify(e.tok.text || e.tok.kind)}` }); } + else if (unread === 1 && !deep) findings.push({ severity: 'note', line: e.tok.line, col: e.tok.col, endCol: e.tok.endCol, message: 'Programming+ cannot read this yet; Compile & run asks the real compiler', id: 'unread', why: `${e.why} at ${JSON.stringify(e.tok.text || e.tok.kind)}` }); + i = Math.max(i, start + 1); + while (peek().kind !== 'newline' && peek().kind !== 'eof') next(); + if (peek().kind === 'newline') next(); + if (peek().kind === 'indent') { let d = 0; do { const x = next(); if (x.kind === 'indent') d++; else if (x.kind === 'dedent') d--; } while (d > 0 && peek().kind !== 'eof'); } + return { k: 'unread', pos: at(tok) }; + } + } + + const body: Stmt[] = []; + while (peek().kind !== 'eof') { + if (peek().kind === 'newline' || peek().kind === 'dedent') { next(); continue; } + if (peek().kind === 'indent') { + // indented code with nothing above to hang off: the compiler reads the block that + // should hold it as empty, and this as stray; Programming+ just skips it + next(); continue; + } + body.push(statement()); + } + // a chain such as a + b + c ... reads without nesting, but makes a tree as tall as it is long + const tall = boundTree(body); + if (tall.replaced) { + unread += tall.replaced; + if (!deepSaid && tall.at) findings.push({ severity: 'note', line: tall.at.line, col: tall.at.col, endCol: tall.at.endCol, message: tooDeep(`more than ${MAX_TREE} levels deep once read (each operand of a chain such as a + b + c, and each elif, is one level)`), id: 'unread', why: 'a syntax tree taller than MAX_TREE' }); + } + return { body, findings, unread }; +} + +/** The positions a node carries, which are not part of its tree. */ +/** A copy of `e` with every name `from` read as `to` (a match guard reads the subject where its capture is named). */ +function renamed(e: Expr, from: string, to: string): Expr { + const walk = (x: any): any => { + if (Array.isArray(x)) return x.map(walk); + if (!x || typeof x !== 'object') return x; + if (x.k === 'name' && x.id === from) return { ...x, id: to }; + if (x.k === 'lambda') return x; + const out: any = {}; + for (const [key, v] of Object.entries(x)) out[key] = POSITIONS.has(key) ? v : walk(v); + return out; + }; + return walk(e); +} + +const POSITIONS = new Set(['pos', 'lparen', 'lbracket', 'dot', 'opPos', 'colon']); +const STATEMENTS = new Set<string>(['expr', 'var', 'assign', 'aug', 'if', 'while', 'for', 'break', 'continue', 'pass', 'return', 'raise', 'try', 'with', 'assert', 'def', 'struct', 'trait', 'import', 'from', 'global']); + +/** + * Every statement whose syntax tree is more than MAX_TREE nodes tall, innermost first, becomes an unread statement, + * so no later stage walks such a tree. The walk keeps its own stack, as it has to on exactly these trees. Returns + * how many statements were replaced, and the position of the first node past the limit. + */ +function boundTree(body: Stmt[]): { replaced: number; at: Pos | null } { + interface Frame { holder: any; key: string | number; value: any; kids: Array<[any, string | number]>; next: number; h: number } + const kidsOf = (v: any): Array<[any, string | number]> => { + const out: Array<[any, string | number]> = []; + if (Array.isArray(v)) { for (let k = 0; k < v.length; k++) if (v[k] && typeof v[k] === 'object') out.push([v, k]); } + else for (const k of Object.keys(v)) if (!POSITIONS.has(k) && v[k] && typeof v[k] === 'object') out.push([v, k]); + return out; + }; + let replaced = 0; + let first: Pos | null = null; + const stack: Frame[] = [{ holder: null, key: 0, value: body, kids: kidsOf(body), next: 0, h: 0 }]; + while (stack.length) { + const f = stack[stack.length - 1]; + if (f.next < f.kids.length) { + const [holder, key] = f.kids[f.next++]; + const value = holder[key]; + stack.push({ holder, key, value, kids: kidsOf(value), next: 0, h: 0 }); + continue; + } + stack.pop(); + const node = !Array.isArray(f.value) && typeof f.value.k === 'string'; + let h = f.h + (node ? 1 : 0); + if (node && h > MAX_TREE) { + first ??= f.value.pos ?? null; + if (STATEMENTS.has(f.value.k)) { f.holder[f.key] = { k: 'unread', pos: f.value.pos }; replaced++; h = 1; } + } + const parent = stack[stack.length - 1]; + if (parent && h > parent.h) parent.h = h; + } + return { replaced, at: first }; +} diff --git a/src/mojo/practices.ts b/src/mojo/practices.ts new file mode 100644 index 0000000..087af90 --- /dev/null +++ b/src/mojo/practices.ts @@ -0,0 +1,679 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The best-practices helper (restructure plan, section 2.6): the gap between the literal Mojo the translator writes + * and the Mojo a person would write, as things to learn from. DOM-free, and never imported by the translator (a + * grep gate keeps it so): the Bridge's Mojo stays the literal translation whether the helper is on or off. + * + * A rule reads the parsed module and the source and returns findings. A finding says where, why, shows a before + * and an after, links the manual, and carries a one-click fix only when the rewrite is safe: it is verified by the + * tests to pass the checker and to print what the original printed. A rule that cannot promise that explains and + * leaves the editing to the student. + */ +import type { QuickFix, Diagnostic } from '../common/types.js'; +import { parseModule, type Expr, type Module, type Stmt } from './parse.js'; +import { checkMojo } from './check.js'; +import { MOJO_PRELUDE_RAISES } from './generated/stdlib.js'; + +const MANUAL = 'https://mojolang.org/docs/manual'; + +export const PRACTICES_ORIGIN = 'best practices'; + +export interface PracticeFinding { + rule: string; + /** The first Mojo line it is about (1-based). */ + line: number; + /** The last line, when it spans several. */ + endLine?: number; + /** The reason, in a sentence or two. */ + message: string; + before: string; + after: string; + docUrl: string; + /** One click, when the rewrite is safe. It replaces whole lines. */ + fix?: QuickFix; + /** In the Bridge: how to write the pseudo or lingo so the translator writes the better form, where that is true. */ + pseudoHint?: string; +} + +export interface PracticeRule { + id: string; + title: string; + why: string; + docUrl: string; + /** A whole program in the literal form, and the same program as the rule would have it. Both run, and print the same. */ + example: { before: string; after: string; /** The compiler refuses the "before" with this message (the rule is about something it forbids); the "after" must pass. */ beforeRefused?: string }; + /** Standard-library methods the rewrite names, as [type, method]: each is checked against the harvested index. */ + apis: Array<[string, string]>; + detect(mod: Module, source: string): PracticeFinding[]; +} + +/* ---------- reading the tree ---------- */ + +type Node = { [key: string]: unknown }; +const STMT_KINDS = new Set(['expr', 'var', 'assign', 'aug', 'if', 'while', 'for', 'break', 'continue', 'pass', 'return', 'raise', 'try', 'with', 'assert', 'def', 'struct', 'trait', 'import', 'from', 'global', 'unread']); +const isNode = (x: unknown): x is Node => typeof x === 'object' && x !== null && typeof (x as Node).k === 'string'; +const isStmt = (x: unknown): x is Stmt => isNode(x) && STMT_KINDS.has(x.k as string); + +function walk(x: unknown, fn: (n: Node) => void): void { + if (Array.isArray(x)) { for (const y of x) walk(y, fn); return; } + if (typeof x !== 'object' || x === null) return; + if (isNode(x)) fn(x); + for (const v of Object.values(x)) if (typeof v === 'object' && v !== null) walk(v, fn); +} +const mentions = (x: unknown, id: string): boolean => { let hit = false; walk(x, (n) => { if (n.k === 'name' && n.id === id) hit = true; }); return hit; }; +const countMentions = (x: unknown, id: string): number => { let c = 0; walk(x, (n) => { if (n.k === 'name' && n.id === id) c++; }); return c; }; + +/** Every statement list in the module: the module's own, and each body inside it. */ +function blocksOf(mod: Module): Stmt[][] { + const out: Stmt[][] = []; + const visit = (x: unknown): void => { + if (Array.isArray(x)) { + if (x.length && x.every(isStmt)) out.push(x as Stmt[]); + for (const y of x) visit(y); + return; + } + if (typeof x !== 'object' || x === null) return; + for (const v of Object.values(x)) if (typeof v === 'object' && v !== null) visit(v); + }; + out.push(mod.body); + visit(mod.body); + return out; +} + +/** The def that contains a statement, with its nearest enclosing block. */ +function defsOf(mod: Module): Array<Extract<Stmt, { k: 'def' }>> { + const out: Array<Extract<Stmt, { k: 'def' }>> = []; + walk(mod.body, (n) => { if (n.k === 'def') out.push(n as unknown as Extract<Stmt, { k: 'def' }>); }); + return out; +} + +const lines = (source: string): string[] => source.split('\n'); +const indentOf = (l: string): string => /^\s*/.exec(l)![0]; +const balanced = (l: string): boolean => { + let d = 0, q: string | null = null; + for (let i = 0; i < l.length; i++) { + const c = l[i]; + if (q) { if (c === '\\') i++; else if (c === q) q = null; } + else if (c === '"' || c === "'") q = c; + else if (c === '#') break; + else if ('([{'.includes(c)) d++; + else if (')]}'.includes(c)) d--; + } + return d === 0 && !q; +}; +const stripComment = (l: string): string => { + let q: string | null = null; + for (let i = 0; i < l.length; i++) { const c = l[i]; if (q) { if (c === '\\') i++; else if (c === q) q = null; } else if (c === '"' || c === "'") q = c; else if (c === '#') return l.slice(0, i).trimEnd(); } + return l; +}; +const lastLineOf = (x: unknown): number => { let m = 0; walk(x, (n) => { const p = n.pos as { line?: number } | undefined; if (p?.line && p.line > m) m = p.line; }); return m; }; +const wholeLines = (line: number, endLine: number, text: string): QuickFix => ({ label: '', line, endLine, col: 1, endCol: null, text }); +const escapeRe = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +/* ---------- 1. declare, then assign ---------- */ + +const declareThenAssign: PracticeRule = { + id: 'declare-then-assign', + title: 'Declare and set a variable in one line', + why: 'Mojo lets `var n = …` declare a variable and give it its first value together, and the type follows from the value. Declaring first and assigning later leaves a stretch where the name exists but holds nothing.', + docUrl: `${MANUAL}/variables/`, + example: { + before: 'def main():\n var n: Int\n n = 5\n print(n)\n', + after: 'def main():\n var n = 5\n print(n)\n', + }, + apis: [], + detect(mod, source) { + const src = lines(source); + const out: PracticeFinding[] = []; + for (const block of blocksOf(mod)) { + // where each name is mentioned in this block, by statement, found once: scanning forward from every + // declaration made a long block (a struct of many fields) quadratic + let at: Map<string, number[]> | null = null; + const mentionedAt = (): Map<string, number[]> => { + if (at) return at; + const m = new Map<string, number[]>(); + block.forEach((t, j) => walk(t, (n) => { if (n.k === 'name') { const a = m.get(n.id as string) ?? []; if (a[a.length - 1] !== j) a.push(j); m.set(n.id as string, a); } })); + return (at = m); + }; + block.forEach((d, i) => { + if (d.k !== 'var' || d.comptime || d.ref || d.value || d.targets.k !== 'name') return; + const id = d.targets.id; + for (const j of mentionedAt().get(id) ?? []) { + if (j <= i) continue; + const t = block[j]; + if (t.k === 'assign' && t.targets.length === 1 && t.targets[0].k === 'name' && t.targets[0].id === id && !mentions(t.value, id)) { + const dl = src[d.pos.line - 1] ?? '', al = src[t.pos.line - 1] ?? ''; + const dm = new RegExp(`^(\\s*)var\\s+${escapeRe(id)}\\s*(?::\\s*(.+?))?\\s*(#.*)?$`).exec(dl); + const am = new RegExp(`^(\\s*)${escapeRe(id)}\\s*=\\s*(.+?)\\s*$`).exec(al); + if (!dm || !am || !balanced(al) || (d.type && !dm[2])) return; + const ty = d.type ? dm[2] : null; + const v = t.value; + const inferable = !ty || (ty === 'Int' && ((v.k === 'num' && !v.float) || (v.k === 'call' && ((v.fn.k === 'attr' && v.fn.name === 'count_codepoints') || (v.fn.k === 'name' && v.fn.id === 'len'))))) + || (ty === 'Float64' && v.k === 'num' && v.float) || (ty === 'String' && v.k === 'str' && !v.t) || (ty === 'Bool' && v.k === 'name' && (v.id === 'True' || v.id === 'False')); + const rhs = stripComment(am[2]); + const nl = `${am[1]}var ${id}${ty && !inferable ? `: ${ty}` : ''} = ${rhs}`; + const between = src.slice(d.pos.line, t.pos.line - 1); + const text = [...between, nl].join('\n'); + out.push({ + rule: 'declare-then-assign', line: d.pos.line, endLine: t.pos.line, + message: `\`${id}\` is declared on line ${d.pos.line} and first given a value on line ${t.pos.line}. Write \`var ${id} = …\` where it gets its value.`, + before: `${dl.trim()}\n${al.trim()}`, after: nl.trim(), + docUrl: this.docUrl, + fix: { ...wholeLines(d.pos.line, t.pos.line, text), label: `Declare ${id} where it is set` }, + }); + } + return; + } + }); + } + return out; + }, +}; + +/* ---------- 2. x = x + 1 -> x += 1 ---------- */ + +const augmentedAssign: PracticeRule = { + id: 'augmented-assign', + title: 'Use += for a running total', + why: '`x += 1` says "change x by 1" in one step and names `x` once, so there is no way to mistype the second one. It is the same operation as `x = x + 1`.', + docUrl: `${MANUAL}/operators/`, + example: { + before: 'def main():\n var total = 0\n for i in range(4):\n total = total + i\n print(total)\n', + after: 'def main():\n var total = 0\n for i in range(4):\n total += i\n print(total)\n', + }, + apis: [], + detect(mod, source) { + const src = lines(source); + const out: PracticeFinding[] = []; + walk(mod.body, (n) => { + const s = n as unknown as Stmt; + if (s.k !== 'assign' || s.targets.length !== 1 || s.targets[0].k !== 'name') return; + const v = s.value; + if (v.k !== 'binary' || !['+', '-', '*'].includes(v.op) || v.l.k !== 'name' || v.l.id !== s.targets[0].id) return; + const id = s.targets[0].id; + const l = src[s.pos.line - 1] ?? ''; + const m = new RegExp(`^(\\s*)${escapeRe(id)}\\s*=\\s*${escapeRe(id)}\\s*\\${v.op}\\s*(.+?)\\s*$`).exec(l); + if (!m || !balanced(l)) return; + const nl = `${m[1]}${id} ${v.op}= ${stripComment(m[2])}`; + out.push({ + rule: 'augmented-assign', line: s.pos.line, + message: `\`${id} = ${id} ${v.op} …\` can be \`${id} ${v.op}= …\`.`, + before: l.trim(), after: nl.trim(), docUrl: this.docUrl, + fix: { ...wholeLines(s.pos.line, s.pos.line, nl), label: `Write ${id} ${v.op}= …` }, + }); + }); + return out; + }, +}; + +/* ---------- 3. an index loop that only reads codepoints ---------- */ + +const codepointLoop: PracticeRule = { + id: 'codepoint-loop', + title: 'Loop over the letters, not their positions', + why: 'When the loop index is only used to read one letter of a string, `for cp in text.codepoint_slices()` gives the letters directly. There is no count to keep in step with the string and no index to get wrong.', + docUrl: `${MANUAL}/control-flow/`, + example: { + before: 'def count_a(t: String) -> Int:\n var n = t.count_codepoints()\n var total = 0\n for i in range(n):\n if String(t[codepoint=i]) == "A":\n total += 1\n return total\n\n\ndef main():\n print(count_a("GATTACA"))\n', + after: 'def count_a(t: String) -> Int:\n var n = t.count_codepoints()\n var total = 0\n for cp in t.codepoint_slices():\n if String(cp) == "A":\n total += 1\n return total\n\n\ndef main():\n print(count_a("GATTACA"))\n', + }, + apis: [['String', 'codepoint_slices'], ['String', 'count_codepoints']], + detect(mod, source) { + const src = lines(source); + const out: PracticeFinding[] = []; + const taken = new Set<string>(); + walk(mod.body, (n) => { if (n.k === 'name') taken.add(n.id as string); }); + for (const d of defsOf(mod)) { + // names set to `X.count_codepoints()` in this function, and never to anything else + const counts = new Map<string, string>(); + const other = new Set<string>(); + walk(d.body, (n) => { + const s = n as unknown as Stmt; + const tgt = s.k === 'var' && s.value ? s.targets : s.k === 'assign' && s.targets.length === 1 ? s.targets[0] : null; + const val = s.k === 'var' ? s.value : s.k === 'assign' ? s.value : null; + if (!tgt || tgt.k !== 'name' || !val) return; + if (val.k === 'call' && val.fn.k === 'attr' && val.fn.name === 'count_codepoints' && val.args.length === 0 && val.fn.obj.k === 'name') { + if (counts.has(tgt.id) && counts.get(tgt.id) !== val.fn.obj.id) other.add(tgt.id); + counts.set(tgt.id, val.fn.obj.id); + } else other.add(tgt.id); + }); + walk(d.body, (n) => { + const f = n as unknown as Stmt; + if (f.k !== 'for' || f.comptime || f.orelse.length || f.target.k !== 'name') return; + const it = f.iter; + if (it.k !== 'call' || it.fn.k !== 'name' || it.fn.id !== 'range' || it.args.length !== 1 || it.args[0].name) return; + const a = it.args[0].value; + let text: string | null = null; + if (a.k === 'call' && a.fn.k === 'attr' && a.fn.name === 'count_codepoints' && a.fn.obj.k === 'name') text = a.fn.obj.id; + else if (a.k === 'name' && counts.has(a.id) && !other.has(a.id) && !mentions(f.body, a.id)) text = counts.get(a.id)!; + if (!text) return; + const i = f.target.id, X = text; + // every use of the index is `X[codepoint=i]`, inside String(…) + let uses = 0, ok = true; + const asIndex = (e: Expr): boolean => e.k === 'index' && e.obj.k === 'name' && e.obj.id === X && e.args.length === 1 && e.args[0].name === 'codepoint' && e.args[0].value.k === 'name' && e.args[0].value.id === i; + walk(f.body, (m) => { + if (m.k === 'call') { + const c = m as unknown as Extract<Expr, { k: 'call' }>; + if (c.fn.k === 'name' && c.fn.id === 'String' && c.args.length === 1 && !c.args[0].name && asIndex(c.args[0].value)) uses++; + } + }); + const idx = countMentions(f.body, i); + if (uses === 0 || idx !== uses) ok = false; + // the string must not be reassigned inside the loop + let reassigned = false; + walk(f.body, (m) => { const s = m as unknown as Stmt; if ((s.k === 'assign' && s.targets.some((t) => mentions(t, X))) || (s.k === 'aug' && mentions(s.target, X))) reassigned = true; }); + if (!ok || reassigned) return; + let cp = 'cp'; for (let k = 2; taken.has(cp); k++) cp = `cp${k}`; + const re = new RegExp(`String\\(\\s*${escapeRe(X)}\\[\\s*codepoint\\s*=\\s*${escapeRe(i)}\\s*\\]\\s*\\)`, 'g'); + const first = f.pos.line, last = lastLineOf(f.body); + const body = src.slice(first, last).map((l) => l.replace(re, `String(${cp})`)); + const hit = src.slice(first, last).reduce((c, l) => c + (l.match(re)?.length ?? 0), 0); + if (hit !== uses) return; // a use the line text does not show plainly: leave it alone + const head = src[first - 1] ?? ''; + const hm = new RegExp(`^(\\s*)for\\s+${escapeRe(i)}\\s+in\\s+range\\(.*\\)\\s*:\\s*$`).exec(head); + if (!hm) return; + const nh = `${hm[1]}for ${cp} in ${X}.codepoint_slices():`; + const lastUse = body.reduce((m, l, k) => (l.includes(`String(${cp})`) ? k : m), 0); + const text2 = [nh, ...body.slice(0, lastUse + 1)].join('\n'); + out.push({ + rule: 'codepoint-loop', line: first, endLine: first + lastUse + 1, + message: `\`${i}\` is only used to read one letter of \`${X}\`. Loop over the letters directly.`, + before: `${head.trim()}\n … String(${X}[codepoint=${i}]) …`, after: `${nh.trim()}\n … String(${cp}) …`, + docUrl: this.docUrl, + pseudoHint: 'In pseudo, `FOR EACH C IN T` (in lingo, "For each letter c of T") is a loop over the letters, where `FOR I <- 0 TO LENGTH(T)` is a loop over positions.', + fix: { ...wholeLines(first, first + lastUse + 1, text2), label: `Loop over the letters of ${X}` }, + }); + }); + } + return out; + }, +}; + +/* ---------- 4. raises on a function that cannot raise ---------- */ + +const SAFE_CALLS = new Set(['print', 'range', 'len', 'String', 'Int', 'Float64', 'Bool', 'min', 'max', 'abs', 'List', 'Dict', 'Set', 'ord', 'chr']); +const SAFE_METHODS = new Set(['append', 'count_codepoints', 'codepoint_slices', 'reserve', 'copy', 'upper', 'lower', 'startswith', 'endswith', 'strip', 'find', 'pop', 'extend', 'insert', 'clear', 'bytes', 'count']); + +const needlessRaises: PracticeRule = { + id: 'unneeded-raises', + title: 'Drop `raises` from a function that cannot raise', + why: '`raises` tells every caller that this function may fail, so each of them has to handle an error that never comes. Mojo checks the claim in the other direction, so when nothing in the body can raise, the word only costs.', + docUrl: `${MANUAL}/errors/`, + example: { + before: 'def double(n: Int) raises -> Int:\n return n * 2\n\n\ndef main() raises:\n print(double(4))\n', + after: 'def double(n: Int) -> Int:\n return n * 2\n\n\ndef main():\n print(double(4))\n', + }, + apis: [], + detect(mod, source) { + const src = lines(source); + const out: PracticeFinding[] = []; + const raising = new Set<string>(MOJO_PRELUDE_RAISES); + let usesDict = false; + walk(mod.body, (n) => { if (n.k === 'name' && (n.id === 'Dict' || n.id === 'Python')) usesDict = true; }); + let usesStruct = false; + walk(mod.body, (n) => { if (n.k === 'struct') usesStruct = true; }); + const defs = defsOf(mod); + for (const d of defs) if (d.raises) raising.add(d.name); + const baseErrors = checkMojo(source).diagnostics.filter((x) => x.severity === 'error').length; + for (const d of defs) { + if (!d.raises || d.raisesType || d.name.startsWith('test_')) continue; // a test function says raises because its runner expects it + // a body that is only `pass` or `...` is a placeholder (often a demonstration of `raises` itself); a user type's indexing may raise + let bad = usesDict || d.body.every((x) => x.k === 'pass' || (x.k === 'expr' && x.e.k === 'ellipsis')) || (usesStruct && d.body.some((x) => { let ix = false; walk(x, (n) => { if (n.k === 'index') ix = true; }); return ix; })); + walk(d.body, (n) => { + if (n.k === 'raise' || n.k === 'try' || n.k === 'with' || n.k === 'import' || n.k === 'from' || n.k === 'lambda') bad = true; + if (n.k === 'call') { + const c = n as unknown as Extract<Expr, { k: 'call' }>; + if (c.fn.k === 'name') { const fid = c.fn.id; if (raising.has(fid) || !(SAFE_CALLS.has(fid) || defs.some((x) => x.name === fid && !x.raises))) bad = true; } + else if (c.fn.k === 'attr') { if (!SAFE_METHODS.has(c.fn.name)) bad = true; } + else bad = true; + } + }); + if (bad) continue; + const l = src[d.pos.line - 1] ?? ''; + const nl = l.replace(/\)\s*raises\b/, ')'); + if (nl === l || !/:\s*$/.test(nl)) continue; + const trial = [...src]; trial[d.pos.line - 1] = nl; + const after = checkMojo(trial.join('\n')).diagnostics.filter((x) => x.severity === 'error').length; + if (after > baseErrors) continue; + out.push({ + rule: 'unneeded-raises', line: d.pos.line, + message: `\`${d.name}\` says it raises, but nothing in it can. Callers have to handle an error that never comes.`, + before: l.trim(), after: nl.trim(), docUrl: this.docUrl, + fix: { ...wholeLines(d.pos.line, d.pos.line, nl), label: `Remove raises from ${d.name}` }, + }); + } + return out; + }, +}; + +/* ---------- 5. a value that never changes ---------- */ + +const constantValue: PracticeRule = { + id: 'comptime-constant', + title: 'A value that never changes can be `comptime`', + why: 'A `comptime` name is worked out while the program is compiled and can never be assigned again, so the compiler holds you to it. A `var` that is only ever read says less than it could.', + docUrl: `${MANUAL}/metaprogramming/comptime-evaluation/`, + example: { + before: 'def main():\n var width = 4\n for i in range(width):\n print(i)\n', + after: 'def main():\n comptime width = 4\n for i in range(width):\n print(i)\n', + }, + apis: [], + detect(mod, source) { + const src = lines(source); + const out: PracticeFinding[] = []; + const written = new Set<string>(); + walk(mod.body, (n) => { + const s = n as unknown as Stmt; + if (s.k === 'assign') for (const t of s.targets) walk(t, (m) => { if (m.k === 'name') written.add(m.id as string); }); + if (s.k === 'aug') walk(s.target, (m) => { if (m.k === 'name') written.add(m.id as string); }); + if (n.k === 'walrus') written.add(n.name as string); + if (s.k === 'for') walk(s.target, (m) => { if (m.k === 'name') written.add(m.id as string); }); + }); + const declared = new Map<string, number>(); + walk(mod.body, (n) => { const s = n as unknown as Stmt; if (s.k === 'var' && s.targets.k === 'name') declared.set(s.targets.id, (declared.get(s.targets.id) ?? 0) + 1); }); + for (const d of defsOf(mod)) { + // the names this function hands to something that might change them, and the names that size something, + // found in one walk (a walk per declaration made a long function quadratic) + let uses: { handed: Set<string>; sizing: Set<string> } | null = null; + const usesOf = () => { + if (uses) return uses; + const handed = new Set<string>(), sizing = new Set<string>(); + walk(d.body, (n) => { + if (n.k === 'call') { + const c = n as unknown as Extract<Expr, { k: 'call' }>; + for (const a of c.args) if (a.value.k === 'name') { + if (!(c.fn.k === 'name' && SAFE_CALLS.has(c.fn.id))) handed.add(a.value.id); + if (c.fn.k === 'name' && c.fn.id === 'range') sizing.add(a.value.id); + } + } + if (n.k === 'attr' && (n as unknown as Extract<Expr, { k: 'attr' }>).obj.k === 'name') handed.add(((n as unknown as Extract<Expr, { k: 'attr' }>).obj as { id: string }).id); + if (n.k === 'index') { const x = n as unknown as Extract<Expr, { k: 'index' }>; if (x.obj.k !== 'name' || /^[A-Z]/.test(x.obj.id)) for (const a of x.args) if (a.value.k === 'name') sizing.add(a.value.id); } + }); + return (uses = { handed, sizing }); + }; + for (const s of d.body) { + if (s.k !== 'var' || s.comptime || s.ref || s.type || !s.value || s.targets.k !== 'name') continue; + const v = s.value, id = s.targets.id; + const literal = v.k === 'num' || (v.k === 'str' && !v.t) || (v.k === 'name' && (v.id === 'True' || v.id === 'False')); + if (!literal || written.has(id) || declared.get(id) !== 1) continue; + // read only: used as an argument of a builtin or in an expression, never handed to a function that might change it + if (usesOf().handed.has(id)) continue; + // only the number that sizes something is worth saying: a loop bound, a type parameter + if (!usesOf().sizing.has(id) || v.k !== 'num') continue; + const l = src[s.pos.line - 1] ?? ''; + out.push({ + rule: 'comptime-constant', line: s.pos.line, + message: `\`${id}\` is set once and never changed. Written as \`comptime ${id} = …\`, the compiler holds it to that.`, + before: l.trim(), after: l.trim().replace(/^var\b/, 'comptime'), docUrl: this.docUrl, + }); + } + } + return out; + }, +}; + +/* ---------- 6. argument conventions ---------- */ + +const argConventions: PracticeRule = { + id: 'argument-convention', + title: 'Say only the argument convention you need', + why: 'An argument is `imm` (immutable) unless the function says otherwise: it can look at the caller\'s value but not change it, and nothing is copied. `mut` is for a function that changes the caller\'s value, and `var` for one that takes ownership. Writing `imm` adds nothing, and `mut` on an argument the function never changes promises a change that does not happen.', + docUrl: `${MANUAL}/values/ownership/`, + example: { + before: 'def show(imm n: Int):\n print(n)\n\n\ndef main():\n show(3)\n', + after: 'def show(n: Int):\n print(n)\n\n\ndef main():\n show(3)\n', + }, + apis: [], + detect(mod, source) { + const src = lines(source); + const out: PracticeFinding[] = []; + for (const d of defsOf(mod)) { + const l = src[d.pos.line - 1] ?? ''; + for (const p of d.params) { + if (p.star || !p.convention || p.name === 'self') continue; + const re = new RegExp(`\\b${p.convention}\\s+${escapeRe(p.name)}\\b`); + if (!re.test(l)) continue; + const nl = l.replace(re, p.name); + if (!/:\s*$/.test(nl)) continue; + if (p.convention === 'imm') { + out.push({ + rule: 'argument-convention', line: d.pos.line, + message: `\`imm\` is what \`${p.name}\` already is without it: the function sees the caller's value and cannot change it.`, + before: l.trim(), after: nl.trim(), docUrl: this.docUrl, + fix: { ...wholeLines(d.pos.line, d.pos.line, nl), label: `Leave out imm on ${p.name}` }, + }); + } else if (p.convention === 'mut') { + // never written, never handed on, never called on + let touched = false; + walk(d.body, (n) => { + const s = n as unknown as Stmt; + if (s.k === 'assign') for (const t of s.targets) if (mentions(t, p.name)) touched = true; + if (s.k === 'aug' && mentions(s.target, p.name)) touched = true; + if (n.k === 'transfer' && mentions(n, p.name)) touched = true; + if (n.k === 'call') { + const c = n as unknown as Extract<Expr, { k: 'call' }>; + if (c.fn.k === 'attr' && mentions(c.fn.obj, p.name)) touched = true; + for (const a of c.args) if (a.value.k === 'name' && a.value.id === p.name && !(c.fn.k === 'name' && ['print', 'String', 'len', 'Int', 'Float64'].includes(c.fn.id))) touched = true; + } + if (n.k === 'for' && mentions((n as unknown as Extract<Stmt, { k: 'for' }>).iter, p.name)) touched = true; + if (n.k === 'walrus') touched = true; + }); + if (touched) continue; + out.push({ + rule: 'argument-convention', line: d.pos.line, + message: `\`mut ${p.name}\` promises the function changes the caller's value, but it never does. Without \`mut\` it is \`imm\`.`, + before: l.trim(), after: nl.trim(), docUrl: this.docUrl, + fix: { ...wholeLines(d.pos.line, d.pos.line, nl), label: `Leave out mut on ${p.name}` }, + }); + } + } + } + return out; + }, +}; + +/* ---------- 7. reserve ---------- */ + +const reserveList: PracticeRule = { + id: 'reserve', + title: 'Reserve room when you know how many items are coming', + why: 'A List that grows one append at a time may have to move its contents to bigger storage several times. When the loop count is known up front, `reserve` asks for the room once.', + docUrl: `${MANUAL}/types/`, + example: { + before: 'def main():\n var xs = List[Int]()\n for i in range(5):\n xs.append(i * i)\n print(len(xs))\n', + after: 'def main():\n var xs = List[Int]()\n xs.reserve(5)\n for i in range(5):\n xs.append(i * i)\n print(len(xs))\n', + }, + apis: [['List', 'reserve'], ['List', 'append']], + detect(mod, source) { + const src = lines(source); + const out: PracticeFinding[] = []; + const simple = (e: Expr): boolean => e.k === 'num' || e.k === 'name' || (e.k === 'call' && e.args.length === 0 && e.fn.k === 'attr' && e.fn.name === 'count_codepoints' && e.fn.obj.k === 'name') || (e.k === 'call' && e.fn.k === 'name' && e.fn.id === 'len' && e.args.length === 1 && e.args[0].value.k === 'name'); + for (const block of blocksOf(mod)) { + block.forEach((d, i) => { + if (d.k !== 'var' || d.targets.k !== 'name' || !d.value) return; + const v = d.value; + const emptyList = (v.k === 'call' && v.args.length === 0 && ((v.fn.k === 'index' && v.fn.obj.k === 'name' && v.fn.obj.id === 'List') || (v.fn.k === 'name' && v.fn.id === 'List'))) || (v.k === 'list' && v.items.length === 0); + if (!emptyList) return; + const id = d.targets.id; + for (let j = i + 1; j < block.length; j++) { + const f = block[j]; + if (!mentions(f, id)) continue; + if (f.k !== 'for' || f.comptime || f.orelse.length) return; + const it = f.iter; + if (it.k !== 'call' || it.fn.k !== 'name' || it.fn.id !== 'range' || it.args.length !== 1 || !simple(it.args[0].value)) return; + const bound = it.args[0].value; + // the loop appends once per pass, at the top level of its body, and does not leave early + const appends = f.body.filter((s) => s.k === 'expr' && s.e.k === 'call' && s.e.fn.k === 'attr' && s.e.fn.name === 'append' && s.e.fn.obj.k === 'name' && s.e.fn.obj.id === id).length; + let early = false; walk(f.body, (n) => { if (n.k === 'break' || n.k === 'return') early = true; }); + if (!appends || early) return; + // the bound must not be changed by the loop + if (bound.k === 'name') { + const bid = bound.id; let w = false; + walk(f.body, (n) => { const s2 = n as unknown as Stmt; if ((s2.k === 'assign' && s2.targets.some((t) => mentions(t, bid))) || (s2.k === 'aug' && mentions(s2.target, bid))) w = true; }); + if (w) return; + } + const l = src[f.pos.line - 1] ?? ''; + const bm = /range\((.*)\)\s*:\s*$/.exec(l); + if (!bm) return; + const ins = `${indentOf(l)}${id}.reserve(${bm[1]})`; + out.push({ + rule: 'reserve', line: f.pos.line, + message: `\`${id}\` is filled by a loop that runs ${bm[1]} times. Ask for the room first.`, + before: l.trim(), after: `${ins.trim()}\n${l.trim()}`, docUrl: this.docUrl, + fix: { ...wholeLines(f.pos.line, f.pos.line, `${ins}\n${l}`), label: `Reserve ${bm[1]} places in ${id}` }, + }); + return; + } + }); + } + return out; + }, +}; + +/* ---------- 8. a value that is handed over for good: ^ ---------- */ + +const localKind = (e: Expr | null): boolean => !!e && ((e.k === 'list') || (e.k === 'str' && !e.t) || (e.k === 'call' && ((e.fn.k === 'name' && (e.fn.id === 'String' || e.fn.id === 'List')) || (e.fn.k === 'index' && e.fn.obj.k === 'name' && e.fn.obj.id === 'List')))); + +const transferLast: PracticeRule = { + id: 'transfer-last-use', + title: 'Hand a value over with ^ when you are done with it', + why: 'A function that takes its argument as `var` owns it. Passing a name copies the value unless this is its last use, and `name^` says so: the value moves, nothing is copied, and the name is not used again.', + docUrl: `${MANUAL}/values/ownership/`, + example: { + before: 'def total(var xs: List[Int]) -> Int:\n var t = 0\n for x in xs:\n t += x\n return t\n\n\ndef main():\n var xs = [1, 2, 3]\n print(total(xs))\n', + after: 'def total(var xs: List[Int]) -> Int:\n var t = 0\n for x in xs:\n t += x\n return t\n\n\ndef main():\n var xs = [1, 2, 3]\n print(total(xs^))\n', + }, + apis: [], + detect(mod, source) { + const out: PracticeFinding[] = []; + const src = lines(source); + const defs = defsOf(mod); + for (const block of blocksOf(mod)) { + block.forEach((s, j) => { + walk(s, (n) => { + if (n.k !== 'call') return; + const c = n as unknown as Extract<Expr, { k: 'call' }>; + if (c.fn.k !== 'name') return; + const target = defs.find((x) => x.name === (c.fn as { id: string }).id); + if (!target) return; + c.args.forEach((a, k) => { + if (a.name || a.star || a.value.k !== 'name') return; + const p = target.params[k]; + if (!p || p.star || (p.convention !== 'var' && p.convention !== 'owned')) return; + const id = a.value.id; + // a local that holds a List or String, declared earlier in this block, with no use after this statement + const decl = block.slice(0, j).find((x) => x.k === 'var' && x.targets.k === 'name' && x.targets.id === id); + if (!decl || decl.k !== 'var' || !localKind(decl.value)) return; + if (block.slice(j + 1).some((x) => mentions(x, id)) || countMentions(s, id) !== 1) return; + // not inside a loop or a branch of this statement that could run again + if (s.k === 'for' || s.k === 'while') return; + const l = src[a.value.pos.line - 1] ?? ''; + const at = a.value.pos.col - 1; + if (l.slice(at, at + id.length) !== id || l[at + id.length] === '^') return; + const nl = `${l.slice(0, at)}${id}^${l.slice(at + id.length)}`; + out.push({ + rule: 'transfer-last-use', line: a.value.pos.line, + message: `\`${id}\` is not used after this call, and \`${target.name}\` takes it as \`var\`. Move it with \`${id}^\` instead of copying it.`, + before: l.trim(), after: nl.trim(), docUrl: this.docUrl, + fix: { ...wholeLines(a.value.pos.line, a.value.pos.line, nl), label: `Move ${id} with ^` }, + }); + }); + }); + }); + } + // one finding per spot + return out.filter((f, i) => out.findIndex((g) => g.line === f.line && g.before === f.before) === i); + }, +}; + +/* ---------- 9. a copy that should say so ---------- */ + +const explicitCopy: PracticeRule = { + id: 'explicit-copy', + title: 'Say .copy() when you mean a copy', + why: 'A List is not copied behind your back in Mojo. `var b = a` while `a` is still needed is an implicit copy, which the compiler refuses; `a.copy()` says you want a second list, and shows a reader where the cost is.', + docUrl: `${MANUAL}/values/ownership/`, + example: { + before: 'def main():\n var a = List[Int]()\n a.append(1)\n var b = a\n b.append(4)\n print(len(a), len(b))\n', + after: 'def main():\n var a = List[Int]()\n a.append(1)\n var b = a.copy()\n b.append(4)\n print(len(a), len(b))\n', + beforeRefused: 'cannot be implicitly copied', + }, + apis: [], + detect(mod, source) { + const out: PracticeFinding[] = []; + const src = lines(source); + for (const block of blocksOf(mod)) { + block.forEach((s, j) => { + // `ref b = a` is another name for the same list, not a copy (the manual's python-to-mojo tests) + if (s.k !== 'var' || s.ref || !s.value || s.value.k !== 'name' || s.targets.k !== 'name') return; + const id = s.value.id; + const decl = block.slice(0, j).find((x) => x.k === 'var' && x.targets.k === 'name' && x.targets.id === id); + if (!decl || decl.k !== 'var' || !decl.value || (decl.value.k !== 'list' && !(decl.value.k === 'call' && ((decl.value.fn.k === 'name' && decl.value.fn.id === 'List') || (decl.value.fn.k === 'index' && decl.value.fn.obj.k === 'name' && decl.value.fn.obj.id === 'List'))))) return; + if (!block.slice(j + 1).some((x) => mentions(x, id))) return; // the last use is a move, not a copy + const l = src[s.value.pos.line - 1] ?? ''; + const at = s.value.pos.col - 1; + if (l.slice(at, at + id.length) !== id) return; + const nl = `${l.slice(0, at)}${id}.copy()${l.slice(at + id.length)}`; + out.push({ + rule: 'explicit-copy', line: s.value.pos.line, + message: `\`${s.targets.id} = ${id}\` copies the list while \`${id}\` is still used below. Say \`.copy()\`.`, + before: l.trim(), after: nl.trim(), docUrl: this.docUrl, + fix: { ...wholeLines(s.value.pos.line, s.value.pos.line, nl), label: `Copy ${id} explicitly` }, + }); + }); + } + return out; + }, +}; + +export const RULES: readonly PracticeRule[] = [declareThenAssign, augmentedAssign, codepointLoop, needlessRaises, constantValue, argConventions, reserveList, transferLast, explicitCopy]; + +/* ---------- running them ---------- */ + +/** Every finding of every rule, in line order. */ +export function findPractices(source: string, module?: Module): PracticeFinding[] { + const mod = module ?? parseModule(source); + if (mod.findings.some((f) => f.severity === 'error')) return []; // advice on code that does not read yet is noise + const all: PracticeFinding[] = []; + for (const r of RULES) { try { all.push(...r.detect(mod, source)); } catch { /* a rule that cannot read this shape says nothing */ } } + return all.sort((a, b) => a.line - b.line || a.rule.localeCompare(b.rule)); +} + +/** Apply one quick fix to the text (whole-line fixes: the same edit the editor makes). */ +export function applyQuickFix(source: string, f: QuickFix): string { + const ls = source.split('\n'); + const last = Math.min(f.endLine ?? f.line, ls.length); + return [...ls.slice(0, f.line - 1), ...f.text.split('\n'), ...ls.slice(last)].join('\n'); +} + +/** + * Apply every safe rewrite to a copy: one at a time, finding again after each (the lines move), and keeping one + * only when the checker finds no more errors than before. Returns the new text and how many rewrites were applied. + */ +export function applyPractices(source: string, only?: ReadonlySet<string>): { source: string; applied: PracticeFinding[] } { + let cur = source; + const applied: PracticeFinding[] = []; + const baseErrors = checkMojo(source).diagnostics.filter((d) => d.severity === 'error').length; + const refused = new Set<string>(); + for (let n = 0; n < 60; n++) { + const f = findPractices(cur).find((x) => x.fix && !refused.has(`${x.rule}:${x.before}`) && (!only || only.has(x.rule))); + if (!f) break; + const next = applyQuickFix(cur, f.fix!); + if (next === cur || checkMojo(next).diagnostics.filter((d) => d.severity === 'error').length > baseErrors) { refused.add(`${f.rule}:${f.before}`); continue; } + cur = next; applied.push(f); + } + return { source: cur, applied }; +} + +/** Findings as editor diagnostics: always `note`, never an error or a warning, so they never pass for the compiler. */ +export function findingsToDiagnostics(findings: readonly PracticeFinding[]): Diagnostic[] { + return findings.map((f) => ({ + lang: 'mojo' as const, severity: 'note' as const, id: `practice-${f.rule}`, line: f.line, message: f.message, + origin: PRACTICES_ORIGIN, docUrl: f.docUrl, ...(f.fix ? { fix: f.fix } : {}), + })); +} + +export function practiceDiagnostics(source: string, module?: Module): Diagnostic[] { + return findingsToDiagnostics(findPractices(source, module)); +} diff --git a/src/mojo/properties.ts b/src/mojo/properties.ts new file mode 100644 index 0000000..ee7b7e6 --- /dev/null +++ b/src/mojo/properties.ts @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Generated-input testing for Mojo (Stage 5), in the shape the Tests view reads + * (core/types.ts, PropertyReport): run each of the program's functions on many + * inputs of its parameters' types, in Programming+'s interpreter, and report the + * inputs that make it fail, the smallest one found for each way of failing. + * + * "Failing" is what stops a compiled Mojo program, from the probes, not from + * Python habits: + * an index out of bounds the program aborts (list-index-oob: exit 132) + * a raise nobody catches the program ends with an error (error-unhandled), unless + * the function says `raises`: then raising is its business, + * and those inputs are listed, not counted as failures + * not finishing the step limit + * Integer overflow wraps and an Int division by zero is 0 (int-overflow-*, int-div-zero): + * those are Mojo's behavior, not failures, and are not reported. + * + * A function is tested if every parameter has a type with a generator here (Int and + * the sized integers, Float64, Bool, String, List of those) and is passed by value + * (no convention, or var). Inputs: the edge cases first (0, ±1, the extremes, the + * empty string and list), then seeded random values, so a report is the same on + * every run. + */ +import type { Expr, Module, Stmt } from './parse.js'; +import { callMojo } from './interp.js'; +import { type V, int, float, bool, str, INT_TYPES, show, reprFloat } from './values.js'; +import type { PropertyReport } from '../common/types.js'; + +type DefStmt = Extract<Stmt, { k: 'def' }>; +export type Gen = { edges: V[]; random: (rnd: () => number) => V; size: (v: V) => number }; + +/** A small seeded generator (mulberry32): the same inputs on every run. */ +export function rng(seed: number): () => number { + let a = seed >>> 0; + return () => { a = (a + 0x6d2b79f5) >>> 0; let t = a; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; }; +} + +export function typeText(e: Expr | null): string { + if (!e) return ''; + if (e.k === 'name') return e.id; + if (e.k === 'index' && e.obj.k === 'name') return `${e.obj.id}[${e.args.map((a) => typeText(a.value)).join(', ')}]`; + return ''; +} + +export function generator(type: string): Gen | null { + const it = INT_TYPES[type]; + if (it) { + const max = it.signed ? (1n << BigInt(it.width - 1)) - 1n : (1n << BigInt(it.width)) - 1n; + const min = it.signed ? -(1n << BigInt(it.width - 1)) : 0n; + const edges = [0n, 1n, ...(it.signed ? [-1n] : []), 2n, 3n, ...(it.signed ? [-2n] : []), 10n, 100n, max, min].filter((x, i, a) => x >= min && x <= max && a.indexOf(x) === i); + return { + edges: edges.map((x) => int(x, type)), + random: (r) => int(BigInt(Math.floor(r() < 0.8 ? (r() * 200 - (it.signed ? 100 : 0)) : (r() - 0.5) * 2e9)), type), + size: (v) => (v.k === 'int' ? Math.min(1e6, Math.abs(Number(v.v))) : 0), + }; + } + if (type === 'Float64' || type === 'Float32') { + const bits = type === 'Float32' ? 32 : 64; + return { edges: [0, 1, -1, 0.5, -0.5, 2.5, 1e10, -1e10].map((x) => float(x, bits)), random: (r) => float(Math.round((r() * 200 - 100) * 100) / 100, bits), size: (v) => (v.k === 'float' ? Math.abs(v.v) : 0) }; + } + if (type === 'Bool') return { edges: [bool(false), bool(true)], random: (r) => bool(r() < 0.5), size: (v) => (v.k === 'bool' && v.v ? 1 : 0) }; + if (type === 'String') { + const alphabet = 'abcAGCT 0é'; + return { + edges: ['', 'a', 'A', 'ab', 'aa', 'ba', 'GATTACA', ' ', 'é', '0'].map(str), + random: (r) => str(Array.from({ length: Math.floor(r() * 7) }, () => alphabet[Math.floor(r() * alphabet.length)]).join('')), + size: (v) => (v.k === 'str' ? v.v.length : 0), + }; + } + const m = /^List\[(.+)\]$/.exec(type); + if (m) { + const inner = generator(m[1]); + if (!inner || m[1].startsWith('List')) return null; + const list = (items: V[]): V => ({ k: 'list', items, elem: m[1], fixed: false, cap: items.length }); + return { + edges: [list([]), list([inner.edges[0]]), list([inner.edges[1] ?? inner.edges[0]]), list(inner.edges.slice(0, 3)), list([inner.edges[0], inner.edges[0]])], + random: (r) => list(Array.from({ length: Math.floor(r() * 6) }, () => inner.random(r))), + size: (v) => (v.k === 'list' ? 10 * v.items.length + v.items.reduce((a, x) => a + inner.size(x), 0) : 0), + }; + } + return null; +} + +/** How a call is written in Mojo: count("a", [1, 2]). */ +export function literal(v: V): string { + switch (v.k) { + case 'str': return JSON.stringify(v.v); + case 'list': return `[${v.items.map(literal).join(', ')}]`; + case 'float': return v.bits === 32 ? `Float32(${reprFloat(v.v)})` : reprFloat(v.v); + case 'int': return v.type === 'Int' ? String(v.v) : `${v.type}(${v.v})`; + default: return show(v); + } +} + +/** The most parameters a function's generated inputs cover, and the most functions one call tests (tests/mojo-robustness.test.mjs). */ +const MAX_PARAMS = 64; +const MAX_FUNCTIONS = 100; + +export function mojoProperties(mod: Module, source: string, count = 300): PropertyReport[] { + if (mod.unread) return []; + const out: PropertyReport[] = []; + const defs = mod.body.filter((s): s is DefStmt => s.k === 'def' && s.name !== 'main'); + defs.forEach((d, i) => out.push(i < MAX_FUNCTIONS ? test(mod, source, d, count) : { name: d.name, checked: 0, failures: [], problem: `Only the first ${MAX_FUNCTIONS} functions of a program get generated inputs.` })); + return out; +} + +function test(mod: Module, source: string, d: DefStmt, count: number): PropertyReport { + const name = d.name; + if (d.typeParams.length) return { name, checked: 0, failures: [], problem: 'This function has compile-time parameters; generated inputs cover plain arguments only.' }; + const params = d.params.filter((p) => p.star !== 'marker' && p.star !== 'positional-marker'); + // the edge cases are combined one parameter deeper per parameter (combine, below) + if (params.length > MAX_PARAMS) return { name, checked: 0, failures: [], problem: `This function has more than ${MAX_PARAMS} parameters; generated inputs cover functions with fewer.` }; + const gens: Gen[] = []; + for (const p of params) { + if (p.star || p.convention === 'mut' || p.convention === 'out' || p.convention === 'ref' || p.convention === 'inout') return { name, checked: 0, failures: [], problem: `${p.name} is passed as ${p.star ? 'a variadic' : p.convention}: generated inputs cover arguments passed by value only.` }; + const g = generator(typeText(p.type)); + if (!g) return { name, checked: 0, failures: [], problem: `No generator for ${p.name}'s type (${typeText(p.type) || 'none written'}): Int, the sized integers, Float64, Bool, String, and a List of those are generated.` }; + gens.push(g); + } + // the edge cases in combination (capped), then random inputs + const inputs: V[][] = []; + const combine = (i: number, acc: V[]) => { if (inputs.length >= count / 2) return; if (i === gens.length) { inputs.push(acc); return; } for (const v of gens[i].edges) combine(i + 1, [...acc, v]); }; + combine(0, []); + const r = rng(0x5eed); + while (inputs.length < count && gens.length) inputs.push(gens.map((g) => g.random(r))); + if (!gens.length) inputs.splice(1); + const size = (args: V[]) => args.reduce((a, v, i) => a + gens[i].size(v), 0); + // per way of failing, the smallest input found + const worst = new Map<string, { args: V[]; message: string; line?: number; kind: string }>(); + const raises = new Map<string, { args: V[]; message: string; line?: number }>(); + let checked = 0; + // one way of failing is one shape of message: the quoted input and the numbers in it abstracted away + const shape = (m: string) => m.replace(/'[^']*'|"[^"]*"/g, "'…'").replace(/-?\d+/g, 'N'); + for (const args of inputs) { + const o = callMojo(mod, source, name, args); + if (o.kind === 'declined') return { name, checked, failures: [...worst.values()].map(row(name)), problem: `Programming+ cannot run ${name} yet: ${o.reason}.` }; + checked++; + const keep = (map: Map<string, any>, key: string, entry: any) => { const prev = map.get(key); if (!prev || size(entry.args) < size(prev.args)) map.set(key, entry); }; + if (o.kind === 'aborted') keep(worst, `abort:${o.line}:${shape(o.message)}`, { args, message: `aborts: ${o.message}`, line: o.line, kind: 'bounds' }); + else if (o.kind === 'limit') keep(worst, 'limit', { args, message: 'ran past the 5,000-step limit: an endless loop or recursion, or just a lot of work (the compiled program may finish; this only says Programming+ stopped waiting)', kind: 'limit' }); + else if (o.kind === 'raised') { + if (d.raises) keep(raises, `${o.line}:${shape(o.message)}`, { args, message: o.message, line: o.line ?? undefined }); + else keep(worst, `raise:${o.line}:${shape(o.message)}`, { args, message: `raises ${o.message}, which nothing catches`, line: o.line ?? undefined, kind: 'raise' }); + } + } + const failures = [...worst.values()].map(row(name)); + // a raising function: what it raises is listed, as information + for (const x of raises.values()) failures.push({ call: `${name}(${x.args.map(literal).join(', ')})`, message: `raises ${x.message} (it says raises, so this may be intended)`, line: x.line, kind: 'raises-declared' }); + return { name, checked, failures }; +} + +const row = (name: string) => (x: { args: V[]; message: string; line?: number; kind: string }) => ({ call: `${name}(${x.args.map(literal).join(', ')})`, message: x.message, line: x.line, kind: x.kind }); diff --git a/src/mojo/reference.ts b/src/mojo/reference.ts new file mode 100644 index 0000000..b1fef7b --- /dev/null +++ b/src/mojo/reference.ts @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Hover text for the Mojo pane, in our own words. Stage 1 covers what the + * pane's colors single out: the argument conventions and the transfer + * sigil (the vocabulary of ownership, which is what Programming+ is for), + * the words 1.1.0 removed, and the keywords that differ from Python. + * Each entry links to the manual page that teaches it. + */ +import { MOJO_REMOVED_WORDS } from './generated/tokens.js'; +import { MOJO_VERSION } from './generated/pin.js'; + +const MANUAL = 'https://mojolang.org/docs/manual'; +interface Entry { title: string; body: string; where?: string } + +const OWNERSHIP = `${MANUAL}/values/ownership/`; +const WORDS: Record<string, Entry> = { + imm: { title: 'imm x: T', body: 'The default argument convention, rarely written: the function gets an immutable reference to the caller\'s value. Nothing is copied, and nothing can change it.', where: OWNERSHIP }, + mut: { title: 'mut x: T', body: 'The function gets a mutable reference: a change inside the function is a change to the caller\'s variable. The same value cannot be passed twice in one call while one of them is mut.', where: OWNERSHIP }, + out: { title: 'out self', body: 'An argument that starts uninitialized and must be initialized before the function returns. Every __init__ takes `out self`; it is never passed by the caller.', where: OWNERSHIP }, + deinit: { title: 'deinit self', body: 'An argument whose lifetime ends in this function: valid on the way in, gone on the way out. __deinit__ and the move constructor take it, and only here may fields be moved out without being put back.', where: OWNERSHIP }, + var: { title: 'var', body: 'Declares a variable that owns its value. In front of a parameter, it means the function owns the argument: the caller transfers it with ^ or hands over a copy.', where: OWNERSHIP }, + ref: { title: 'ref', body: 'A reference whose mutability follows the value it refers to; used to return a reference into a value, or to bind one with `ref x = list[0]`.', where: `${MANUAL}/values/lifetimes/` }, + '^': { title: 'value^', body: 'Transfer: ownership moves out of the variable, which is uninitialized afterwards until it is assigned again. Using it in between is the error "use of uninitialized value".', where: OWNERSHIP }, + comptime: { title: 'comptime', body: 'Evaluated while compiling, not while running: a compile-time constant (`comptime N = 4`), or `comptime if` / `comptime for`, which decide what code exists at all.', where: `${MANUAL}/metaprogramming/comptime-evaluation/` }, + raises: { title: 'raises', body: 'This function may raise an error. Mojo checks it: calling a raising function from one that does not raise, outside a try, is a compile error.', where: `${MANUAL}/errors/` }, + struct: { title: 'struct', body: 'A type with fields fixed at compile time; Mojo\'s structs have no hidden dictionary and are laid out in memory like a C struct.', where: `${MANUAL}/structs/` }, + def: { title: 'def', body: `The one way to write a function in Mojo ${MOJO_VERSION}. Argument types are declared, and a function that can raise says \`raises\`.`, where: `${MANUAL}/functions/` }, +}; + +export function explainMojo(word: string): Entry | null { + if (WORDS[word]) return WORDS[word]; + const repl = MOJO_REMOVED_WORDS[word]; + if (repl) return { title: word, body: `Removed in Mojo ${MOJO_VERSION}: write ${repl}. Older tutorials still use it.`, where: `https://mojolang.org/releases/` }; + return null; +} diff --git a/src/mojo/resolve.ts b/src/mojo/resolve.ts new file mode 100644 index 0000000..fe72e18 --- /dev/null +++ b/src/mojo/resolve.ts @@ -0,0 +1,233 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Stage 2a's checks over the syntax tree: structure, and names. Every message + * is the compiler's, with its position and underline (godbolt's verdicts on + * Mojo Quest's starters, tests/mojo-check.test.mjs), and every check is one + * the compiler is certain to make: + * + * expressions must not appear at file scope a call or expression at the top level + * argument type must be specified a def parameter without a type (self excepted) + * self argument must be present in instance method a struct's def without self, not static + * module does not contain a 'main' function (no position: the compiler gives none) + * cannot declare 'x' without a contextual type `var x` with neither type nor value + * implicit declaration of 'x' is not allowed assigning a name nothing declared + * use of unknown declaration 'x' reading a name nothing declared, with the + * compiler's import hint for a std name + * + * Names are checked only where a value is expected (never inside a type + * annotation, a decorator, or a t-string), and only when the parser read the + * whole file: a name this checker cannot see declared is not a mistake if the + * declaration is in code it could not read. + */ +import type { Expr, Module, Param, Pos, Stmt } from './parse.js'; +import { MOJO_KEYWORDS, MOJO_INTERNAL_KEYWORDS, MOJO_PRELUDE } from './generated/tokens.js'; +import { MOJO_STD_EXPORTS } from './generated/stdlib.js'; + +export interface Check { line: number; col: number; endCol: number; message: string; id: string } + +const KNOWN = new Set<string>([...MOJO_PRELUDE, ...MOJO_KEYWORDS, ...MOJO_INTERNAL_KEYWORDS, '_']); +/** The compiler's own built-ins (__mlir_op, __mlir_type, __type_of, ...) are never unknown. */ +const known = (n: string) => KNOWN.has(n) || n.startsWith('__'); + +const under = (p: Pos, name: string) => ({ line: p.line, col: p.col, endCol: p.col + name.length }); + +class Scope { + readonly names = new Set<string>(); + constructor(readonly parent: Scope | null, readonly kind: 'module' | 'function' | 'struct' | 'block') { } + has(n: string): boolean { return this.names.has(n) || (this.parent?.has(n) ?? false); } +} + +/** Every name a statement list declares, at its own level (hoisted: Programming+ does not judge order). */ +function declare(stmts: Stmt[], scope: Scope, nested: boolean): void { + for (const s of stmts) { + switch (s.k) { + case 'def': case 'struct': case 'trait': scope.names.add(s.name); break; + case 'var': if (s.type || s.value) for (const n of targetNames(s.targets)) scope.names.add(n); break; + case 'import': scope.names.add(s.alias ?? s.module.split('.')[0]); break; + case 'from': for (const n of s.names) scope.names.add(n.alias ?? n.name); break; + case 'global': for (const n of s.names) scope.names.add(n); break; + case 'for': for (const n of targetNames(s.target)) scope.names.add(n); if (nested) { declare(s.body, scope, true); declare(s.orelse, scope, true); } break; + case 'with': for (const it of s.items) if (it.as) for (const n of targetNames(it.as)) scope.names.add(n); if (nested) declare(s.body, scope, true); break; + case 'try': for (const h of s.handlers) if (h.name) scope.names.add(h.name); if (nested) { declare(s.body, scope, true); for (const h of s.handlers) declare(h.body, scope, true); declare(s.orelse, scope, true); declare(s.finalbody, scope, true); } break; + case 'if': if (nested) { declare(s.body, scope, true); declare(s.orelse, scope, true); } break; + case 'while': if (nested) { declare(s.body, scope, true); declare(s.orelse, scope, true); } break; + default: break; + } + } +} + +function targetNames(e: Expr): string[] { + if (e.k === 'name') return [e.id]; + if (e.k === 'tuple' || e.k === 'list') return e.items.flatMap(targetNames); + if (e.k === 'unary' && e.op === '*') return targetNames(e.x); + return []; +} + +export function check(mod: Module): Check[] { + const out: Check[] = []; + const complete = mod.unread === 0; + const starImport = mod.body.some((s) => s.k === 'from' && s.star); + + /* structure: file scope, main. A line the lexer already rejected gets nothing more. */ + const lexed = new Set(mod.findings.filter((f) => f.severity === 'error').map((f) => f.line)); + for (const s of mod.body) { + if (s.k === 'expr' && s.e.k !== 'str' && !lexed.has(s.pos.line)) out.push({ ...exprStart(s.e), message: 'expressions must not appear at file scope; move this into a function body', id: 'file-scope' }); + } + if (complete && !mod.body.some((s) => s.k === 'def' && s.name === 'main')) out.push({ line: 1, col: 1, endCol: 1, message: "module does not contain a 'main' function", id: 'no-main' }); + + const moduleScope = new Scope(null, 'module'); + declare(mod.body, moduleScope, false); + + const visitParams = (ps: Param[], scope: Scope, inStruct: boolean, def: Extract<Stmt, { k: 'def' }>) => { + const real = ps.filter((p) => p.star !== 'marker' && p.star !== 'positional-marker'); + const staticMethod = def.decorators.some((d) => (d.k === 'name' && d.id === 'staticmethod') || (d.k === 'call' && d.fn.k === 'name' && d.fn.id === 'staticmethod')); + for (const [i, p] of real.entries()) { + scope.names.add(p.name); + // a method's first parameter is self whatever its name (the compiler's tests: `def b(c, d: a)`), + // and a parameter named self is never "untyped" + const isSelf = p.name === 'self' || (inStruct && !staticMethod && i === 0); + if (!p.type && !isSelf && !p.star) out.push({ ...under(p.pos, p.name), message: 'argument type must be specified', id: 'untyped-argument' }); + } + // certain only when there is no parameter at all to be self (MQ-504); `out self` may come later + if (inStruct && !staticMethod && real.length === 0 && !/^__\w+__$/.test(def.name)) out.push({ ...under(def.pos, def.name), message: 'self argument must be present in instance method', id: 'missing-self' }); + }; + + function stmts(list: Stmt[], scope: Scope, inStruct: boolean): void { + for (const s of list) stmt(s, scope, inStruct); + } + + function stmt(s: Stmt, scope: Scope, inStruct: boolean): void { + switch (s.k) { + case 'def': { + const fn = new Scope(scope, 'function'); + for (const p of s.typeParams) fn.names.add(p.name); + visitParams(s.params, fn, inStruct, s); + for (const p of s.params) if (p.default) value(p.default, scope); + declare(s.body, fn, true); + stmts(s.body, fn, false); + return; + } + case 'struct': case 'trait': { + const st = new Scope(scope, 'struct'); + for (const p of s.typeParams) st.names.add(p.name); + st.names.add('Self'); + declare(s.body, st, false); + for (const b of s.body) { + if (b.k === 'def') stmt(b, st, true); + else if (b.k === 'var') { if (b.value) { const inner = b.typeParams?.length ? new Scope(st, 'block') : st; for (const p of b.typeParams ?? []) inner.names.add(p.name); value(b.value, inner); } } + else stmt(b, st, false); + } + return; + } + case 'var': { + if (!s.type && !s.value && !s.comptime && !s.ref) for (const n of targetNames(s.targets)) out.push({ ...under(s.targets.k === 'name' ? s.targets.pos : s.pos, n), message: `cannot declare '${n}' without a contextual type from its initializer`, id: 'untyped-var' }); + if (s.value) { + const inner = s.typeParams?.length ? new Scope(scope, 'block') : scope; + for (const p of s.typeParams ?? []) inner.names.add(p.name); + value(s.value, inner); + } + return; + } + case 'assign': { + value(s.value, scope); + for (const t of s.targets) assignTarget(t, scope); + return; + } + case 'aug': value(s.value, scope); value(s.target, scope); return; + case 'expr': value(s.e, scope); return; + case 'if': value(s.test, scope); stmts(s.body, scope, false); stmts(s.orelse, scope, false); return; + case 'while': value(s.test, scope); stmts(s.body, scope, false); stmts(s.orelse, scope, false); return; + case 'for': value(s.iter, scope); stmts(s.body, scope, false); stmts(s.orelse, scope, false); return; + case 'return': case 'raise': if (s.value) value(s.value, scope); return; + case 'assert': value(s.test, scope); if (s.msg) value(s.msg, scope); return; + case 'try': stmts(s.body, scope, false); for (const h of s.handlers) stmts(h.body, scope, false); stmts(s.orelse, scope, false); stmts(s.finalbody, scope, false); return; + case 'with': for (const it of s.items) value(it.e, scope); stmts(s.body, scope, false); return; + default: return; + } + } + + function assignTarget(t: Expr, scope: Scope): void { + if (t.k === 'name') { + if (complete && !starImport && !scope.has(t.id) && !known(t.id)) { + out.push({ ...under(t.pos, t.id), message: `implicit declaration of '${t.id}' is not allowed; add 'var' to declare a new name`, id: 'implicit-declaration' }); + // as the compiler does: having said so once, treat the name as declared from here on + let fn: Scope | null = scope; while (fn && fn.kind !== 'function' && fn.kind !== 'module') fn = fn.parent; + (fn ?? scope).names.add(t.id); + } + return; + } + if (t.k === 'tuple' || t.k === 'list') { for (const x of t.items) assignTarget(x, scope); return; } + value(t, scope); // a.b = ..., a[i] = ...: reads a + } + + function value(e: Expr, scope: Scope): void { + switch (e.k) { + case 'name': + if (complete && !starImport && !e.escaped && !scope.has(e.id) && !known(e.id)) { + const from = MOJO_STD_EXPORTS[e.id]; + // the compiler adds "; did you mean 'count'?" when a declaration of its own is close; + // its rule is unknown here, so near a declared name Programming+ says nothing + if (!from && nearDeclared(e.id, scope)) return; + out.push({ + ...under(e.pos, e.id), id: 'unknown-name', + message: from ? `use of unknown declaration '${e.id}'; did you mean to import it from '${from}'? Add 'from ${from} import ${e.id}'` : `use of unknown declaration '${e.id}'` + }); + } + return; + case 'list': case 'set': case 'tuple': for (const x of e.items) value(x, scope); return; + case 'dict': for (const [k, v] of e.entries) { value(k, scope); value(v, scope); } return; + case 'comp': { + value(e.iter, scope); + const inner = new Scope(scope, 'block'); + for (const n of targetNames(e.vars)) inner.names.add(n); + value(e.elt, inner); if (e.cond) value(e.cond, inner); + return; + } + case 'call': value(e.fn, scope); for (const a of e.args) value(a.value, scope); return; + case 'index': value(e.obj, scope); for (const a of e.args) value(a.value, scope); return; + case 'attr': value(e.obj, scope); return; + case 'unary': case 'not': case 'transfer': value(e.x, scope); return; + case 'binary': case 'bool': value(e.l, scope); value(e.r, scope); return; + case 'compare': for (const x of e.operands) value(x, scope); return; + case 'ifexp': value(e.cond, scope); value(e.a, scope); value(e.b, scope); return; + case 'walrus': scope.names.add(e.name); value(e.value, scope); return; + case 'slice': for (const x of [e.lo, e.hi, e.step]) if (x) value(x, scope); return; + case 'lambda': { const inner = new Scope(scope, 'function'); for (const p of e.params) inner.names.add(p); value(e.body, inner); return; } + case 'record': for (const f of e.fields) value(f.value, scope); return; + default: return; + } + } + + stmts(mod.body, moduleScope, false); + return out; +} + +/** Whether the program declares (or the language reserves) a name one or two edits from `n`. */ +function nearDeclared(n: string, scope: Scope): boolean { + const names: string[] = []; + for (let s: Scope | null = scope; s; s = s.parent) names.push(...s.names); + names.push(...MOJO_KEYWORDS); + return names.some((m) => m !== n && Math.abs(m.length - n.length) <= 2 && editDistance(m, n) <= 2); +} + +function editDistance(a: string, b: string): number { + const d = Array.from({ length: a.length + 1 }, (_, i) => [i, ...Array(b.length).fill(0)]); + for (let j = 1; j <= b.length; j++) d[0][j] = j; + for (let i = 1; i <= a.length; i++) for (let j = 1; j <= b.length; j++) + d[i][j] = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1)); + return d[a.length][b.length]; +} + +/** Where the compiler points for an expression: its leftmost token. */ +function exprStart(e: Expr): { line: number; col: number; endCol: number } { + let x: Expr = e; + for (; ;) { + if (x.k === 'call') x = x.fn; + else if (x.k === 'index' || x.k === 'attr') x = x.obj; + else if (x.k === 'binary' || x.k === 'bool') x = x.l; + else if (x.k === 'compare') x = x.operands[0]; + else break; + } + return { line: x.pos.line, col: x.pos.col, endCol: x.pos.col + 1 }; +} diff --git a/src/mojo/tokenize.ts b/src/mojo/tokenize.ts new file mode 100644 index 0000000..8ce05d2 --- /dev/null +++ b/src/mojo/tokenize.ts @@ -0,0 +1,165 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Mojo highlighting. The words come from the compiler itself, through the + * corpus (src/mojo/generated/tokens.ts): its token table, the soft keywords + * its parser accepts, the words it rejects as removed, the decorators the + * reference documents, and the names the prelude imports. + * + * The document is tokenized as a whole, so a triple-quoted docstring that + * spans lines is a string on every line it covers. The classes are the + * editor's existing ones plus two that carry what Programming+ is about: + * + * tok-convention imm, mut, out, deinit, var, ref in front of a parameter + * tok-transfer the postfix ^ that moves a value (not the XOR operator) + * tok-error a word the pinned Mojo rejects as removed (fn; read as a convention) + * + * This is a highlighter, not a parser: when the context is ambiguous it + * falls back to the plain class, never to a wrong one. + */ +import { MOJO_KEYWORDS, MOJO_INTERNAL_KEYWORDS, MOJO_SOFT_KEYWORDS, MOJO_REMOVED_KEYWORDS, MOJO_REMOVED_CONVENTIONS, MOJO_EFFECTS, MOJO_DECORATORS, MOJO_PRELUDE, MOJO_PUNCTUATION } from './generated/tokens.js'; + +export interface Token { text: string; cls: string } + +const KEYWORDS = new Set<string>([...MOJO_KEYWORDS, ...MOJO_INTERNAL_KEYWORDS]); +const PRELUDE = new Set<string>(MOJO_PRELUDE); +const EFFECTS = new Set<string>(MOJO_EFFECTS); +const DECORATORS = new Set<string>(MOJO_DECORATORS); +/** Argument conventions: the soft ones the parser names, plus the two that are hard keywords. */ +export const CONVENTIONS = new Set<string>([...MOJO_SOFT_KEYWORDS.filter((w) => w !== 'where'), 'var', 'ref']); +/** A removed keyword (fn) is an error anywhere; a removed convention (read) only in front of a parameter. */ +const REMOVED_HARD = new Set<string>(MOJO_REMOVED_KEYWORDS); +const REMOVED_CONVENTION = new Set<string>(MOJO_REMOVED_CONVENTIONS); +const PUNCT = [...MOJO_PUNCTUATION].sort((a, b) => b.length - a.length); + +/** A string still open at the end of a line: which quote closes it, and whether it is a t-string. */ +interface State { quote: '"""' | "'''" | null; t: boolean } + +const IDENT = /^[A-Za-z_][A-Za-z0-9_]*/; +const NUMBER = /^(?:0[xX][0-9a-fA-F_]+|0[oO][0-7_]+|0[bB][01_]+|(?:\d[\d_]*)?\.?\d[\d_]*(?:[eE][+-]?\d+)?)/; +const STRING_START = /^([rRtT]{0,2})("""|'''|"|')/; + +function scanString(line: string, i: number, quote: string, t: boolean, raw: boolean, out: Token[]): { i: number; closed: boolean } { + let buf = ''; + while (i < line.length) { + if (!raw && line[i] === '\\') { buf += line.slice(i, i + 2); i += 2; continue; } + if (line.startsWith(quote, i)) { buf += quote; i += quote.length; out.push({ text: buf, cls: 'tok-string' }); return { i, closed: true }; } + if (t && line[i] === '{' && line[i + 1] !== '{') { + // a t-string interpolation: the expression inside highlights as code + if (buf) out.push({ text: buf, cls: 'tok-string' }); + buf = ''; + let depth = 0, j = i; + for (; j < line.length; j++) { if (line[j] === '{') depth++; else if (line[j] === '}' && --depth === 0) break; } + out.push({ text: '{', cls: 'tok-punct' }); + for (const tok of tokenizeCode(line.slice(i + 1, Math.min(j, line.length)))) out.push(tok); + if (j < line.length) out.push({ text: '}', cls: 'tok-punct' }); + i = j + 1; + continue; + } + if (t && line[i] === '{' && line[i + 1] === '{') { buf += '{{'; i += 2; continue; } + buf += line[i]; i++; + } + if (buf) out.push({ text: buf, cls: 'tok-string' }); + return { i, closed: false }; +} + +/** Code with no string state: what a t-string interpolation holds, and every line not inside a docstring. */ +function tokenizeCode(line: string, state: State = { quote: null, t: false }): Token[] { + const out: Token[] = []; + let i = 0; + while (i < line.length) { + const rest = line.slice(i); + const c = line[i]; + if (c === ' ' || c === '\t') { const m = /^[ \t]+/.exec(rest)!; out.push({ text: m[0], cls: '' }); i += m[0].length; continue; } + if (c === '#') { out.push({ text: rest, cls: 'tok-comment' }); break; } + const s = STRING_START.exec(rest); + if (s && (s[1] === '' || /^[rRtT]{1,2}$/.test(s[1]))) { + const prefix = s[1], quote = s[2]; + const raw = /[rR]/.test(prefix), t = /[tT]/.test(prefix); + out.push({ text: prefix + quote, cls: 'tok-string' }); + const r = scanString(line, i + prefix.length + quote.length, quote, t, raw, out); + i = r.i; + if (!r.closed && quote.length === 3) { state.quote = quote as '"""' | "'''"; state.t = t; } + continue; + } + if (c === '`') { const j = line.indexOf('`', i + 1); const end = j < 0 ? line.length : j + 1; out.push({ text: line.slice(i, end), cls: 'tok-name' }); i = end; continue; } + if (/\d/.test(c) || (c === '.' && /\d/.test(line[i + 1] || ''))) { + const m = NUMBER.exec(rest); + if (m && m[0]) { out.push({ text: m[0], cls: 'tok-number' }); i += m[0].length; continue; } + } + const id = IDENT.exec(rest); + if (id) { out.push({ text: id[0], cls: '?' }); i += id[0].length; continue; } + const p = PUNCT.find((x) => rest.startsWith(x)); + if (p) { out.push({ text: p, cls: 'tok-op' }); i += p.length; continue; } + out.push({ text: c, cls: '' }); i++; + } + return classify(out); +} + +const isSpace = (t: Token | undefined) => !!t && t.cls === '' && /^[ \t]+$/.test(t.text); +function nextSolid(toks: Token[], k: number): Token | undefined { for (let j = k + 1; j < toks.length; j++) if (!isSpace(toks[j])) return toks[j]; return undefined; } +function prevSolid(toks: Token[], k: number): Token | undefined { for (let j = k - 1; j >= 0; j--) if (!isSpace(toks[j])) return toks[j]; return undefined; } + +/** Decide each word's class from its neighbors on the line. */ +function classify(toks: Token[]): Token[] { + const firstSolid = toks.findIndex((t) => !isSpace(t)); + const isDefLine = firstSolid >= 0 && (toks[firstSolid].text === 'def' || toks[firstSolid].text === 'fn'); + for (let k = 0; k < toks.length; k++) { + const t = toks[k]; + if (t.cls === 'tok-op' && t.text === '@' && k === firstSolid && toks[k + 1]?.cls === '?') { + t.cls = 'tok-label'; + let j = k + 1; + while (toks[j]?.cls === '?' || (toks[j]?.text === '.' && toks[j + 1]?.cls === '?')) { toks[j].cls = 'tok-label'; j++; } + continue; + } + if (t.cls === 'tok-op' && t.text === '^') { + // postfix transfer: right after a name or a closing bracket, then the end of the argument + const before = toks[k - 1]; + const after = nextSolid(toks, k); + const glued = before && !isSpace(before) && (before.cls === '?' || before.cls === 'tok-name' || before.cls === 'tok-builtin' || before.text === ')' || before.text === ']'); + if (glued && (!after || [')', ',', ']', '}', ':'].includes(after.text) || after.cls === 'tok-comment')) t.cls = 'tok-transfer'; + continue; + } + if (t.cls !== '?') continue; + const w = t.text; + let at = k + 1; + while (at < toks.length && isSpace(toks[at])) at++; + const next = toks[at]; + const prev = prevSolid(toks, k); + // from the index found, not toks.indexOf(next): that made a long line quadratic + const afterNext = next ? nextSolid(toks, at) : undefined; + // `mut x: T` / `out self` inside a parameter list; at the start of a line only a word that cannot + // begin a statement (so `var x: T` there is a declaration, and `mut x: T` a parameter on its own line) + const paramShape = !!next && next.cls === '?' && (afterNext?.text === ':' || next.text === 'self'); + const asConvention = paramShape && (prev ? ['(', ',', '[', '*'].includes(prev.text) : w !== 'var' && w !== 'ref'); + if (REMOVED_HARD.has(w)) t.cls = 'tok-error'; + else if (REMOVED_CONVENTION.has(w) && asConvention) t.cls = 'tok-error'; + else if (CONVENTIONS.has(w) && asConvention) t.cls = 'tok-convention'; + else if (KEYWORDS.has(w)) t.cls = 'tok-keyword'; + else if (w === 'raises' || (isDefLine && EFFECTS.has(w) && prev?.text === ')')) t.cls = 'tok-keyword'; + else if (prev?.text === '@' && DECORATORS.has(w)) t.cls = 'tok-label'; + else if (PRELUDE.has(w) && prev?.text !== '.') t.cls = 'tok-builtin'; + else t.cls = 'tok-name'; + } + return toks; +} + +/** Tokenize a whole document: one token list per line, with strings carried across lines. */ +export function tokenizeMojo(source: string): Token[][] { + const state: State = { quote: null, t: false }; + return source.split('\n').map((line) => { + if (state.quote) { + const out: Token[] = []; + const r = scanString(line, 0, state.quote, state.t, false, out); + if (!r.closed) return out.length ? out : [{ text: line, cls: 'tok-string' }]; + state.quote = null; state.t = false; + return [...out, ...tokenizeCode(line.slice(r.i), state)]; + } + return tokenizeCode(line, state); + }); +} + +/** One line on its own, assuming it does not start inside a string (hover, short samples). */ +export function tokenizeMojoLine(line: string): Token[] { + return tokenizeCode(line); +} diff --git a/src/mojo/typecheck.ts b/src/mojo/typecheck.ts new file mode 100644 index 0000000..4918a66 --- /dev/null +++ b/src/mojo/typecheck.ts @@ -0,0 +1,524 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Stage 2c: what types make certain, in the compiler's words (godbolt's verdicts + * on Mojo Quest's starters and the compiler's own tests, tests/mojo-check.test.mjs). + * + * A small model of types, and it only ever claims a type it is certain of: a + * literal, an annotation, a constructor call (the student's struct, or + * `List[Int]()`), `Int` arithmetic (Int / Int is an + * Int: MQ-211), a field of the student's struct, a call to the student's own + * function. Anything else is unknown, and nothing is said about it. + * + * What the standard library contributes comes from its own declarations + * (src/mojo/generated/types.ts): which structs are implicitly copyable, and the + * methods a trait gives by default (Copyable's copy, Equatable's reflective + * __eq__), so a struct is never told it lacks a method a trait gives it. + * + * 'S' value has no attribute 'f' at the dot + * 'S' does not implement the '__add__' method at the operator + * 'S' is not subscriptable, it does not implement ... at the [ + * value of type 'T' cannot be implicitly copied, ... at the name; even at its last use + * cannot implicitly convert 'StringLiteral["5"]' / 'None' / 'Int' value to 'T' + * invalid call to '__mul__': value passed to 'rhs' cannot be converted from 'Float64' to 'Int' + * invalid call to 'f': value passed to 'p' cannot be converted from 'IntLiteral[25]' to 'S' + * no matching function in initialization List() with nothing to say what it holds, + * or a constructor no argument types fit + * cannot call function that may raise ... d[k] on a Dict, outside raises or try + */ +import type { Expr, Module, Param, Pos, Stmt } from './parse.js'; +import { parseModule } from './parse.js'; +import type { QuickFix } from '../common/types.js'; +import { MOJO_STD_STRUCTS, MOJO_STD_TRAITS } from './generated/types.js'; + +export interface Finding3 { severity: 'error' | 'note'; line: number; col: number; message: string; id: string; question?: { text: string; options: QuickFix[] } } + +type DefStmt = Extract<Stmt, { k: 'def' }>; +type StructStmt = Extract<Stmt, { k: 'struct' | 'trait' }>; + +/** A type the model is certain of. */ +type Ty = + | { k: 'named'; name: string; args: Ty[] } + | { k: 'lit'; kind: 'int' | 'float' | 'string' | 'bool' | 'none'; text: string }; + +const SCALARS = new Set(['Int', 'UInt', 'Bool', 'Float64', 'Float32', 'Float16', 'Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64']); +const NUMERIC = new Set([...SCALARS].filter((s) => s !== 'Bool')); +const ARITH: Record<string, string> = { '+': '__add__', '-': '__sub__', '*': '__mul__', '/': '__truediv__', '//': '__floordiv__', '%': '__mod__', '**': '__pow__', '&': '__and__', '|': '__or__', '^': '__xor__', '<<': '__lshift__', '>>': '__rshift__', '@': '__matmul__' }; +const REFLECTED: Record<string, string> = Object.fromEntries(Object.entries(ARITH).map(([o, d]) => [o, d.replace('__', '__r')])); +const COMPARE: Record<string, string> = { '==': '__eq__', '!=': '__ne__', '<': '__lt__', '<=': '__le__', '>': '__gt__', '>=': '__ge__' }; +const UNARY: Record<string, string> = { '-': '__neg__', '+': '__pos__', '~': '__invert__' }; + +const named = (name: string, args: Ty[] = []): Ty => ({ k: 'named', name, args }); +const fmt = (t: Ty): string => t.k === 'lit' ? ({ int: `IntLiteral[${t.text}]`, float: `FloatLiteral[${t.text}]`, string: `StringLiteral["${t.text}"]`, bool: 'Bool', none: 'None' })[t.kind] : t.args.length ? `${t.name}[${t.args.map(fmt).join(', ')}]` : t.name; +/** A literal materializes to its default type: an int literal is an Int, and so on. */ +const concrete = (t: Ty | null): Ty | null => !t ? null : t.k === 'named' ? t : t.kind === 'int' ? named('Int') : t.kind === 'float' ? named('Float64') : t.kind === 'string' ? named('String') : t.kind === 'bool' ? named('Bool') : null; + +function exprStart(e: Expr): Pos { + let x: Expr = e; + for (; ;) { + if (x.k === 'call') x = x.fn; + else if (x.k === 'index' || x.k === 'attr') x = x.obj; + else if (x.k === 'binary' || x.k === 'bool') x = x.l; + else if (x.k === 'compare') x = x.operands[0]; + else return x.pos; + } +} +/** Where the compiler points at an expression's value: an operator's position for arithmetic, else its start. */ +const valueAt = (e: Expr): Pos => (e.k === 'binary' ? e.pos : e.k === 'call' ? e.lparen : exprStart(e)); + +export function check3(mod: Module): Finding3[] { + if (mod.unread) return []; + const out: Finding3[] = []; + const err = (p: Pos, message: string, id: string, question?: Finding3['question']) => out.push({ severity: 'error', line: p.line, col: p.col, message, id, question }); + const note = (p: Pos, message: string) => out.push({ severity: 'note', line: p.line, col: p.col, message, id: 'note' }); + + /* ---------------- the module's own types ---------------- */ + + const count = new Map<string, number>(); + for (const s of mod.body) if (s.k === 'def' || s.k === 'struct' || s.k === 'trait') count.set(s.name, (count.get(s.name) ?? 0) + 1); + const structs = new Map<string, StructStmt>(); + const traits = new Map<string, StructStmt>(); + const functions = new Map<string, DefStmt>(); + for (const s of mod.body) { + if ((s.k === 'struct' || s.k === 'trait') && count.get(s.name) === 1 && !s.typeParams.length) (s.k === 'struct' ? structs : traits).set(s.name, s); + if (s.k === 'def' && count.get(s.name) === 1 && !s.typeParams.length) functions.set(s.name, s); + } + const imported = new Set<string>(); + for (const s of mod.body) { if (s.k === 'from') for (const n of s.names) imported.add(n.alias ?? n.name); if (s.k === 'import') imported.add(s.alias ?? s.module.split('.')[0]); } + + /** Everything a struct has, or null when it cannot be known (an unknown trait, __getattr__). */ + const memberCache = new Map<string, Set<string> | null>(); + function members(name: string): Set<string> | null { + if (memberCache.has(name)) return memberCache.get(name)!; + const st = structs.get(name); + let result: Set<string> | null = null; + if (st && !st.decorators.some((d) => !(d.k === 'name' && d.id === 'fieldwise_init'))) { + const m = new Set<string>(); + for (const b of st.body) { + if (b.k === 'var') for (const n of b.targets.k === 'name' ? [b.targets.id] : []) m.add(n); + if (b.k === 'def' || b.k === 'struct') m.add(b.name); + } + let known = !m.has('__getattr__') && !m.has('__getattribute__'); + const visit = (trait: string, seen: Set<string>) => { + if (seen.has(trait)) return; seen.add(trait); + const user = traits.get(trait); + if (user) { + for (const b of user.body) if (b.k === 'def' && b.body.some((x) => !(x.k === 'expr' && (x.e.k === 'ellipsis' || x.e.k === 'str')))) m.add(b.name); + for (const base of user.bases) if (base.k === 'name') visit(base.id, seen); else known = false; + return; + } + const std = MOJO_STD_TRAITS[trait]; + if (!std) { known = false; return; } + for (const d of std.defaults) m.add(d); + for (const base of std.bases) visit(base, seen); + }; + for (const b of st.bases) { if (b.k === 'name') visit(b.id, new Set()); else known = false; } + if (known) result = m; + } + memberCache.set(name, result); + return result; + } + + function reaches(trait: string, goal: string, seen = new Set<string>()): boolean | null { + if (trait === goal) return true; + if (seen.has(trait)) return false; + seen.add(trait); + const user = traits.get(trait); + if (user) { let any = false; for (const b of user.bases) { if (b.k !== 'name') return null; const r = reaches(b.id, goal, seen); if (r === null) return null; any ||= r; } return any; } + const std = MOJO_STD_TRAITS[trait]; + if (!std) return null; + return std.bases.some((b) => reaches(b, goal, seen) === true); + } + /** Whether a type is implicitly copyable: true, false, or null for not known. */ + function implicitlyCopyable(t: Ty): boolean | null { + if (t.k !== 'named') return null; + if (SCALARS.has(t.name) || t.name === 'String') return true; + const st = structs.get(t.name); + if (st) { + if (st.decorators.length && !st.decorators.every((d) => d.k === 'name' && d.id === 'fieldwise_init')) return null; + let r = false; + for (const b of st.bases) { if (b.k !== 'name') return null; const x = reaches(b.id, 'ImplicitlyCopyable'); if (x === null) return null; r ||= x; } + return r; + } + if (imported.has(t.name) && !MOJO_STD_STRUCTS[t.name]) return null; + const std = MOJO_STD_STRUCTS[t.name]; + return std ? std.implicitlyCopyable : null; + } + const copyable = (t: Ty): boolean => { + if (t.k !== 'named') return false; + const st = structs.get(t.name); + if (st) return st.bases.some((b) => b.k === 'name' && reaches(b.id, 'Copyable') === true); + return !!MOJO_STD_STRUCTS[t.name]?.bases.some((b) => reaches(b, 'Copyable') === true); + }; + + /* ---------------- types of expressions ---------------- */ + + function annotation(e: Expr | null): Ty | null { + if (!e) return null; + if (e.k === 'name') return named(e.id); + if (e.k === 'index' && e.obj.k === 'name' && e.args.every((a) => a.name === null)) { + const args = e.args.map((a) => annotation(a.value)); + return args.every(Boolean) ? named(e.obj.id, args as Ty[]) : null; + } + return null; + } + + function typeOf(e: Expr, env: Map<string, Ty | null>): Ty | null { + switch (e.k) { + case 'num': return /^\d+$/.test(e.text) ? { k: 'lit', kind: 'int', text: e.text } : e.float ? { k: 'lit', kind: 'float', text: e.text } : null; + case 'str': return !e.t && /^[^"\\\n]*$/.test(e.value) ? { k: 'lit', kind: 'string', text: e.value } : null; + case 'name': + if (e.id === 'True' || e.id === 'False') return named('Bool'); + if (e.id === 'None') return { k: 'lit', kind: 'none', text: 'None' }; + return env.get(e.id) ?? null; + case 'call': { + if (e.fn.k === 'name' && !env.has(e.fn.id)) { + if (structs.has(e.fn.id)) return named(e.fn.id); + const f = functions.get(e.fn.id); + if (f) return annotation(f.ret); + if (e.fn.id === 'len') return named('Int'); + } + if (e.fn.k === 'index' && e.fn.obj.k === 'name' && ['List', 'Dict', 'Set', 'Optional'].includes(e.fn.obj.id) && !env.has(e.fn.obj.id)) return annotation(e.fn); + return null; + } + // an unannotated list literal is a fixed-size Array in Mojo, not a List (probes/list-ops: + // "'Array[Int, Int(3)]' value has no attribute 'append'"); its exact type is not modeled + case 'list': return null; + case 'attr': { + const o = typeOf(e.obj, env); + if (o?.k !== 'named') return null; + const st = structs.get(o.name); + const field = st?.body.find((b): b is Extract<Stmt, { k: 'var' }> => b.k === 'var' && b.targets.k === 'name' && b.targets.id === e.name); + return field ? annotation(field.type) : null; + } + case 'binary': { + const l = concrete(typeOf(e.l, env)), r = concrete(typeOf(e.r, env)); + if (l?.k === 'named' && r?.k === 'named' && l.name === 'Int' && r.name === 'Int' && ARITH[e.op] && e.op !== '@') return named('Int'); + if (l?.k === 'named' && r?.k === 'named' && l.name === 'Float64' && r.name === 'Float64' && ['+', '-', '*', '/'].includes(e.op)) return named('Float64'); + return null; + } + default: return null; + } + } + + /* ---------------- the checks ---------------- */ + + interface Ctx { def: DefStmt | null; canRaise: boolean; ret: Ty | null } + + function expr(e: Expr, env: Map<string, Ty | null>, ctx: Ctx, store = false): void { + switch (e.k) { + case 'attr': { + expr(e.obj, env, ctx); + const o = typeOf(e.obj, env); + if (o?.k === 'named' && !store) { + const m = members(o.name); + if (m && !m.has(e.name)) err(e.dot, `'${o.name}' value has no attribute '${e.name}'`, 'no-attribute'); + } + return; + } + case 'call': { + // `x[1](2)` on a struct with a parametric __call__ binds a parameter, it does not subscript + if (e.fn.k === 'index') { expr(e.fn.obj, env, ctx); for (const a of e.fn.args) expr(a.value, env, ctx); } + else expr(e.fn, env, ctx); + for (const a of e.args) expr(a.value, env, ctx); + call(e, env); + return; + } + case 'index': { + expr(e.obj, env, ctx); + for (const a of e.args) expr(a.value, env, ctx); + const o = typeOf(e.obj, env); + if (o?.k === 'named') { + const m = members(o.name); + // the message names both methods: only a struct with neither (nor __getitem_param__) is not subscriptable + if (m && ![...m].some((x) => x.startsWith('__getitem') || x.startsWith('__setitem'))) err(e.lbracket, `'${o.name}' is not subscriptable, it does not implement the \`__getitem__\`/\`__setitem__\` methods`, 'not-subscriptable'); + // reading a missing key raises + if (o.name === 'Dict' && !store && !ctx.canRaise && ctx.def) { + err(e.lbracket, 'cannot call function that may raise in a context that cannot raise', 'raises'); + note(e.lbracket, "try surrounding the call in a 'try' block"); + note(ctx.def.pos, "or mark surrounding function as 'raises'"); + } + } + return; + } + case 'binary': { + expr(e.l, env, ctx); expr(e.r, env, ctx); + const l = typeOf(e.l, env), r = typeOf(e.r, env); + if (l?.k === 'named' && r?.k === 'named' && ARITH[e.op]) { + const lm = members(l.name), rm = members(r.name); + if (lm && rm && !lm.has(ARITH[e.op]) && !rm.has(REFLECTED[e.op])) err(e.pos, `'${l.name}' does not implement the '${ARITH[e.op]}' method`, 'no-operator'); + // two different scalar types meet without a conversion + if (NUMERIC.has(l.name) && NUMERIC.has(r.name) && l.name !== r.name && ['+', '-', '*', '/', '//', '%'].includes(e.op)) + err(exprStart(e.r), `invalid call to '${ARITH[e.op]}': value passed to 'rhs' cannot be converted from '${r.name}' to '${l.name}'`, 'mixed-arithmetic'); + } + return; + } + case 'compare': { + e.operands.forEach((x) => expr(x, env, ctx)); + e.ops.forEach((op, i) => { + const l = typeOf(e.operands[i], env), r = typeOf(e.operands[i + 1], env); + if (!COMPARE[op] || l?.k !== 'named' || r?.k !== 'named' || l.name !== r.name) return; + const m = members(l.name); + const need = op === '!=' ? ['__ne__', '__eq__'] : [COMPARE[op]]; + if (m && !need.some((d) => m.has(d))) err(e.opPos[i], `'${l.name}' does not implement the '${COMPARE[op]}' method`, 'no-operator'); + }); + return; + } + case 'unary': { + expr(e.x, env, ctx); + const t = typeOf(e.x, env); + if (t?.k === 'named' && UNARY[e.op]) { const m = members(t.name); if (m && !m.has(UNARY[e.op])) err(e.pos, `'${t.name}' does not implement the '${UNARY[e.op]}' method`, 'no-operator'); } + return; + } + case 'list': case 'set': case 'tuple': e.items.forEach((x) => expr(x, env, ctx)); return; + case 'dict': e.entries.forEach(([k, v]) => { expr(k, env, ctx); expr(v, env, ctx); }); return; + case 'not': case 'transfer': expr(e.x, env, ctx); return; + case 'bool': expr(e.l, env, ctx); expr(e.r, env, ctx); return; + case 'ifexp': expr(e.cond, env, ctx); expr(e.a, env, ctx); expr(e.b, env, ctx); return; + case 'record': e.fields.forEach((f) => expr(f.value, env, ctx)); return; + default: return; + } + } + + /** A value copied where it is not implicitly copyable: `var b = a`, `b = a`, a `var` argument. */ + function copied(v: Expr, env: Map<string, Ty | null>): void { + if (v.k !== 'name') return; + const t = env.get(v.id); + if (!t || implicitlyCopyable(t) !== false) return; + const at = { line: v.pos.line, col: v.pos.col + v.id.length, endCol: v.pos.col + v.id.length }; + const options: QuickFix[] = [{ label: "transferring the value with '^'", line: at.line, col: at.col, endCol: at.col, text: '^' }]; + if (copyable(t)) options.push({ label: "copy it explicitly with '.copy()'", line: at.line, col: at.col, endCol: at.col, text: '.copy()' }); + err(v.pos, `value of type '${fmt(t)}' cannot be implicitly copied, it does not conform to 'ImplicitlyCopyable'`, 'implicit-copy', { text: 'The compiler suggests:', options }); + } + + /** + * A t-string is a TString, not a String; print takes one, a String does not. The compiler names its type by + * where the interpolated values live, not by their types (1.1.0 said 'TString[String, Int]'): for + * `t"{name}: {score}"` with parameters name: String and score: Int, Mojo 1.2.0 says + * 'TString[origin_of(anonymous*, name), origin_of(__call_result_tmp__)]' (stress/string-builder). The first list + * holds each value's origin, sorted by character code: a variable's name once however often it is read, and + * `anonymous*` once for every read of a parameter the function receives by value in a register (a number or a + * Bool with no convention written). Anything else interpolated (a call, a field, an index, a literal) has a + * spelling this does not model, and then nothing is said. Returns the type's text, or null. + */ + function tstringType(e: Extract<Expr, { k: 'str' }>, env: Map<string, Ty | null>, def: DefStmt | null): string | null { + const origins: string[] = []; + const raw = e.value; + for (let i = 0; i < raw.length; i++) { + if (raw.startsWith('{{', i) || raw.startsWith('}}', i)) { i++; continue; } + if (raw[i] !== '{') continue; + let d = 0, j = i; + for (; j < raw.length; j++) { if (raw[j] === '{') d++; else if (raw[j] === '}' && --d === 0) break; } + const m = parseModule(`def __t():\n return (${raw.slice(i + 1, j)})\n`); + const r = m.unread ? null : (m.body[0] as any)?.body?.[0]; + const v: Expr | null = r?.k === 'return' ? r.value : null; + if (v?.k !== 'name' || !env.has(v.id)) return null; + const t = env.get(v.id); + const param = def?.params.find((p) => p.name === v.id); + // a local is named whatever its type; a parameter's type decides whether it arrived in a register + if (param && t?.k !== 'named') return null; + const inRegister = !!param && !param.convention && t?.k === 'named' && SCALARS.has(t.name); + if (inRegister) origins.push('anonymous*'); + else if (!origins.includes(v.id)) origins.push(v.id); + i = j; + } + if (!origins.length) return null; + origins.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + return `TString[origin_of(${origins.join(', ')}), origin_of(__call_result_tmp__)]`; + } + + /** A value of a known type where a declared type wants another, and no conversion exists. */ + function converts(v: Expr, want: Ty | null, env: Map<string, Ty | null>, def: DefStmt | null): void { + if (want?.k !== 'named') return; + if (v.k === 'str' && v.t && want.name === 'String') { + const tt = tstringType(v, env, def); + if (tt) err(v.pos, `cannot implicitly convert '${tt}' value to 'String'`, 'conversion'); + return; + } + const got = typeOf(v, env); + if (!got) return; + const target = want.name; + if (got.k === 'lit' && got.kind === 'string' && SCALARS.has(target)) err(exprStart(v), `cannot implicitly convert '${fmt(got)}' value to '${target}'`, 'conversion'); + else if (got.k === 'lit' && got.kind === 'none' && (SCALARS.has(target) || target === 'String')) err(exprStart(v), `cannot implicitly convert 'None' value to '${target}'`, 'conversion'); + else if (got.k === 'named' && got.name === 'Int' && target === 'Float64' && v.k !== 'num') err(valueAt(v), `cannot implicitly convert 'Int' value to 'Float64'`, 'conversion'); + } + + function call(e: Extract<Expr, { k: 'call' }>, env: Map<string, Ty | null>): void { + if (e.fn.k !== 'name' || env.has(e.fn.id) || e.args.some((a) => a.star)) return; + const name = e.fn.id; + // List(), Set(), Dict(): nothing says what they hold + if (['List', 'Set', 'Dict'].includes(name) && !e.args.length && !structs.has(name)) { pendingBareCollection.push(e); return; } + const f = functions.get(name); + if (f) { + const ps = f.params.filter((p) => p.convention !== 'out' && p.star !== 'marker' && p.star !== 'positional-marker' && !p.star); + const kwOnlyFrom = f.params.findIndex((p) => p.star === 'marker' || p.star === '*'); + e.args.filter((a) => a.name === null).forEach((a, i) => { + const p = ps[i]; + if (!p || (kwOnlyFrom >= 0 && f.params.indexOf(p) > kwOnlyFrom)) return; + if (p.convention === 'var' || p.convention === 'owned') copied(a.value, env); + const want = annotation(p.type); + const got = typeOf(a.value, env); + // a literal where the student's own struct is wanted, and the struct has no @implicit constructor + if (want?.k === 'named' && structs.has(want.name) && got?.k === 'lit' && (got.kind === 'int' || got.kind === 'string') && !hasImplicitInit(want.name)) + err(exprStart(a.value), `invalid call to '${name}': value passed to '${p.name}' cannot be converted from '${fmt(got)}' to '${want.name}'`, 'argument-conversion'); + }); + return; + } + const st = structs.get(name); + if (st) constructByType(e, st, env); + } + + function hasImplicitInit(name: string): boolean { + const st = structs.get(name)!; + return st.body.some((b) => b.k === 'def' && b.name === '__init__' && b.decorators.some((d) => d.k === 'name' && d.id === 'implicit')); + } + + /** Every constructor whose arity fits has a parameter of a scalar type given the student's struct: no match. */ + function constructByType(e: Extract<Expr, { k: 'call' }>, st: StructStmt, env: Map<string, Ty | null>): void { + if (e.args.some((a) => a.name !== null) || !st.bases.some((b) => b.k === 'name' && ['Copyable', 'Movable', 'ImplicitlyCopyable'].includes(b.id))) return; + const inits = st.body.filter((b): b is DefStmt => b.k === 'def' && b.name === '__init__'); + if (!inits.length || st.decorators.length || inits.some((d) => d.typeParams.length)) return; + const n = e.args.length; + const fitting = inits.map((d) => d.params.filter((p: Param) => !(p.convention === 'out' || p.name === 'self') && !p.star)).filter((ps) => ps.length >= ps.filter((p) => !p.default).length && n >= ps.filter((p) => !p.default).length && n <= ps.length); + if (!fitting.length) return; + const mismatch = (ps: Param[]) => e.args.some((a, i) => { + const want = annotation(ps[i].type), got = typeOf(a.value, env); + return want?.k === 'named' && (SCALARS.has(want.name) || want.name === 'String') && got?.k === 'named' && structs.has(got.name) && got.name !== want.name; + }); + if (fitting.every(mismatch)) err(e.lparen, 'no matching function in initialization', 'no-matching-init'); + } + + const pendingBareCollection: Array<Extract<Expr, { k: 'call' }>> = []; + + function stmts(list: Stmt[], env: Map<string, Ty | null>, ctx: Ctx): void { for (const s of list) stmt(s, env, ctx); } + + function stmt(s: Stmt, env: Map<string, Ty | null>, ctx: Ctx): void { + switch (s.k) { + case 'var': { + if (s.value) { + pendingBareCollection.length = 0; + expr(s.value, env, ctx); + // List() with no annotation and nothing else to say what it holds + if (!s.type && s.value.k === 'call' && pendingBareCollection.includes(s.value)) err(s.value.lparen, 'no matching function in initialization', 'no-matching-init'); + pendingBareCollection.length = 0; + if (s.type) converts(s.value, annotation(s.type), env, ctx.def); + if (!s.ref) copied(s.value, env); // `ref view = data` binds, it does not copy + } + if (s.targets.k === 'name') env.set(s.targets.id, s.type ? annotation(s.type) : s.value ? concrete(typeOf(s.value, env)) : null); + return; + } + case 'assign': { + expr(s.value, env, ctx); + for (const t of s.targets) { if (t.k !== 'name') expr(t, env, ctx, true); else if (env.has(t.id)) { converts(s.value, env.get(t.id) ?? null, env, ctx.def); copied(s.value, env); } } + return; + } + case 'aug': expr(s.value, env, ctx); expr(s.target, env, ctx); return; + case 'expr': expr(s.e, env, ctx); return; + case 'return': if (s.value) { expr(s.value, env, ctx); converts(s.value, ctx.ret, env, ctx.def); } return; + case 'raise': if (s.value) expr(s.value, env, ctx); return; + case 'assert': expr(s.test, env, ctx); return; + case 'if': case 'while': expr(s.test, env, ctx); stmts(s.body, env, ctx); stmts(s.orelse, env, ctx); return; + case 'for': { + expr(s.iter, env, ctx); + for (const n of s.target.k === 'name' ? [s.target.id] : []) env.set(n, null); + stmts(s.body, env, ctx); stmts(s.orelse, env, ctx); + return; + } + case 'try': stmts(s.body, env, { ...ctx, canRaise: ctx.canRaise || s.handlers.length > 0 }); for (const h of s.handlers) { if (h.name) env.set(h.name, null); stmts(h.body, env, ctx); } stmts(s.orelse, env, ctx); stmts(s.finalbody, env, ctx); return; + case 'with': for (const it of s.items) expr(it.e, env, ctx); stmts(s.body, env, ctx); return; + default: return; + } + } + + function fn(d: DefStmt, owner: StructStmt | null): void { + const env = new Map<string, Ty | null>(); + d.params.forEach((p, i) => { + if (p.star) { env.set(p.name, null); return; } + env.set(p.name, owner && (p.name === 'self' || i === 0) && !p.type ? named(owner.name) : annotation(p.type)); + }); + for (const tp of d.typeParams) env.set(tp.name, null); + stmts(d.body, env, { def: d, canRaise: d.raises && !(d.raisesType?.k === 'name' && d.raisesType.id === 'Never'), ret: annotation(d.ret) }); + } + + for (const s of mod.body) { + if (s.k === 'def') fn(s, null); + if (s.k === 'struct' && !s.typeParams.length && count.get(s.name) === 1) for (const b of s.body) if (b.k === 'def') fn(b, s); + } + functionTypedArguments(); + return out; + + /** + * A function type written as an ordinary parameter's type (`f: def(Int) -> Int`) is a type, not a + * value's: in Mojo a function is passed through a type parameter (`F: def(Int) -> Int, //` with + * `f: F`) or a compile-time parameter (`[f: def(Int) thin -> Int]`), as the manual's lambda tests do. + * The compiler's words, from godbolt (stress/lambda-map, stress/lambda-capture-param): at each call + * through it, and at each value passed to it. + */ + function functionTypedArguments(): void { + const text = (e: Expr | null): string | null => { + if (!e) return null; + if (e.k === 'name') return e.id; + if (e.k === 'index' && e.obj.k === 'name') { const a = e.args.map((x) => text(x.value)); return a.every(Boolean) ? `${e.obj.id}[${a.join(', ')}]` : null; } + return null; + }; + const lambdas: Array<Extract<Expr, { k: 'lambda' }>> = []; + const findLambdas = (x: any) => { if (!x || typeof x !== 'object') return; if (Array.isArray(x)) { x.forEach(findLambdas); return; } if (x.k === 'lambda') lambdas.push(x); for (const v of Object.values(x)) if (v && typeof v === 'object') findLambdas(v); }; + findLambdas(mod.body); + for (const d of functions.values()) { + if (d.typeParams.length) continue; + const fnParams = d.params.filter((p) => !p.star && p.type?.k === 'opaque' && p.type.what === 'def' && p.type.fn); + if (fnParams.length !== 1) continue; + const p = fnParams[0]; + const ft = (p.type as Extract<Expr, { k: 'opaque' }>).fn!; + if (ft.effects.length) continue; + const ptypes = ft.params.map(text), ret = text(ft.ret); + if (ptypes.some((x) => !x) || !ret) continue; + const target = `def(${ptypes.join(', ')}) -> ${ret}`; + // each call through it, inside the function + const calls = (x: any) => { if (!x || typeof x !== 'object') return; if (Array.isArray(x)) { x.forEach(calls); return; } if (x.k === 'def' || x.k === 'lambda') return; if (x.k === 'call' && x.fn.k === 'name' && x.fn.id === p.name) err(x.lparen, "invalid call to '__call__': value cannot be converted from type value '$0' to an instance of '$0'; did you mean to instantiate '$0'?", 'function-type-argument'); for (const v of Object.values(x)) if (v && typeof v === 'object') calls(v); }; + calls(d.body); + // each value passed to it, at the call sites + const index = d.params.filter((q) => q.star !== 'marker' && q.star !== 'positional-marker').indexOf(p); + const sites = (x: any, enclosing: DefStmt | null) => { + if (!x || typeof x !== 'object') return; + if (Array.isArray(x)) { x.forEach((y) => sites(y, enclosing)); return; } + if (x.k === 'def') { sites(x.body, x); return; } + if (x.k === 'call' && x.fn.k === 'name' && x.fn.id === d.name && !x.args.some((a: any) => a.name || a.star)) { + const a = x.args[index]; + const kind = a ? valueFunctionType(a.value, enclosing) : null; + if (a && kind) err(exprStartOf(a.value), `invalid call to '${d.name}': value passed to '${p.name}' cannot be converted from '${kind}' to '${target}'`, 'function-type-argument'); + } + for (const v of Object.values(x)) if (v && typeof v === 'object') sites(v, enclosing); + }; + sites(mod.body, null); + } + /** How the compiler spells a function value's type, for the shapes godbolt showed; else null. */ + function valueFunctionType(e: Expr, enclosing: DefStmt | null): string | null { + if (e.k === 'name') { + const g = functions.get(e.id); + if (!g || g.typeParams.length || g.raises || !g.ret) return null; + const ps = g.params.filter((q) => q.star !== 'marker' && q.star !== 'positional-marker'); + if (ps.some((q) => q.star || q.convention || q.default || !text(q.type))) return null; + const r = text(g.ret); + return r ? `def ${g.name}(${ps.map((q) => `${q.name}: ${text(q.type)}`).join(', ')}) thin -> ${r}` : null; + } + if (e.k === 'lambda' && e.typed && e.ret && !e.captures) { + const ps = e.typed; + if (ps.some((q) => q.star || q.convention || q.default || !text(q.type))) return null; + const r = text(e.ret); + if (!r) return null; + const sig = `(${ps.map((q) => `${q.name}: ${text(q.type)}`).join(', ')})`; + // does it use a local of the function around it? then it captures, and has no name or thin + const locals = new Set<string>(); + if (enclosing) { for (const q of enclosing.params) locals.add(q.name); const v = (x: any) => { if (!x || typeof x !== 'object') return; if (Array.isArray(x)) { x.forEach(v); return; } if (x.k === 'var' && x.targets?.k === 'name') locals.add(x.targets.id); for (const y of Object.values(x)) if (y && typeof y === 'object') v(y); }; v(enclosing.body); } + const used = new Set<string>(); const u = (x: any) => { if (!x || typeof x !== 'object') return; if (Array.isArray(x)) { x.forEach(u); return; } if (x.k === 'name') used.add(x.id); for (const y of Object.values(x)) if (y && typeof y === 'object') u(y); }; u(e.body); + const captures = [...used].some((n) => locals.has(n) && !ps.some((q) => q.name === n)); + if (captures) return `def${sig} -> ${r}`; + return lambdas.length === 1 ? `def \`lambda_0${sig} thin -> ${r}` : null; // the numbering of several is not verified + } + return null; + } + } +} + +function exprStartOf(e: Expr): Pos { return exprStart(e); } diff --git a/src/mojo/values.ts b/src/mojo/values.ts new file mode 100644 index 0000000..57a8b9e --- /dev/null +++ b/src/mojo/values.ts @@ -0,0 +1,246 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The values Programming+'s Mojo interpreter computes with, and how the pinned + * Mojo prints them. Every rule here is a probe's answer (library/mojo/probes/, asked + * of godbolt's compiler at the pin; none of them changed from 1.1.0 to 1.2.0), not a + * guess from Python: + * + * Int wraps at 64 bits, Int8/UInt8/... at their width int-overflow-*, int-uint-wrap + * / truncates on Int; // floors; % takes the divisor's sign int-truediv, int-floordiv, int-mod + * an Int division by zero is 0 int-div-zero + * Float64 prints as Python's repr (0.30000000000000004, 1e+20, 1e-07, -0.0) float-print + * Float32 prints its shortest round-trip digits (0.1, 0.33333334) float32-print + * round() rounds half to even and stays a float float-round + * True/False; None; [1, 2] and [a, b] (no quotes); {a: 1}; (1, a, 2.5) print-* + */ + +export type V = + | { k: 'int'; v: bigint; width: number; signed: boolean; type: string } + | { k: 'float'; v: number; bits: 32 | 64 } + | { k: 'bool'; v: boolean } + | { k: 'str'; v: string } + | { k: 'none' } + | { k: 'list'; items: V[]; elem: string; fixed: boolean; cap?: number } + | { k: 'dict'; map: Map<string, [V, V]>; key: string; val: string } + | { k: 'set'; map: Map<string, V>; elem: string } + | { k: 'tuple'; items: V[] } + | { k: 'opt'; v: V | null; elem: string } + | { k: 'struct'; name: string; fields: Map<string, V | undefined> } + | { k: 'error'; msg: string; kind?: string } + | { k: 'writer'; out: string[] } + | { k: 'fn'; name: string; self?: { get(): V | undefined; set(v: V | undefined): void } } + | { k: 'type'; name: string; args: string[] } + | { k: 'range'; start: bigint; stop: bigint; step: bigint } + /** A closure: a nested def (or lambda) with what it captured: a reference to an outer variable, or its own copy. */ + | { k: 'closure'; name: string; def: any; captures: Map<string, { loc?: any; cell?: any }> } + /** SIMD[dtype, width]: one scalar value per lane, all of the dtype's type. */ + | { k: 'simd'; dtype: string; items: V[] }; + +/** The integer types and their widths. Int and UInt are 64 bits on the machines godbolt runs. */ +export const INT_TYPES: Record<string, { width: number; signed: boolean }> = { + Int: { width: 64, signed: true }, UInt: { width: 64, signed: false }, + Int8: { width: 8, signed: true }, Int16: { width: 16, signed: true }, Int32: { width: 32, signed: true }, Int64: { width: 64, signed: true }, + UInt8: { width: 8, signed: false }, UInt16: { width: 16, signed: false }, UInt32: { width: 32, signed: false }, UInt64: { width: 64, signed: false }, +}; + +export function int(v: bigint, type = 'Int'): V { + const t = INT_TYPES[type] ?? INT_TYPES.Int; + return { k: 'int', v: t.signed ? BigInt.asIntN(t.width, v) : BigInt.asUintN(t.width, v), width: t.width, signed: t.signed, type }; +} +export const float = (v: number, bits: 32 | 64 = 64): V => ({ k: 'float', v: bits === 32 ? Math.fround(v) : v, bits }); +export const bool = (v: boolean): V => ({ k: 'bool', v }); +export const str = (v: string): V => ({ k: 'str', v }); +export const NONE: V = { k: 'none' }; + +/** Floor division and a modulo with the divisor's sign, on BigInts; dividing by zero gives 0 (int-div-zero). */ +export function floordiv(a: bigint, b: bigint): bigint { + if (b === 0n) return 0n; + const q = a / b; + return (a % b !== 0n && (a < 0n) !== (b < 0n)) ? q - 1n : q; +} +export function mod(a: bigint, b: bigint): bigint { + if (b === 0n) return 0n; + const r = a % b; + return r !== 0n && (r < 0n) !== (b < 0n) ? r + b : r; +} +export function truncdiv(a: bigint, b: bigint): bigint { return b === 0n ? 0n : a / b; } + +/** Python's repr of a double: shortest round-trip digits, fixed notation for exponents -4..15. */ +export function reprFloat(x: number, shortest?: (x: number) => string): string { + if (Number.isNaN(x)) return 'nan'; + if (!Number.isFinite(x)) return x > 0 ? 'inf' : '-inf'; + if (x === 0) return Object.is(x, -0) ? '-0.0' : '0.0'; + const e = shortest ? shortest(x) : x.toExponential(); // "d.ddde±X", the shortest digits that round-trip + const m = /^(-?)(\d)(?:\.(\d+))?e([+-]\d+)$/.exec(e)!; + const sign = m[1], digits = m[2] + (m[3] ?? ''), exp = Number(m[4]); + if (exp < -4 || exp >= 16) { + const mant = digits.length > 1 ? `${digits[0]}.${digits.slice(1)}` : digits; + const ex = Math.abs(exp) < 10 ? `0${Math.abs(exp)}` : String(Math.abs(exp)); + return `${sign}${mant}e${exp < 0 ? '-' : '+'}${ex}`; + } + if (exp < 0) return `${sign}0.${'0'.repeat(-exp - 1)}${digits}`; + const intPart = digits.slice(0, exp + 1).padEnd(exp + 1, '0'); + const frac = digits.slice(exp + 1); + return `${sign}${intPart}.${frac || '0'}`; +} + +/** A Float32 printed with the fewest digits that read back as the same Float32 (float32-print: 0.1, 0.33333334). */ +export function reprFloat32(x: number): string { + return reprFloat(x, (v) => { + for (let p = 1; p <= 9; p++) { const s = v.toExponential(p - 1); if (Math.fround(Number(s)) === v) return s; } + return v.toExponential(8); + }); +} + +/** + * Round half to even, as Mojo's round() does (float-round: 2.5 → 2.0, 3.5 → 4.0, -2.5 → -2.0). A result of zero + * keeps the sign of what was rounded (float-round-sign: round(-0.3) and round(-0.5) print -0.0). + */ +export function roundHalfEven(x: number): number { + const f = Math.floor(x), d = x - f; + const r = d > 0.5 ? f + 1 : d < 0.5 ? f : f % 2 === 0 ? f : f + 1; + return r === 0 && (x < 0 || Object.is(x, -0)) ? -0 : r; +} + +/** A value print() cannot write: the interpreter declines rather than print something made up. */ +export class Unprintable extends Error { constructor(readonly kind: string) { super(kind); } } + +/** + * What the trace shows of a long value: the first BRIEF_ITEMS of a collection and the first BRIEF_TEXT characters + * of a String. Every step records every variable, so a value in full would be copied into the trace at each step. + */ +const BRIEF_ITEMS = 20; +const BRIEF_TEXT = 1000; +const more = (n: number) => (n > BRIEF_ITEMS ? `, … ${n - BRIEF_ITEMS} more` : ''); +function* firstOf<T>(xs: Iterable<T>, n: number): Generator<T> { if (n <= 0) return; for (const x of xs) { yield x; if (--n <= 0) return; } } + +/** What print() writes for a value; `brief`: as the trace shows it, long collections and Strings cut short. */ +export function show(v: V, call?: (s: V) => string, brief = false): string { + const cut = brief ? BRIEF_ITEMS : Infinity; + switch (v.k) { + case 'int': return String(v.v); + case 'float': return v.bits === 32 ? reprFloat32(v.v) : reprFloat(v.v); + case 'bool': return v.v ? 'True' : 'False'; + case 'str': return brief && v.v.length > BRIEF_TEXT ? `${v.v.slice(0, BRIEF_TEXT)}… (${v.v.length.toLocaleString('en-US')} characters)` : v.v; + case 'none': return 'None'; + case 'list': return `[${[...firstOf(v.items, cut)].map((x) => show(x, call, brief)).join(', ')}${brief ? more(v.items.length) : ''}]`; + case 'tuple': return `(${[...firstOf(v.items, cut)].map((x) => show(x, call, brief)).join(', ')}${brief ? more(v.items.length) : ''})`; + case 'dict': return `{${[...firstOf(v.map.values(), cut)].map(([k, x]) => `${show(k, call, brief)}: ${show(x, call, brief)}`).join(', ')}${brief ? more(v.map.size) : ''}}`; + case 'set': return `{${[...firstOf(v.map.values(), cut)].map((x) => show(x, call, brief)).join(', ')}${brief ? more(v.map.size) : ''}}`; + case 'opt': return v.v ? show(v.v, call, brief) : 'None'; + case 'simd': return `[${[...firstOf(v.items, cut)].map((x) => show(x)).join(', ')}${brief ? more(v.items.length) : ''}]`; // probes/simd-basics: [1.0, 2.0, 3.0, 4.0] + case 'error': return v.kind ? `${v.kind}()` : v.msg; + case 'struct': return call ? call(v) : v.name; + default: throw new Unprintable(v.k); + } +} + +/** How the trace shows a value: as print would, but a string in quotes, so "3" and 3 differ. */ +export function display(v: V | undefined, call?: (s: V) => string): string { + if (v === undefined) return '(moved out)'; + if (v.k === 'fn' || v.k === 'type') return v.name; + if (v.k === 'closure') return `closure ${v.name}`; + if (v.k === 'writer') return 'Writer'; + if (v.k === 'range') return `range(${v.start}, ${v.stop}${v.step !== 1n ? `, ${v.step}` : ''})`; + if (v.k === 'str') return v.v.length > BRIEF_TEXT ? `${JSON.stringify(v.v.slice(0, BRIEF_TEXT)).slice(0, -1)}…" (${v.v.length.toLocaleString('en-US')} characters)` : JSON.stringify(v.v); + if (v.k === 'list') return `[${v.items.slice(0, BRIEF_ITEMS).map((x) => display(x, call)).join(', ')}${more(v.items.length)}]`; + if (v.k === 'struct') return `${v.name}(${[...v.fields].map(([n, x]) => `${n}=${display(x, call)}`).join(', ')})`; + return show(v, call, true); +} + +/** How many values a copy of `v` makes: the elements of its collections, nested ones included (a scalar: none). */ +export function elementsIn(v: V): number { + switch (v.k) { + case 'list': case 'tuple': case 'simd': { let n = v.items.length; for (const x of v.items) if (x.k !== 'int' && x.k !== 'float' && x.k !== 'bool' && x.k !== 'str') n += elementsIn(x); return n; } + case 'dict': { let n = v.map.size; for (const [a, b] of v.map.values()) n += elementsIn(a) + elementsIn(b); return n; } + case 'set': { let n = v.map.size; for (const x of v.map.values()) n += elementsIn(x); return n; } + case 'opt': return v.v ? elementsIn(v.v) : 0; + case 'struct': { let n = 0; for (const x of v.fields.values()) if (x) n += elementsIn(x); return n; } + default: return 0; + } +} + +/** The type the trace names for a value. */ +export function typeName(v: V | undefined): string { + if (!v) return ''; + switch (v.k) { + case 'int': return v.type; + case 'float': return v.bits === 32 ? 'Float32' : 'Float64'; + case 'bool': return 'Bool'; + case 'str': return 'String'; + case 'none': return 'NoneType'; + case 'list': return v.fixed ? `Array[${v.elem}, ${v.items.length}]` : `List[${v.elem}]`; + case 'dict': return `Dict[${v.key}, ${v.val}]`; + case 'set': return `Set[${v.elem}]`; + case 'tuple': return 'Tuple'; + case 'opt': return `Optional[${v.elem}]`; + case 'simd': return `SIMD[DType.${v.dtype}, ${v.items.length}]`; + case 'struct': return v.name; + case 'error': return 'Error'; + default: return v.k; + } +} + +/** A key for a dict or set: equal values, equal keys. */ +export function keyOf(v: V): string { + switch (v.k) { + case 'int': return `i${v.v}`; + case 'float': return `f${v.v}`; + case 'bool': return `b${v.v}`; + case 'str': return `s${v.v}`; + case 'tuple': return `t(${v.items.map(keyOf).join(',')})`; + // by its fields: right for Equatable's and Hashable's defaults (the interpreter declines a struct with its own) + case 'struct': return `S${v.name}(${[...v.fields].map(([n, x]) => `${n}=${x ? keyOf(x) : '_'}`).join(',')})`; + default: throw new Unprintable(`${v.k} as a key`); + } +} + +/** Mojo's value semantics: a copy shares nothing with the original. */ +export function deepCopy(v: V): V { + switch (v.k) { + case 'list': return { ...v, items: v.items.map(deepCopy) }; + case 'tuple': return { ...v, items: v.items.map(deepCopy) }; + case 'dict': return { ...v, map: new Map([...v.map].map(([k, [a, b]]) => [k, [deepCopy(a), deepCopy(b)]])) }; + case 'set': return { ...v, map: new Map([...v.map].map(([k, a]) => [k, deepCopy(a)])) }; + case 'opt': return { ...v, v: v.v ? deepCopy(v.v) : null }; + case 'simd': return { ...v, items: v.items.slice() }; + case 'struct': return { ...v, fields: new Map([...v.fields].map(([k, a]) => [k, a === undefined ? undefined : deepCopy(a)])) }; + default: return v; + } +} + +export function truthy(v: V): boolean { + switch (v.k) { + case 'bool': return v.v; + case 'int': return v.v !== 0n; + case 'float': return v.v !== 0; + case 'str': return v.v.length > 0; + case 'none': return false; + case 'list': case 'tuple': return v.items.length > 0; + case 'dict': case 'set': return v.map.size > 0; + case 'opt': return v.v !== null; + default: return true; + } +} + +/** Equality the interpreter is sure of; null when it does not know (a struct, a type), so it can decline. */ +export function equal(a: V, b: V): boolean | null { + if (a.k === 'int' && b.k === 'int') return a.v === b.v; + if (a.k === 'float' && b.k === 'float') return a.v === b.v; + if ((a.k === 'int' && b.k === 'float') || (a.k === 'float' && b.k === 'int')) return Number((a as any).v) === Number((b as any).v); + if (a.k === 'bool' && b.k === 'bool') return a.v === b.v; + if (a.k === 'str' && b.k === 'str') return a.v === b.v; + if (a.k === 'none' && b.k === 'none') return true; + if ((a.k === 'list' && b.k === 'list') || (a.k === 'tuple' && b.k === 'tuple')) { + if (a.items.length !== b.items.length) return false; + for (let i = 0; i < a.items.length; i++) { const e = equal(a.items[i], b.items[i]); if (e !== true) return e; } + return true; + } + if (a.k === 'str' || b.k === 'str' || a.k === 'bool' || b.k === 'bool' || a.k === 'none' || b.k === 'none') return a.k === b.k ? null : false; + return null; +} + +/** UTF-8 bytes and codepoints: Mojo strings are UTF-8, so byte and codepoint positions differ. */ +export const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s); +export const codepoints = (s: string): string[] => Array.from(s); diff --git a/src/mojo/warnings.ts b/src/mojo/warnings.ts new file mode 100644 index 0000000..7395af7 --- /dev/null +++ b/src/mojo/warnings.ts @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Warnings the compiler gives about values that are never used, in its words + * and at its positions (godbolt's answers on accepted programs, scored in + * tests/mojo-check.test.mjs). They come from the same analysis that decides + * when a value dies (src/mojo/lifetimes.ts): a value nobody reads is destroyed + * the moment it is made, which is what these warnings are about. + * + * assignment to 'x' was never used; assign to '_' instead? at the value: a call's (, a subscript's [, a literal + * ref 'i' was never used, remove it? a loop variable the body never reads + * '__del__' is deprecated; use '__deinit__' at the method's name + */ +import type { Expr, Module, Pos, Stmt } from './parse.js'; +import { lifetimes } from './lifetimes.js'; + +export interface Warning { line: number; col: number; message: string; id: string } + +/** Where the compiler points at an assigned value. */ +function valueAt(e: Expr): Pos | null { + switch (e.k) { + case 'call': return e.lparen; + case 'index': return e.lbracket; + case 'num': case 'str': case 'name': case 'list': case 'dict': case 'tuple': return e.pos; + default: return null; // not observed for other expressions: say nothing + } +} + +export function warnings(mod: Module): Warning[] { + if (mod.unread) return []; + const out: Warning[] = []; + const defs: Array<Extract<Stmt, { k: 'def' }>> = []; + for (const s of mod.body) { + if (s.k === 'def') defs.push(s); + if (s.k === 'struct') for (const b of s.body) if (b.k === 'def') { + defs.push(b); + if (b.name === '__del__') out.push({ line: b.pos.line, col: b.pos.col, message: "'__del__' is deprecated; use '__deinit__'", id: 'deprecated-del' }); + } + } + for (const d of defs) { + const life = lifetimes(d); + for (const [stmt, names] of life.unreadAfter) { + const value = stmt.k === 'var' && !stmt.comptime && !stmt.ref ? stmt.value : stmt.k === 'assign' && stmt.targets.length === 1 ? stmt.value : null; + if (!value) continue; + const at = valueAt(value); + if (!at) continue; + // a parameter's new value can be its caller's (out, mut, ref), and _names are unused on purpose + const params = new Set(d.params.map((x) => x.name)); + for (const n of names) if (!n.startsWith('_') && !params.has(n) && !life.unmodeled.has(n)) out.push({ line: at.line, col: at.col, message: `assignment to '${n}' was never used; assign to '_' instead?`, id: 'unused-assignment' }); + } + unusedLoopVars(d.body, out); + } + return out; +} + +function unusedLoopVars(body: Stmt[], out: Warning[]): void { + for (const s of body) { + if (s.k === 'for' && !s.comptime && s.target.k === 'name' && s.target.id !== '_') { + const id = s.target.id; + if (!mentions([...s.body, ...s.orelse], id)) out.push({ line: s.target.pos.line, col: s.target.pos.col, message: `ref '${id}' was never used, remove it?`, id: 'unused-loop-variable' }); + } + for (const inner of ['body', 'orelse', 'finalbody'] as const) if (Array.isArray((s as any)[inner]) && s.k !== 'def' && s.k !== 'struct') unusedLoopVars((s as any)[inner], out); + if (s.k === 'try') for (const h of s.handlers) unusedLoopVars(h.body, out); + } +} + +function mentions(x: unknown, name: string): boolean { + if (!x || typeof x !== 'object') return false; + if (Array.isArray(x)) return x.some((y) => mentions(y, name)); + const o = x as Record<string, unknown>; + if ((o.k === 'name' && o.id === name) || (o.k === 'str' && o.t && typeof o.value === 'string' && new RegExp(`\\b${name}\\b`).test(o.value))) return true; + return Object.values(o).some((v) => v && typeof v === 'object' && mentions(v, name)); +} diff --git a/src/services/asm.ts b/src/services/asm.ts new file mode 100644 index 0000000..b2d131f --- /dev/null +++ b/src/services/asm.ts @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * x86-64: wraps the course's asm86 simulator. `parse` assembles, `run` + * executes with a setup block (registers, memory, stdin) in the sandbox's + * syntax, and the trace carries per-instruction diffs plus the hardware + * model (caches, pipeline, out-of-order core) lazily through `raw.hardware`. + */ +import { assemble, run as runAsm, AsmError } from '../engines/asm86/interp.js'; +import { hardwareFor } from '../engines/asm86/machine.js'; +import { MACHINE_PRESETS, machineName, machineSpec, parseMachine } from '../common/machine.js'; +import type { Diagnostic, LanguageService, Parsed, RunOptions, Trace } from '../common/types.js'; + +export type AsmProgram = any; +export type AsmState = any; +export type AsmStep = any; + +export interface AsmSetup { registers: Record<string, bigint>; memory: any[]; flags: Record<string, boolean>; stdin: string } + +function num(tok: string): bigint { + const t = tok.trim(); + if (/^'.'$/.test(t)) return BigInt(t.charCodeAt(1)); + if (/^-?(0x[0-9a-f_]+|0b[01_]+|\d[\d_]*)$/i.test(t)) { + const neg = t.startsWith('-'); + const v = BigInt(t.replace(/^-/, '').replace(/_/g, '')); + return neg ? -v : v; + } + throw new Error(`"${t}" is not a number`); +} + +/** + * Text in double quotes, with its escapes: \n, \t, \", \\, and for a byte that is not a printable character \0 + * and \xHH (two hexadecimal digits). The listing's read-only data is written this way (src/mojo/att.ts), about one + * character a byte, where `bytes 0x70 0x6b ...` took five. Read with a loop, not one expression over the whole + * text: a block can be half a megabyte long. + */ +function unquote(s: string): string { + const t = s.trim(); + const bad = () => new Error(`expected text in double quotes, found ${t.length > 60 ? `${t.slice(0, 60)}...` : t}`); + if (t.length < 2 || t[0] !== '"' || t[t.length - 1] !== '"') throw bad(); + const out: string[] = []; + let from = 1; + for (let i = 1; i < t.length - 1; i++) { + const c = t[i]; + if (c === '"') throw bad(); + if (c !== '\\') continue; + if (i + 1 >= t.length - 1) throw bad(); // a backslash before the closing quote escapes it: the text never closes + out.push(t.slice(from, i)); + const e = t[i + 1]; + const hex = e === 'x' ? /^[0-9a-fA-F]{2}$/.exec(t.slice(i + 2, i + 4)) : null; + if (hex && i + 3 < t.length - 1) { out.push(String.fromCharCode(parseInt(hex[0], 16))); i += 3; } + else { out.push(e === 'n' ? '\n' : e === 't' ? '\t' : e === '0' ? '\0' : e === '"' ? '"' : e === '\\' ? '\\' : `\\${e}`); i += 1; } + from = i + 1; + } + out.push(t.slice(from, t.length - 1)); + return out.join(''); +} + +/** A setup line without its comment (`#`, `;` or `//` at the start or after a space), leaving one inside quoted text alone. */ +function withoutComment(raw: string): string { + let quoted = false; + for (let i = 0; i < raw.length; i++) { + const c = raw[i]; + if (quoted) { if (c === '\\') i++; else if (c === '"') quoted = false; continue; } + if (c === '"') { quoted = true; continue; } + if ((c === '#' || c === ';' || (c === '/' && raw[i + 1] === '/')) && (i === 0 || /\s/.test(raw[i - 1]))) return raw.slice(0, i); + } + return raw; +} + +/** The sandbox's setup language: `rdi = 0x1000`, `memory 0x1000 "GATTACA"`, `stdin "..."`, `flags CF ZF`. */ +export function parseSetup(text: string): AsmSetup { + const out: AsmSetup = { registers: {}, memory: [], flags: {}, stdin: '' }; + String(text || '').split('\n').forEach((raw, i) => { + const line = withoutComment(raw).trim(); + if (!line) return; + try { + let m: RegExpExecArray | null; + if ((m = /^([a-z][a-z0-9]*)\s*=\s*(.+)$/i.exec(line))) { + out.registers[m[1].toLowerCase()] = num(m[2]); + } else if ((m = /^memory\s+(\S+)\s+(.*)$/i.exec(line))) { + const addr = Number(num(m[1])); + const rest = m[2].trim(); + const name = `m${out.memory.length}_${addr.toString(16)}`; + const label = `0x${addr.toString(16)}`; + let r: RegExpExecArray | null; + if (rest.startsWith('"')) out.memory.push({ name, label, addr, text: unquote(rest) }); + else if ((r = /^bytes\s+(.*)$/i.exec(rest))) out.memory.push({ name, label, addr, bytes: r[1].trim().split(/[\s,]+/).map((b) => Number(BigInt.asUintN(8, num(b)))) }); + else if ((r = /^qwords\s+(.*)$/i.exec(rest))) { + const bytes: number[] = []; + for (const q of r[1].trim().split(/[\s,]+/)) { let v = BigInt.asUintN(64, num(q)); for (let k = 0; k < 8; k++) { bytes.push(Number(v & 0xffn)); v >>= 8n; } } + out.memory.push({ name, label, addr, bytes }); + } else if ((r = /^zero\s+(\S+)$/i.exec(rest))) out.memory.push({ name, label, addr, size: Number(num(r[1])) }); + else throw new Error('after the address, write "text", bytes ..., qwords ..., or zero n'); + } else if ((m = /^stdin\s+(.*)$/i.exec(line))) { + out.stdin = unquote(m[1]); + } else if ((m = /^flags\s+(.*)$/i.exec(line))) { + for (const f of m[1].split(/[\s,]+/)) { + if (!['ZF', 'SF', 'CF', 'OF', 'PF'].includes(f.toUpperCase())) throw new Error(`no flag named ${f}`); + out.flags[f.toUpperCase()] = true; + } + } else throw new Error('not a setup line (register = value, memory addr ..., stdin "...", flags ...)'); + } catch (e: any) { + throw new Error(`setup line ${i + 1}: ${e.message}`); + } + }); + return out; +} + +export function parseAsm(source: string): Parsed<AsmProgram> { + try { + const program = assemble(source); + const diagnostics: Diagnostic[] = []; + for (const l of program.lines || []) { + if (l.error) diagnostics.push({ lang: 'asm', severity: 'error', id: 'assemble', line: (l.lineNo ?? 0) + 1, col: 1, message: String(l.error) }); + } + return { program, diagnostics, ok: diagnostics.length === 0 }; + } catch (e: any) { + const line = e?.line != null ? e.line + 1 : e?.lineNo != null ? e.lineNo + 1 : 1; + const message = String(e.message).replace(/^line \d+:\s*/, ''); + return { program: null, diagnostics: [{ lang: 'asm', severity: 'error', id: e instanceof AsmError ? e.kind || 'assemble' : 'assemble', line, col: 1, message }], ok: false }; + } +} + +/** The machine presets; a state's `machine` may add `+paging` to one (core/machine.ts). */ +export const MACHINES = MACHINE_PRESETS; + +export function runAsmProgram(parsed: Parsed<AsmProgram>, options: RunOptions): Trace<AsmState, AsmStep> { + if (!parsed.program || !parsed.ok) throw new Error('The listing has a line the assembler refused; fix it before running.'); + const setup = parseSetup(options.setup || ''); + const machine = machineName(parseMachine(options.machine)); + // the hardware model reads the machine from the trace's spec, so the board's views and the counters agree on it + const spec = { program: parsed.program, registers: setup.registers, memory: setup.memory, flags: setup.flags, stdin: setup.stdin, maxSteps: options.maxSteps ?? 20000, machine: machineSpec(machine) }; + const tr: any = runAsm(spec); + const hits = new Map<number, number>(); + for (const instr of parsed.program.instrs) { const n = tr.hits[instr.index] || 0; if (n) hits.set(instr.lineNo + 1, n); } + const err = tr.stats.error || tr.stats.fault; + const output: string[] = []; + if (tr.stats.output) output.push(tr.stats.output); + output.push(`rax = ${tr.stats.result} (0x${BigInt.asUintN(64, tr.stats.result).toString(16)})`); + if (tr.stats.exitCode != null) output.push(`exit code ${tr.stats.exitCode}`); + return { + lang: 'asm', + steps: tr.steps, + states: tr.states, + count: tr.count, + output, + error: err ? { kind: err.kind || 'fault', message: err.message || String(err), line: err.lineNo != null ? err.lineNo + 1 : null } : null, + truncated: !!tr.stats.truncated, + hits, + raw: { trace: tr, machine, hardware: () => hardwareFor(tr) }, + }; +} + +const MNEMONIC = /^(mov|movabs|movzx|movsx|movsxd|lea|add|sub|adc|sbb|inc|dec|imul|mul|div|idiv|cqo|cdq|cdqe|and|or|xor|shl|sal|shr|sar|rol|ror|cmp|test|bt|stc|clc|cmc|push|pop|leave|jmp|j[a-z]{1,3}|cmov[a-z]{1,3}|set[a-z]{1,3}|call|ret|syscall|neg|not|xchg|popcnt|tzcnt|lzcnt|bsf|bsr|nop|movdq[ua]|movd|movq|pcmpeqb|pmovmskb|pand|por|pxor|psubb|paddb)$/i; +const REG = /^(r(ax|bx|cx|dx|si|di|bp|sp|8|9|1[0-5])|e(ax|bx|cx|dx|si|di|bp|sp)|r(8|9|1[0-5])[dwb]|[abcd][xlh]|[sd]il?|[bs]pl?|xmm\d{1,2}|rip)$/i; + +export function tokenizeAsm(line: string): Array<{ text: string; cls: string }> { + const out: Array<{ text: string; cls: string }> = []; + const re = /([;#].*$|\/\/.*$)|('.'|"(?:[^"\\]|\\.)*")|(\b0x[0-9a-f]+\b|\b\d+\b)|(\.?[A-Za-z_$][\w$.]*:)|(\.?[A-Za-z_$][\w$.]*)|(\s+)|(.)/g; + let m: RegExpExecArray | null; + let first = true; + while ((m = re.exec(line))) { + const [text, c, s, n, lbl, w] = m; + let cls = ''; + if (c) cls = 'tok-comment'; + else if (s) cls = 'tok-string'; + else if (n) cls = 'tok-number'; + else if (lbl) cls = 'tok-label'; + else if (w) { + if (first && MNEMONIC.test(w)) cls = 'tok-keyword'; + else if (REG.test(w)) cls = 'tok-builtin'; + else if (/^(byte|word|dword|qword|xmmword|ptr)$/i.test(w)) cls = 'tok-op'; + else cls = 'tok-name'; + first = false; + } + out.push({ text, cls }); + } + return out; +} + +export const asmService: LanguageService<AsmProgram, AsmState, AsmStep> = { + id: 'asm', + name: 'x86-64', + parse: parseAsm, + run: runAsmProgram, + tokenize: tokenizeAsm, +}; diff --git a/src/services/compile-backend.ts b/src/services/compile-backend.ts new file mode 100644 index 0000000..d2c538d --- /dev/null +++ b/src/services/compile-backend.ts @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Where Compile & run is sent. Every compile goes through one CompileBackend, so the app can move + * off godbolt.org to a service of its own (a self-hosted Compiler Explorer, or the checker service + * the proof stages plan, which will answer Mojo, Dafny and Lean) without touching the session or + * the views. The default is godbolt.org with the pinned compiler (src/mojo/generated/pin.ts). + * + * A backend answers exactly what godbolt answers: the same request body (mojoRequestBody, which + * tests/mojo.test.mjs holds to what the corpus pipeline sends) and the same answer shape + * (normalizeAnswer), so a stored answer and a live one mean the same thing. Rule 3 in AGENTS.md + * still holds whichever backend answers: the app never says a program compiles without one. + */ +import { compileMojo, GODBOLT_TARGET, type CompileOptions, type CompileOutcome, type CompileTarget } from './godbolt.js'; + +export interface CompileBackend { + /** A short id, such as 'godbolt'. */ + id: string; + /** The host a person sees in messages and tooltips, such as 'godbolt.org'. */ + host: string; + /** The compiler id the service runs, such as 'mojo_1_2_0'. */ + compiler: string; + compile(source: string, opts?: CompileOptions): Promise<CompileOutcome>; +} + +/** A backend that speaks Compiler Explorer's API at `target`. */ +export function explorerBackend(id: string, target: CompileTarget): CompileBackend { + return { id, host: target.host, compiler: target.compiler, compile: (source, opts = {}) => compileMojo(source, opts, target) }; +} + +export const GODBOLT: CompileBackend = explorerBackend('godbolt', GODBOLT_TARGET); + +let current: CompileBackend = GODBOLT; + +/** The backend Compile & run uses now. */ +export function compileBackend(): CompileBackend { return current; } + +/** Send compiles somewhere else from now on (a self-hosted service, or a fake in a test). Returns the previous one. */ +export function setCompileBackend(b: CompileBackend): CompileBackend { const prev = current; current = b; return prev; } diff --git a/src/services/godbolt.ts b/src/services/godbolt.ts new file mode 100644 index 0000000..3512701 --- /dev/null +++ b/src/services/godbolt.ts @@ -0,0 +1,283 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Compiler Explorer (godbolt.org): the real Mojo compiler behind the Compile + * button. Programming+ ships no compiler and never says a program compiles + * without asking one. + * + * One request compiles, returns the listing, and runs the program (what the + * corpus calls mode "both"; probes/godbolt-asm-and-run confirmed one request + * returns both). The body is the one scripts/mojo/mojocorpus/godbolt.py + * sends, and tests/mojo.test.mjs compares the two, so the button and the + * corpus can never ask different questions. + * + * What the probes taught us about Mojo on godbolt, which shapes this file (first asked of 1.1.0, and asked again at every pin): + * - godbolt runs `mojo build --emit=asm -o /app/output.s <args> /app/example.mojo` + * and passes no Intel-syntax flag, so the listing is AT&T (see src/mojo/att.ts); + * - without --target-cpu, `mojo build` targets the machine it runs on, and godbolt's + * machines differ (the same request answered with AVX2 one hour and AVX-512 the + * next), so every request carries the pin's build arguments (mojoBuildArgs below, + * probes/godbolt-target-cpu); + * - errors arrive as `<source>:LINE:COL: error: ...`, then the source line, a + * caret line, and sometimes a fix-it line (the replacement or insertion); + * - the user's own functions are labeled "example::name(args)", and only an + * unoptimized build (-O0) keeps them out of line; + * - since 1.2.0 a plain build's instructions carry source lines too (probes/godbolt-opt-O0); + * - with --debug-level=line-tables (probes/godbolt-line-tables*), instructions + * carry source lines, at no cost in compile time; but instructions inlined + * from the standard library carry lines of the library's own files (print's + * default arguments point at line 218 of io.mojo), so a line is the + * program's only when godbolt says it comes from the program's file. + */ +import type { Diagnostic, QuickFix } from '../common/types.js'; +import { GODBOLT_BASE, GODBOLT_COMPILER, MOJO_BUILD_ARGS, MOJO_VERSION } from '../mojo/generated/pin.js'; + +/** + * The arguments a program is built with, everywhere Programming+ builds one or asks godbolt to: the pin's + * (library/mojo/pin.json `build.args`, which say why each is there), then the program's own (`# args:`). An option + * the program names replaces the pin's option of that name, value and all: Mojo refuses a second --target-cpu, and + * a probe about --fp-mode states its own. `x86_64: false` is for a build on a machine that is not x86-64 (an Apple + * silicon Mac): the pin's --target-cpu, an x86-64 level, is left out there, because Mojo refuses an x86-64 CPU for + * any other target. Godbolt's machines are x86-64. scripts/mojo/mojocorpus/common.py (`build_args`) does the same, + * and tests/build-args.test.mjs holds the two to each other and every builder to them. + */ +export function mojoBuildArgs(own = '', { base = MOJO_BUILD_ARGS, x86_64 = true }: { base?: string; x86_64?: boolean } = {}): string { + const mine = own.split(/\s+/).filter(Boolean); + const named = new Set(mine.filter((t) => t.startsWith('-')).map((t) => t.split('=')[0])); + if (!x86_64) named.add('--target-cpu'); + const words = base.split(/\s+/).filter(Boolean); + const kept: string[] = []; + for (let i = 0; i < words.length;) { + const takesValue = !words[i].includes('=') && i + 1 < words.length && !words[i + 1].startsWith('-'); + const group = words.slice(i, i + (takesValue ? 2 : 1)); + if (!named.has(group[0].split('=')[0])) kept.push(...group); + i += group.length; + } + return [...kept, ...mine].join(' '); +} + +/** How the program is built: as written (every function kept, -O0) or as `mojo build` does by default. */ +export type BuildLevel = 'O0' | 'default'; +/** What each level adds to the pin's arguments. Line tables on both: they map instructions to lines and cost nothing in compile time (probes/godbolt-line-tables*). */ +export const BUILD_LEVELS: Record<BuildLevel, string> = { O0: '-O0 --debug-level=line-tables', default: '--debug-level=line-tables' }; +/** Every argument the Compile button sends at each level: the pin's, then the level's (probes/godbolt-compile-button asks exactly this). */ +export const BUILD_ARGS: Record<BuildLevel, string> = { O0: mojoBuildArgs(BUILD_LEVELS.O0), default: mojoBuildArgs(BUILD_LEVELS.default) }; + +const ASM_FILTERS = { binary: false, execute: false, intel: true, demangle: true, labels: true, libraryCode: false, directives: true, commentOnly: true, trim: true }; + +/** The request body, field for field what the Python harness sends in mode "both". */ +export function mojoRequestBody(source: string, stdin = '', args = ''): Record<string, unknown> { + return { + source, + options: { + userArguments: args, + executeParameters: { args: [], stdin }, + compilerOptions: { executorRequest: false, skipAsm: false }, + filters: { ...ASM_FILTERS, execute: true }, + tools: [], + libraries: [], + }, + lang: 'mojo', + allowStoreCodeDebug: false, + }; +} + +export function mojoRequestUrl(base = GODBOLT_BASE, compiler = GODBOLT_COMPILER): string { + return `${base.replace(/\/+$/, '')}/api/compiler/${encodeURIComponent(compiler)}/compile`; +} + +/** + * One listing line. `line` is a line of the program, when the instruction came + * from it; `from` says where an instruction inlined from elsewhere (the standard + * library) came from, so it is never mistaken for a line of the program. + */ +export interface AsmLine { text: string; line: number | null; from?: { file: string; line: number } } + +/** + * Which file a source line belongs to. Accepts each way godbolt can mark the + * program's own file: `mainsource: true`, no file named, or the input file's + * name (`inputFilename`, example.mojo). + */ +export function sourceOf(src: any, inputFile = 'example.mojo'): Pick<AsmLine, 'line' | 'from'> { + if (!src || typeof src.line !== 'number') return { line: null }; + const file: string | null = src.file ? String(src.file) : null; + const base = file ? file.split('/').pop()! : null; + const main = src.mainsource === true || (src.mainsource !== false && (!file || base === inputFile)); + return main ? { line: src.line } : { line: null, from: { file: base ?? '?', line: src.line } }; +} + +/** One answer, in the shape the Python harness normalizes to. */ +export interface Answer { + compiled: boolean; + buildStderr: string; + executed: boolean; + exitCode: number | null; + stdout: string; + stderr: string; + timedOut: boolean; + asm: AsmLine[]; +} + +const joinLines = (xs: unknown): string => (Array.isArray(xs) ? xs.map((l: any) => (typeof l === 'string' ? l : l?.text ?? '')).join('\n') : ''); + +export function normalizeAnswer(data: any): Answer { + const exe = data?.execResult ?? {}; + return { + compiled: data?.code === 0, + buildStderr: joinLines(data?.stderr), + executed: !!exe.didExecute, + exitCode: typeof exe.code === 'number' ? exe.code : null, + stdout: joinLines(exe.stdout), + stderr: joinLines(exe.stderr), + timedOut: !!exe.timedOut, + asm: Array.isArray(data?.asm) ? data.asm.filter((a: any) => a && typeof a === 'object').map((a: any) => ({ text: String(a.text ?? ''), ...sourceOf(a.source, data?.inputFilename || 'example.mojo') })) : [], + }; +} + +/* --------------------------------------------------------------------------- + * Compiler diagnostics + * ------------------------------------------------------------------------- */ + +const HEAD = /^<source>:(\d+):(\d+): (error|warning|note|remark): (.*)$/; +const ORIGIN = `the Mojo ${MOJO_VERSION} compiler`; + +interface RawDiag { severity: string; line: number; col: number; message: string; excerpt: string; caret: string; fixit: string | null } + +function readRaw(stderr: string): RawDiag[] { + const lines = stderr.split('\n'); + const out: RawDiag[] = []; + for (let i = 0; i < lines.length; i++) { + const m = HEAD.exec(lines[i]); + if (!m) continue; + const d: RawDiag = { severity: m[3], line: Number(m[1]), col: Number(m[2]), message: m[4], excerpt: '', caret: '', fixit: null }; + const body: string[] = []; + for (let j = i + 1; j < lines.length && !HEAD.test(lines[j]) && !/^\S.*: (error|warning): /.test(lines[j]); j++) body.push(lines[j]); + d.excerpt = body[0] ?? ''; + d.caret = body[1] ?? ''; + if (body[2] !== undefined && body[2].trim() && /^[ ~^]*$/.test(d.caret)) d.fixit = body[2]; + out.push(d); + } + return out; +} + +/** The fix a caret line and a fix-it line describe: a replacement under a range, else an insertion. */ +function fixOf(d: RawDiag): QuickFix | null { + if (d.fixit == null) return null; + const start = d.caret.search(/[\^~]/); + if (start < 0) return null; + const end = d.caret.replace(/\s+$/, '').length; // one past the last ^ or ~ + const at = d.fixit.search(/\S/); + const text = d.fixit.trim(); + if (at === start && end - start > 1) return { label: `Replace with ${text}`, line: d.line, col: start + 1, endCol: end + 1, text }; + return { label: `Insert ${text}`, line: d.line, col: at + 1, endCol: at + 1, text }; +} + +/** + * The compiler's messages as diagnostics on the Mojo pane. Notes stay + * notes; notes that carry a fix-it become the choices of a question on the + * error they follow ("consider transferring the value with '^'" or "you can + * copy it explicitly with '.copy()'"); an error's own fix-it is its fix. + */ +export function compilerDiagnostics(stderr: string): Diagnostic[] { + const out: Diagnostic[] = []; + let parent: Diagnostic | null = null; + for (const d of readRaw(stderr)) { + const sev = d.severity === 'remark' ? 'note' : d.severity as 'error' | 'warning' | 'note'; + const diag: Diagnostic = { lang: 'mojo', severity: sev, id: 'compiler', line: d.line, col: d.col, message: d.message, origin: ORIGIN }; + const fix = fixOf(d); + if (sev === 'note' && parent) { + if (fix) { + parent.question ??= { text: 'The compiler suggests:', options: [] }; + parent.question.options.push({ ...fix, label: d.message.replace(/^consider /, '').replace(/^you can /, '') }); + } + out.push(diag); + continue; + } + if (fix) diag.fix = fix; + out.push(diag); + if (sev !== 'note') parent = diag; + } + return out; +} + +/* --------------------------------------------------------------------------- + * The listing, as functions + * ------------------------------------------------------------------------- */ + +export type FunctionKind = 'user' | 'entry' | 'library' | 'data'; +export interface ListingFunction { label: string; name: string; kind: FunctionKind; lines: AsmLine[] } + +const TYPE_NAMES: Record<string, string> = { int: 'Int', uint: 'UInt', bool: 'Bool', float64: 'Float64', float32: 'Float32', float16: 'Float16', int8: 'Int8', int16: 'Int16', int32: 'Int32', int64: 'Int64', uint8: 'UInt8', uint16: 'UInt16', uint32: 'UInt32', uint64: 'UInt64' }; + +/** `"example::add(::SIMD[DType.int, 1],::SIMD[DType.int, 1])"` reads as `add(Int, Int)`. */ +export function demangle(label: string): string { + let s = label.replace(/^"|"$/g, '').replace(/^example::/, ''); + s = s.replace(/::SIMD\[DType\.(\w+), 1\]/g, (_, t: string) => TYPE_NAMES[t] ?? `Scalar[${t}]`); + s = s.replace(/(^|[(,\s\[])::/g, '$1'); + return s.replace(/,(?=\S)/g, ', '); +} + +function kindOf(label: string): FunctionKind { + if (label.startsWith('"example::') || label.startsWith('example::')) return 'user'; + if (label === 'main') return 'entry'; + if (/^(static_string_\w+|\.L\.str[\w.]*)$/.test(label)) return 'data'; // a read-only string, not code + return 'library'; +} + +const LABEL = /^(?:"((?:[^"\\]|\\.)*)"|([A-Za-z_.$][\w.$@]*)):\s*$/; + +/** Split the listing at each function label; local labels (.LBB2_2) stay inside their function. */ +export function listingFunctions(asm: AsmLine[]): ListingFunction[] { + const out: ListingFunction[] = []; + let cur: ListingFunction | null = null; + for (const a of asm) { + const m = LABEL.exec(a.text.trim()); + if (m && !(m[2] ?? '').startsWith('.L')) { + const label = m[1] !== undefined ? `"${m[1]}"` : m[2]; + cur = { label, name: m[1] !== undefined ? demangle(label) : label, kind: kindOf(m[1] !== undefined ? m[1] : label), lines: [] }; + out.push(cur); + continue; + } + if (!cur) continue; + if (!a.text.trim() || /^\s*\.(type|size|section|globl|p2align|cfi_\w+|file|loc|text|data)\b/.test(a.text)) continue; + cur.lines.push(a); + } + return out.filter((f) => f.lines.length); +} + +/* --------------------------------------------------------------------------- + * Asking + * ------------------------------------------------------------------------- */ + +export interface CompileOutcome { answer: Answer | null; error: string | null; seconds: number } + +/** What a compile asks for, whichever service answers it. */ +export interface CompileOptions { stdin?: string; level?: BuildLevel; signal?: AbortSignal; timeoutMs?: number } +/** Where a compile is sent: a service that speaks Compiler Explorer's API, at a base URL. */ +export interface CompileTarget { base: string; compiler: string; host: string } +export const GODBOLT_TARGET: CompileTarget = { base: GODBOLT_BASE, compiler: GODBOLT_COMPILER, host: 'godbolt.org' }; + +/** Ask godbolt (or another Compiler Explorer). A network or HTTP failure is an outcome to show, not an exception. */ +export async function compileMojo(source: string, opts: CompileOptions = {}, target: CompileTarget = GODBOLT_TARGET): Promise<CompileOutcome> { + const t0 = Date.now(); + const ctl = new AbortController(); + const timer = setTimeout(() => ctl.abort(), opts.timeoutMs ?? 60000); + opts.signal?.addEventListener('abort', () => ctl.abort()); + const host = target.host; + try { + const res = await fetch(mojoRequestUrl(target.base, target.compiler), { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify(mojoRequestBody(source, opts.stdin ?? '', BUILD_ARGS[opts.level ?? 'O0'])), + signal: ctl.signal, + }); + if (res.status === 429) return { answer: null, error: `${host} is rate-limiting requests from here. Wait a minute and compile again.`, seconds: (Date.now() - t0) / 1000 }; + if (!res.ok) return { answer: null, error: `${host} answered ${res.status} ${res.statusText}.`, seconds: (Date.now() - t0) / 1000 }; + return { answer: normalizeAnswer(await res.json()), error: null, seconds: (Date.now() - t0) / 1000 }; + } catch (e: any) { + const msg = e?.name === 'AbortError' ? `${host} did not answer in time.` : `Could not reach ${host} (${e?.message || e}).`; + return { answer: null, error: msg, seconds: (Date.now() - t0) / 1000 }; + } finally { + clearTimeout(timer); + } +} diff --git a/src/services/index.ts b/src/services/index.ts new file mode 100644 index 0000000..ee143dd --- /dev/null +++ b/src/services/index.ts @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +import type { LangId, LanguageService } from '../common/types.js'; +import { lingoService } from './lingo.js'; +import { mathoService } from './matho.js'; +import { pseudoService } from './pseudo.js'; +import { specoService } from './speco.js'; +import { asmService } from './asm.js'; +import { mojoService } from './mojo.js'; + +export const SERVICES: Record<LangId, LanguageService<any, any, any>> = { + lingo: lingoService, + matho: mathoService, + pseudo: pseudoService, + speco: specoService, + mojo: mojoService, + asm: asmService, +}; + +export const LANG_ORDER: LangId[] = ['lingo', 'matho', 'pseudo', 'speco', 'mojo', 'asm']; diff --git a/src/services/lingo.ts b/src/services/lingo.ts new file mode 100644 index 0000000..5bf96b1 --- /dev/null +++ b/src/services/lingo.ts @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * lingo: the controlled-English service. Wraps `readSteps` from the course's + * english.js. Today the engine lowers English to pseudocode *text* and + * re-parses it; this adapter exposes that as `lower(..., 'pseudo')` with the + * engine's own line map, and runs English programs through the pseudocode + * service so the trace is the same shape. + */ +import { readSteps, SENTENCE_FORMS } from '../engines/pseudo/english.js'; +import type { Diagnostic, LanguageService, Lowered, Parsed, RunOptions, Trace } from '../common/types.js'; +import { toDiagnostic, studyOf, runPseudo, defaultCall as pseudoDefaultCall } from './pseudo.js'; + +export type Reading = any; + +export function readLingo(source: string): Parsed<Reading> { + const r = readSteps(source); + const diagnostics: Diagnostic[] = r.diags.map((d: any) => toDiagnostic(d, 'lingo')); + return { program: r, diagnostics, ok: !!r.ok }; +} + +export function lowerLingo(parsed: Parsed<Reading>, target: string): Lowered | null { + const r = parsed.program; + if (!r || !r.pseudo) return null; + const lowered: Lowered = { + target: 'pseudo', + source: r.pseudo, + // lineMap[i] describes pseudocode line i+1 and names the English line it came from + map: (r.lineMap || []).map((m: any, i: number) => ({ from: m?.line ?? 0, to: i + 1 })).filter((m: { from: number }) => m.from > 0), + notes: [], + }; + if (target === 'pseudo') return lowered; + return null; +} + +/** Run an English program: lower it, then run the pseudocode with the English line numbers mapped back. */ +export function runLingo(parsed: Parsed<Reading>, options: RunOptions): Trace { + const low = lowerLingo(parsed, 'pseudo'); + if (!low) throw new Error('The steps do not read yet; answer the questions in the problems panel first.'); + const st = studyOf(low.source); + if (!st.ok) throw new Error('The pseudocode these steps became has an error: ' + (st.diagnostics.find((d) => d.severity === 'error')?.message || '')); + // The call in the toolbar usually names the pseudocode pane's routine; aim it at the English routine instead. + let call = options.call ?? null; + const subs: Map<string, any> = st.program.program.subs; + const main = parsed.program.main; + const m = call ? /^\s*([A-Za-z_]\w*)\s*\((.*)\)\s*$/s.exec(call) : null; + if (m && main && !subs.has(m[1].toUpperCase())) { + const sub = subs.get(String(main.pseudo).toUpperCase()); + const argc = m[2].trim() ? m[2].split(/,(?![^\[\]{}()]*[\]})])/).length : 0; + call = sub && sub.params.length === argc ? `${sub.name}(${m[2]})` : null; + } + const tr = runPseudo(st, { ...options, call: call ?? pseudoDefaultCall(st.program) }); + const toEnglish = new Map(low.map.map((m) => [m.to, m.from])); + const hits = new Map<number, number>(); + for (const [line, n] of tr.hits) { const e = toEnglish.get(line); if (e) hits.set(e, (hits.get(e) || 0) + n); } + return { + ...tr, + lang: 'lingo', + hits, + steps: tr.steps.map((s: any) => ({ ...s, pseudoLine: s.line, line: toEnglish.get(s.line) ?? s.line })), + error: tr.error ? { ...tr.error, line: tr.error.line != null ? toEnglish.get(tr.error.line) ?? tr.error.line : null } : null, + }; +} + +export function defaultCallLingo(parsed: Parsed<Reading>): string | null { + const low = lowerLingo(parsed, 'pseudo'); + if (!low) return null; + const st = studyOf(low.source); + return st.ok ? pseudoDefaultCall(st.program) : null; +} + +const STEP_WORDS = /^(to|start|let|set|make|add|append|put|insert|subtract|increase|decrease|multiply|double|halve|count|remove|sort|swap|show|print|check|if|otherwise|for|repeat|while|until|stop|skip|the answer|example)\b/i; + +export function tokenizeLingo(line: string): Array<{ text: string; cls: string }> { + const out: Array<{ text: string; cls: string }> = []; + const re = /("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)?')|(\b\d+(?:\.\d+)?\b)|(\b[A-Za-z][A-Za-z0-9_]*\b)|(\s+)|(.)/g; + let m: RegExpExecArray | null; + let first = true; + while ((m = re.exec(line))) { + const [text, s, n, w] = m; + let cls = ''; + if (s) cls = 'tok-string'; + else if (n) cls = 'tok-number'; + else if (w) { + if (first && STEP_WORDS.test(line.trim())) cls = 'tok-keyword'; + else if (/^[A-Z]/.test(w) || w.length === 1) cls = 'tok-name'; + first = false; + } + out.push({ text, cls }); + } + return out; +} + +export const sentenceForms: Array<{ job: string; example: string }> = (SENTENCE_FORMS as any[]).map((f) => ({ job: f.job || f.label || '', example: f.example || f.text || String(f) })); + +export const lingoService: LanguageService<Reading> = { + id: 'lingo', + name: 'lingo', + parse: readLingo, + run: runLingo, + lower: lowerLingo, + defaultCall: defaultCallLingo, + tokenize: tokenizeLingo, +}; diff --git a/src/services/matho.ts b/src/services/matho.ts new file mode 100644 index 0000000..ec9f27c --- /dev/null +++ b/src/services/matho.ts @@ -0,0 +1,1241 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * matho: equations, written in LaTeX. + * + * A student pastes the formula from the paper or the textbook, written the + * way they would write it in a LaTeX document, and the pane becomes pseudo: + * one FUNCTION per equation, sums and products as accumulator loops, cases + * as IF/ELSE, Greek letters as names, subscripts as list positions. + * + * \bar{x} = \frac{1}{n} \sum_{i=0}^{n-1} x_i + * + * FUNCTION X_BAR(N : INTEGER, X : LIST OF FLOAT) RETURNS FLOAT + * DECLARE Total : FLOAT + * Total <- 0.0 + * FOR I <- 0 TO N + * Total <- Total + X[I] + * NEXT I + * RETURN 1 / N * Total + * ENDFUNCTION + * + * The reader knows a deliberate subset of mathematics: arithmetic, fractions, + * roots, powers, e^x, |x|, floors and ceilings, sums and products with + * bounds or over a set, binomial coefficients, factorials, the usual + * functions (sin, cos, tan, exp, ln, log), max and min, dot products and + * norms of vectors, piecewise definitions with \begin{cases}, and + * references to equations defined higher up. Anything else (integrals, + * limits, infinity, derivatives) is refused with a sentence that says what + * finite procedure to write instead, because a procedure is what a program + * is, and that is the lesson. + * + * The decisions a translation has to make (which letters are whole numbers, + * which are lists, where a 1-based sum meets a 0-based list) are reported + * as notes on the equation's line, so nothing is decided silently. + */ +import type { Diagnostic, LanguageService, Lowered, Parsed, RunOptions, Trace } from '../common/types.js'; +import { studyOf, runPseudo, defaultCall as pseudoDefaultCall } from './pseudo.js'; +import { reservedNames } from '../common/reference.js'; + +/* ------------------------------------------------------------------ */ +/* Tokens */ +/* ------------------------------------------------------------------ */ + +type Tok = { t: 'cmd' | 'num' | 'id' | 'sym' | 'text' | 'end'; v: string; pos: number }; + +const TEXT_CMDS = new Set(['text', 'mathrm', 'operatorname', 'textrm', 'mathit', 'mbox']); +const SKIP_CMDS = new Set([',', ';', ':', '!', ' ', 'quad', 'qquad', 'displaystyle', 'nonumber', 'left', 'right', 'big', 'Big', 'bigg', 'Bigg', 'limits', 'nolimits', 'cdots', 'dots', 'ldots']); + +function tokenize(src: string): { toks: Tok[]; error: string | null } { + const toks: Tok[] = []; + let i = 0; + let error: string | null = null; + const n = src.length; + while (i < n) { + const ch = src[i]; + if (ch === '%') { while (i < n && src[i] !== '\n') i++; continue; } + if (/\s/.test(ch)) { i++; continue; } + if (ch === '\\') { + const m = /^\\([A-Za-z]+|.)/.exec(src.slice(i)); + if (!m) { i++; continue; } + const name = m[1]; + i += m[0].length; + if (SKIP_CMDS.has(name)) continue; + if (TEXT_CMDS.has(name)) { + // \text{...}: the braces hold words, not maths + let j = i; while (j < n && /\s/.test(src[j])) j++; + if (src[j] === '{') { + let depth = 0; let k = j; + for (; k < n; k++) { if (src[k] === '{') depth++; else if (src[k] === '}') { depth--; if (depth === 0) break; } } + toks.push({ t: 'text', v: src.slice(j + 1, k).trim(), pos: i - m[0].length }); + i = k + 1; + continue; + } + } + if (name === 'label') { // \label{...} is for the document, not the maths + const close = src.indexOf('}', i); i = close < 0 ? n : close + 1; continue; + } + toks.push({ t: 'cmd', v: name, pos: i - m[0].length }); + continue; + } + if (/[0-9]/.test(ch) || (ch === '.' && /[0-9]/.test(src[i + 1] || ''))) { + const m = /^[0-9]*\.?[0-9]+(?:[eE][+-]?[0-9]+)?|^[0-9]+\./.exec(src.slice(i))!; + toks.push({ t: 'num', v: m[0], pos: i }); + i += m[0].length; + continue; + } + if (/[A-Za-z]/.test(ch)) { toks.push({ t: 'id', v: ch, pos: i }); i++; continue; } + if ('{}()[]^_+-*/=<>,|!&:;\'.'.includes(ch)) { toks.push({ t: 'sym', v: ch, pos: i }); i++; continue; } + if ('·×÷'.includes(ch)) { toks.push({ t: 'cmd', v: ch === '·' || ch === '×' ? 'cdot' : 'div', pos: i }); i++; continue; } + if ('≤≥≠'.includes(ch)) { toks.push({ t: 'cmd', v: ch === '≤' ? 'le' : ch === '≥' ? 'ge' : 'ne', pos: i }); i++; continue; } + if (ch === 'π') { toks.push({ t: 'cmd', v: 'pi', pos: i }); i++; continue; } + if (ch === '∑') { toks.push({ t: 'cmd', v: 'sum', pos: i }); i++; continue; } + if (ch === '∏') { toks.push({ t: 'cmd', v: 'prod', pos: i }); i++; continue; } + if (ch === '√') { toks.push({ t: 'cmd', v: 'sqrt', pos: i }); i++; continue; } + if (ch === '∈') { toks.push({ t: 'cmd', v: 'in', pos: i }); i++; continue; } + if (ch === '$') { i++; continue; } + if (!error) error = `The character ${JSON.stringify(ch)} is not part of the mathematics this reader knows.`; + i++; + } + toks.push({ t: 'end', v: '', pos: n }); + return { toks, error }; +} + +/* ------------------------------------------------------------------ */ +/* Syntax tree */ +/* ------------------------------------------------------------------ */ + +type Decor = 'bar' | 'hat' | 'vec' | 'tilde' | 'bold' | null; + +export type Expr = + | { k: 'num'; v: string } + | { k: 'var'; name: string; decor: Decor; sub: Expr | null; subText: string | null; pos: number } + | { k: 'bin'; op: '+' | '-' | '*' | '/' | 'mod'; l: Expr; r: Expr } + | { k: 'neg'; e: Expr } + | { k: 'pow'; b: Expr; e: Expr } + | { k: 'call'; fn: string; args: Expr[]; pos: number } + | { k: 'sqrt'; e: Expr; n: Expr | null } + | { k: 'abs'; e: Expr } + | { k: 'norm'; e: Expr } + | { k: 'floor'; e: Expr } + | { k: 'ceil'; e: Expr } + | { k: 'big'; op: 'sum' | 'prod'; idx: string; lo: Expr | null; hi: Expr | null; over: Expr | null; body: Expr; pos: number } + | { k: 'binom'; n: Expr; m: Expr } + | { k: 'fact'; e: Expr } + | { k: 'pi' } + | { k: 'cases'; arms: Array<{ value: Expr; cond: Cond | null }> } + | { k: 'dot'; l: Expr; r: Expr }; + +export type Cond = + | { k: 'cmp'; op: '<' | '<=' | '>' | '>=' | '==' | '!='; l: Expr; r: Expr } + | { k: 'and'; l: Cond; r: Cond } + | { k: 'or'; l: Cond; r: Cond } + | { k: 'not'; c: Cond }; + +export interface Equation { + /** 1-based line of the source where the equation starts. */ + line: number; + source: string; + lhs: Expr | null; + rhs: Expr; +} + +export interface MathoProgram { + equations: Equation[]; + pseudo: string; + /** pseudo line (1-based) → equation index */ + lineMap: number[]; + /** Name and parameters of every function the pseudo defines, in order. */ + defs: Array<{ name: string; params: Array<{ name: string; type: string }>; line: number }>; +} + +class ParseError extends Error { constructor(msg: string, public pos: number, public id = 'matho-syntax') { super(msg); } } + +/** + * How deep an equation may nest (brackets, groups, \frac, \sqrt, powers in braces, functions of functions, signs) + * and how many tokens one equation may have. The lowering, the emitter and the pseudocode they write walk the tree + * by calling themselves, so without these an equation of thousands of nested groups would end in the engine's own + * stack error. A tree can be no taller than its tokens. Past either: one matho-too-deep or matho-too-long error + * (tests/reader-limits.test.mjs, docs/limits.md). + */ +const MAX_NESTING = 200; +const MAX_TOKENS = 2000; +/** The longest pseudocode the equations may become: it is checked as pseudocode, whose checks grow faster than its length. */ +const MAX_PSEUDO_LINES = 2000; +/** The most equations one pane holds (each becomes a FUNCTION; their lowering looks across all of them). */ +const MAX_EQUATIONS = 500; + +const GREEK: Record<string, string> = { + alpha: 'Alpha', beta: 'Beta', gamma: 'Gamma', delta: 'Delta', epsilon: 'Epsilon', varepsilon: 'Epsilon', zeta: 'Zeta', eta: 'Eta', theta: 'Theta', vartheta: 'Theta', + iota: 'Iota', kappa: 'Kappa', lambda: 'Lambda', mu: 'Mu', nu: 'Nu', xi: 'Xi', rho: 'Rho', varrho: 'Rho', sigma: 'Sigma', tau: 'Tau', upsilon: 'Upsilon', phi: 'Phi', varphi: 'Phi', + chi: 'Chi', psi: 'Psi', omega: 'Omega', Gamma: 'GammaU', Delta: 'DeltaU', Theta: 'ThetaU', Lambda: 'LambdaU', Xi: 'XiU', Pi: 'PiU', Sigma: 'SigmaU', Phi: 'PhiU', Psi: 'PsiU', Omega: 'OmegaU', ell: 'Ell', hbar: 'HBar', +}; +const FUNCS: Record<string, string> = { sin: 'SIN', cos: 'COS', tan: 'TAN', exp: 'EXP', ln: 'LOG', log: 'LOG10', sqrt: 'SQRT', max: 'MAX', min: 'MIN', arctan: 'ATAN', atan: 'ATAN', abs: 'ABS' }; +const REFUSED: Record<string, string> = { + int: 'An integral has no finite procedure behind it. Write the sum it approximates instead: \\sum_{i=0}^{n-1} f(a + i \\, h) \\, h with a step h, which becomes a loop.', + iint: 'A double integral has no finite procedure behind it. Write it as nested sums over a grid of steps.', + oint: 'A contour integral has no finite procedure behind it. Write it as a sum over the points of the path.', + infty: 'A program cannot run to infinity. Give the sum or product an upper bound, such as n, and make n a parameter.', + lim: 'A limit is not a procedure. Pick a small step or a large n and write what the expression is there.', + partial: 'A derivative is not a procedure. Write the difference quotient it approximates: (f(x + h) - f(x)) / h for a step h.', + nabla: 'A gradient is not a procedure. Write one difference quotient per coordinate.', + prime: 'A derivative is not a procedure. Write the difference quotient it approximates.', + forall: 'A statement about all values is a specification, not a procedure. Put it in the speco pane as an ENSURE.', + exists: 'A statement about some value is a specification, not a procedure. Put it in the speco pane, or write the loop that searches for the value.', + cup: 'Set union is not a single step in pseudo. Write the loop that ADDs each member of one SET to the other.', + cap: 'Set intersection is not a single step in pseudo. Write the loop that keeps the members found in both.', + to: 'An arrow is not an operation. Write the value the expression becomes.', + rightarrow: 'An arrow is not an operation. Write the value the expression becomes.', + approx: 'Write = and the expression you actually want computed; a program cannot be approximately equal.', + propto: 'Proportionality leaves the constant out. Name the constant and write = with it in.', + sim: 'A distribution symbol is not a procedure. Write RANDN() or RAND(x) for the sample you want drawn.', +}; + +/* ------------------------------------------------------------------ */ +/* Parser */ +/* ------------------------------------------------------------------ */ + +class Parser { + toks: Tok[]; + i = 0; + /** While the left side is being read, `f(x)` defines f even if f is new. */ + private lhsMode = false; + /** How many \| ... \| norms are open, so that the closing \| is not read as a new factor. */ + private normOpen = 0; + /** Groups, brackets and signs open around the token being read. */ + private depth = 0; + /** Functions defined by earlier equations, by their LaTeX name key. */ + constructor(toks: Tok[], private knownFns: Set<string>) { this.toks = toks; } + + /** Read one more level of nesting; past MAX_NESTING the equation is not read. */ + private nested<T>(read: () => T): T { + if (this.depth >= MAX_NESTING) throw new ParseError(`This equation is nested more than ${MAX_NESTING} levels deep, which is more than this reader reads. Give the inner parts names of their own, as equations above this one.`, this.tok.pos, 'matho-too-deep'); + this.depth++; + try { return read(); } finally { this.depth--; } + } + + get tok(): Tok { return this.toks[this.i]; } + peek(k = 1): Tok { return this.toks[Math.min(this.i + k, this.toks.length - 1)]; } + next(): Tok { return this.toks[this.i++]; } + isSym(v: string): boolean { return this.tok.t === 'sym' && this.tok.v === v; } + isCmd(v: string): boolean { return this.tok.t === 'cmd' && this.tok.v === v; } + expectSym(v: string, what: string): void { + if (!this.isSym(v)) throw new ParseError(`Expected ${JSON.stringify(v)} ${what}, but found ${this.describe(this.tok)}.`, this.tok.pos); + this.next(); + } + describe(t: Tok): string { return t.t === 'end' ? 'the end of the equation' : t.t === 'cmd' ? `\\${t.v}` : t.t === 'text' ? `the text "${t.v}"` : JSON.stringify(t.v); } + + /** The whole equation: `lhs = rhs`, `rhs`, or cases. */ + equation(): { lhs: Expr | null; rhs: Expr } { + if (this.tok.t === 'end') throw new ParseError('The equation is empty.', 0); + this.lhsMode = true; + const first = this.expr(); + this.lhsMode = false; + if (this.isSym('=') || this.isCmd('equiv') || (this.isSym(':') && this.peek().t === 'sym' && this.peek().v === '=')) { + if (this.isSym(':')) this.next(); + this.next(); + const rhs = this.exprOrCases(); + if (this.isSym('=')) throw new ParseError('A chain of equalities (a = b = c) is a statement, not a definition. Keep the two sides you want computed, and put the rest in a comment.', this.tok.pos, 'matho-chain'); + this.endOrError(); + return { lhs: first, rhs }; + } + this.endOrError(); + return { lhs: null, rhs: first }; + } + + private endOrError(): void { + if (this.tok.t !== 'end') { + const t = this.tok; + if (t.t === 'sym' && (t.v === '<' || t.v === '>')) throw new ParseError('A comparison on its own is a condition, not a value. Use it inside \\begin{cases} ... \\end{cases}, or in the speco pane.', t.pos, 'matho-condition'); + throw new ParseError(`Did not expect ${this.describe(t)} here.`, t.pos); + } + } + + private exprOrCases(): Expr { + if (this.isCmd('begin')) return this.cases(); + return this.expr(); + } + + private cases(): Expr { + this.next(); // \begin + const env = this.braceText(); + if (env !== 'cases') throw new ParseError(`\\begin{${env}} is not something this reader knows; only \\begin{cases} is.`, this.tok.pos, 'matho-command'); + const arms: Array<{ value: Expr; cond: Cond | null }> = []; + for (; ;) { + const value = this.expr(); + let cond: Cond | null = null; + if (this.isSym('&')) { + this.next(); + cond = this.condition(); + } + arms.push({ value, cond }); + if (this.isCmd('\\')) { this.next(); if (this.isCmd('end')) break; continue; } + if (this.isCmd('end')) break; + throw new ParseError(`Expected \\\\ between the cases or \\end{cases}, but found ${this.describe(this.tok)}.`, this.tok.pos); + } + this.next(); // \end + const close = this.braceText(); + if (close !== 'cases') throw new ParseError(`\\begin{cases} was closed by \\end{${close}}.`, this.tok.pos); + if (!arms.some((a) => a.cond === null)) throw new ParseError('The cases need a last line with no condition (write "& \\text{otherwise}"), so that every input has a value.', this.tok.pos, 'matho-cases-otherwise'); + if (arms.findIndex((a) => a.cond === null) !== arms.length - 1) throw new ParseError('The case with no condition ("otherwise") has to be the last one.', this.tok.pos, 'matho-cases-otherwise'); + return { k: 'cases', arms }; + } + + private braceText(): string { + this.expectSym('{', 'after \\begin or \\end'); + let s = ''; + while (!this.isSym('}') && this.tok.t !== 'end') s += this.next().v; + this.expectSym('}', 'to close the environment name'); + return s; + } + + /** A condition in a cases arm: `\text{if } x < 0`, `x \ge 0 \text{ and } y > 0`, `0 \le x < 1`, `\text{otherwise}`. */ + private condition(): Cond | null { + if (this.tok.t === 'text') { + const w = this.tok.v.toLowerCase().replace(/[^a-z]/g, ''); + if (w === 'otherwise' || w === 'else') { this.next(); return null; } + if (w === 'if' || w === 'when' || w === 'for' || w === 'where') this.next(); + } + return this.orCond(); + } + private orCond(): Cond { + let l = this.andCond(); + while (this.isCmd('lor') || this.isCmd('vee') || (this.tok.t === 'text' && /^or$/i.test(this.tok.v.trim()))) { this.next(); l = { k: 'or', l, r: this.andCond() }; } + return l; + } + private andCond(): Cond { + let l = this.cmp(); + while (this.isCmd('land') || this.isCmd('wedge') || this.isSym(',') || (this.tok.t === 'text' && /^and$/i.test(this.tok.v.trim()))) { this.next(); l = { k: 'and', l, r: this.cmp() }; } + return l; + } + private cmpOp(): '<' | '<=' | '>' | '>=' | '==' | '!=' | null { + const t = this.tok; + if (t.t === 'sym') { + if (t.v === '<') { this.next(); if (this.isSym('=')) { this.next(); return '<='; } return '<'; } + if (t.v === '>') { this.next(); if (this.isSym('=')) { this.next(); return '>='; } return '>'; } + if (t.v === '=') { this.next(); if (this.isSym('=')) this.next(); return '=='; } + if (t.v === '!' && this.peek().t === 'sym' && this.peek().v === '=') { this.next(); this.next(); return '!='; } + } + if (t.t === 'cmd') { + const m: Record<string, any> = { le: '<=', leq: '<=', leqslant: '<=', ge: '>=', geq: '>=', geqslant: '>=', ne: '!=', neq: '!=', lt: '<', gt: '>' }; + if (m[t.v]) { this.next(); return m[t.v]; } + } + return null; + } + private cmp(): Cond { + const l = this.expr(); + const op = this.cmpOp(); + if (!op) throw new ParseError(`A condition needs a comparison (<, \\le, =, \\ne, ...) after ${this.describe(this.toks[this.i - 1])}.`, this.tok.pos, 'matho-condition'); + const r = this.expr(); + let c: Cond = { k: 'cmp', op, l, r }; + // a chain such as 0 \le x < 1 + const op2 = this.cmpOp(); + if (op2) { const r2 = this.expr(); c = { k: 'and', l: c, r: { k: 'cmp', op: op2, l: r, r: r2 } }; } + return c; + } + + /* ---- expressions ---- */ + + expr(): Expr { + let l = this.term(); + for (; ;) { + if (this.isSym('+')) { this.next(); l = { k: 'bin', op: '+', l, r: this.term() }; } + else if (this.isSym('-')) { this.next(); l = { k: 'bin', op: '-', l, r: this.term() }; } + else if (this.isCmd('pm') || this.isCmd('mp')) throw new ParseError('\\pm gives two values; a function returns one. Write the + version and the - version as two equations.', this.tok.pos, 'matho-pm'); + else break; + } + return l; + } + + private startsFactor(t: Tok): boolean { + if (t.t === 'num' || t.t === 'id' || t.t === 'text') return true; + if (t.t === 'sym') return t.v === '(' || t.v === '{' || t.v === '['; + if (t.t === 'cmd' && t.v === '|') return this.normOpen === 0; + if (t.t === 'cmd') return !['cdot', 'times', 'div', 'bmod', 'mod', 'le', 'leq', 'ge', 'geq', 'ne', 'neq', 'lt', 'gt', 'land', 'lor', 'end', '\\', 'equiv', 'in', 'pm', 'mp', 'right', 'rfloor', 'rceil', 'rvert', 'rVert', 'rangle'].includes(t.v); + return false; + } + + private term(): Expr { + let l = this.unary(); + for (; ;) { + if (this.isCmd('cdot') || this.isCmd('times') || this.isSym('*')) { + const dotted = this.isCmd('cdot'); + this.next(); + const r = this.unary(); + l = dotted && isVector(l) && isVector(r) ? { k: 'dot', l, r } : { k: 'bin', op: '*', l, r }; + } else if (this.isSym('/') || this.isCmd('div')) { this.next(); l = { k: 'bin', op: '/', l, r: this.unary() }; } + else if (this.isCmd('bmod') || this.isCmd('mod')) { this.next(); l = { k: 'bin', op: 'mod', l, r: this.unary() }; } + else if (this.startsFactor(this.tok) && !(this.tok.t === 'sym' && this.tok.v === '[' && false)) { + // juxtaposition is multiplication: 2 \pi r, m v^2, \frac{1}{n} \sum ... + l = { k: 'bin', op: '*', l, r: this.unary() }; + } else break; + } + return l; + } + + private unary(): Expr { + if (this.isSym('-')) { this.next(); return { k: 'neg', e: this.nested(() => this.unary()) }; } + if (this.isSym('+')) { this.next(); return this.nested(() => this.unary()); } + return this.postfix(); + } + + /** An atom with its subscripts, superscripts, and factorials. */ + private postfix(): Expr { + let e = this.atom(); + for (; ;) { + if (this.isSym('^')) { + this.next(); + const ex = this.scriptArg(); + if (e.k === 'var' && e.name === 'e' && !e.sub && !e.decor) e = { k: 'call', fn: 'EXP', args: [ex], pos: e.pos }; + else e = { k: 'pow', b: e, e: ex }; + } else if (this.isSym('_')) { + this.next(); + if (e.k !== 'var' || e.sub) throw new ParseError('A subscript belongs on a name: x_i, v_{\\max}, a_{i+1}.', this.tok.pos, 'matho-subscript'); + const save = this.i; + const argToks = this.scriptTokens(); + const text = argToks.every((t) => t.t === 'id' || t.t === 'text' || t.t === 'num') ? argToks.map((t) => t.v).join('') : null; + this.i = save; + e = { ...e, sub: this.scriptArg(), subText: text }; + } else if (this.isSym('!')) { this.next(); e = { k: 'fact', e }; } + else if (this.isSym('\'')) throw new ParseError(REFUSED.prime, this.tok.pos, 'matho-refused'); + else break; + } + return e; + } + + /** The tokens of the next `{...}` group or single token, without consuming them. */ + private scriptTokens(): Tok[] { + const out: Tok[] = []; + if (this.isSym('{')) { + let depth = 0; + for (let j = this.i; j < this.toks.length; j++) { + const t = this.toks[j]; + if (t.t === 'sym' && t.v === '{') { depth++; if (depth === 1) continue; } + if (t.t === 'sym' && t.v === '}') { depth--; if (depth === 0) break; } + out.push(t); + } + } else out.push(this.tok); + return out; + } + + private scriptArg(): Expr { + // a power's or a subscript's group nests like a bracket: x^{x^{x^{...}}} + if (this.isSym('{')) return this.nested(() => { this.next(); const e = this.expr(); this.expectSym('}', 'to close the group'); return e; }); + // a single token: x^2, x_i, e^x, \sum_i + const t = this.tok; + if (t.t === 'num' || t.t === 'id' || t.t === 'cmd' || t.t === 'text') return this.atom(); + throw new ParseError(`Expected a group in braces or a single symbol after ${this.describe(this.toks[this.i - 1])}.`, t.pos); + } + + /** Every way an expression holds another (a bracket, a group, a command's argument) passes through here. */ + private atom(): Expr { return this.nested(() => this.atomInner()); } + + private atomInner(): Expr { + const t = this.tok; + if (t.t === 'num') { this.next(); return { k: 'num', v: t.v }; } + if (t.t === 'id') { + this.next(); + // f(x) where f names an earlier equation: a call + if (this.isSym('(') && (this.knownFns.has(t.v) || this.lhsMode)) return this.callArgs(t.v, t.pos); + return { k: 'var', name: t.v, decor: null, sub: null, subText: null, pos: t.pos }; + } + if (t.t === 'text') { + this.next(); + const name = t.v.replace(/\s+/g, '_'); + if (this.isSym('(') && (this.knownFns.has(name) || this.lhsMode)) return this.callArgs(name, t.pos); + return { k: 'var', name, decor: null, sub: null, subText: null, pos: t.pos }; + } + if (t.t === 'sym') { + if (t.v === '(' || t.v === '[') { + this.next(); + const e = this.expr(); + this.expectSym(t.v === '(' ? ')' : ']', 'to close the bracket'); + return e; + } + if (t.v === '{') { this.next(); const e = this.expr(); this.expectSym('}', 'to close the group'); return e; } + if (t.v === '|') { this.next(); const e = this.expr(); this.expectSym('|', 'to close the absolute value'); return { k: 'abs', e }; } + throw new ParseError(`Did not expect ${this.describe(t)} here.`, t.pos); + } + if (t.t === 'cmd') return this.command(); + throw new ParseError('The equation ends too early.', t.pos); + } + + private callArgs(fn: string, pos: number): Expr { + this.expectSym('(', 'after the function name'); + const args: Expr[] = []; + if (!this.isSym(')')) { args.push(this.expr()); while (this.isSym(',')) { this.next(); args.push(this.expr()); } } + this.expectSym(')', 'to close the arguments'); + return { k: 'call', fn, args, pos }; + } + + private group(what: string): Expr { + this.expectSym('{', what); + const e = this.expr(); + this.expectSym('}', 'to close the group'); + return e; + } + + private command(): Expr { + const t = this.next(); + const c = t.v; + if (REFUSED[c]) throw new ParseError(REFUSED[c], t.pos, 'matho-refused'); + if (GREEK[c]) return { k: 'var', name: '\\' + c, decor: null, sub: null, subText: null, pos: t.pos }; + if (c === 'pi') return { k: 'pi' }; + if (c === 'frac' || c === 'dfrac' || c === 'tfrac') { const n = this.group('for the top of the fraction'); const d = this.group('for the bottom of the fraction'); return { k: 'bin', op: '/', l: n, r: d }; } + if (c === 'sqrt') { + let n: Expr | null = null; + if (this.isSym('[')) { this.next(); n = this.expr(); this.expectSym(']', 'to close the root index'); } + return { k: 'sqrt', e: this.group('for what the root is taken of'), n }; + } + if (c === 'binom' || c === 'dbinom' || c === 'tbinom') return { k: 'binom', n: this.group('for the top of the binomial'), m: this.group('for the bottom of the binomial') }; + if (c === 'bar' || c === 'overline' || c === 'hat' || c === 'vec' || c === 'tilde' || c === 'mathbf' || c === 'boldsymbol') { + const decor: Decor = c === 'overline' ? 'bar' : c === 'mathbf' || c === 'boldsymbol' ? 'bold' : (c as Decor); + const inner = this.group(`for what \\${c} applies to`); + if (inner.k !== 'var' || inner.sub) throw new ParseError(`\\${c} goes over a single name: \\${c}{x}.`, t.pos, 'matho-decor'); + return { ...inner, decor }; + } + if (c === 'sum' || c === 'prod') return this.bigOp(c, t.pos); + if (c === 'lfloor') { const e = this.expr(); if (!this.isCmd('rfloor')) throw new ParseError('\\lfloor needs a matching \\rfloor.', this.tok.pos); this.next(); return { k: 'floor', e }; } + if (c === 'lceil') { const e = this.expr(); if (!this.isCmd('rceil')) throw new ParseError('\\lceil needs a matching \\rceil.', this.tok.pos); this.next(); return { k: 'ceil', e }; } + if (c === 'lvert') { const e = this.expr(); if (!this.isCmd('rvert')) throw new ParseError('\\lvert needs a matching \\rvert.', this.tok.pos); this.next(); return { k: 'abs', e }; } + if (c === 'lVert' || c === '|') { + this.normOpen++; + const e = this.expr(); + this.normOpen--; + if (!(this.isCmd('rVert') || this.isCmd('|'))) throw new ParseError('\\lVert (or \\|) needs a matching \\rVert (or \\|).', this.tok.pos); + this.next(); + return { k: 'norm', e }; + } + if (c === 'langle') { const l = this.expr(); this.expectSym(',', 'between the two vectors of the inner product'); const r = this.expr(); if (!this.isCmd('rangle')) throw new ParseError('\\langle needs a matching \\rangle.', this.tok.pos); this.next(); return { k: 'dot', l, r }; } + if (c === 'log' && this.isSym('_')) { + this.next(); + const base = this.scriptArg(); + const arg = this.funcArg(); + if (base.k === 'num' && base.v === '10') return { k: 'call', fn: 'LOG10', args: [arg], pos: t.pos }; + if (base.k === 'num' && base.v === '2') return { k: 'call', fn: 'LOG2', args: [arg], pos: t.pos }; + if (base.k === 'var' && base.name === 'e' && !base.sub) return { k: 'call', fn: 'LOG', args: [arg], pos: t.pos }; + return { k: 'bin', op: '/', l: { k: 'call', fn: 'LOG', args: [arg], pos: t.pos }, r: { k: 'call', fn: 'LOG', args: [base], pos: t.pos } }; + } + if (FUNCS[c]) { + if (this.isSym('^')) { + // \sin^2 x + this.next(); + const p = this.scriptArg(); + const arg = this.funcArg(); + return { k: 'pow', b: { k: 'call', fn: FUNCS[c], args: [arg], pos: t.pos }, e: p }; + } + if (c === 'max' || c === 'min') { + this.expectSym('(', `after \\${c}`); + const args = [this.expr()]; + while (this.isSym(',')) { this.next(); args.push(this.expr()); } + this.expectSym(')', `to close \\${c}`); + return { k: 'call', fn: FUNCS[c], args, pos: t.pos }; + } + return { k: 'call', fn: FUNCS[c], args: [this.funcArg()], pos: t.pos }; + } + if (c === 'begin') throw new ParseError('\\begin{cases} can only be the whole right-hand side of an equation.', t.pos, 'matho-cases-place'); + if (c === 'mathbb' || c === 'mathcal') { const inner = this.group(`for \\${c}`); return inner; } + throw new ParseError(`\\${c} is not a command this reader knows. It knows arithmetic, \\frac, \\sqrt, powers, e^x, |x|, \\lfloor \\rfloor, \\sum, \\prod, \\binom, n!, \\sin \\cos \\tan \\exp \\ln \\log, \\max \\min, \\vec and \\|v\\|, \\begin{cases}, and Greek letters.`, t.pos, 'matho-command'); + } + + /** What a function such as \sin applies to: a bracketed argument, or the next power. */ + private funcArg(): Expr { + if (this.isSym('(')) { this.next(); const e = this.expr(); this.expectSym(')', 'to close the argument'); return e; } + if (this.isSym('{')) return this.group(''); + return this.postfix(); + } + + private bigOp(op: 'sum' | 'prod', pos: number): Expr { + let idx: string | null = null; + let lo: Expr | null = null; + let hi: Expr | null = null; + let over: Expr | null = null; + for (let k = 0; k < 2; k++) { + if (this.isSym('_')) { + this.next(); + const braced = this.isSym('{'); + if (braced) this.next(); + const v = this.tok; + if (v.t !== 'id' && !(v.t === 'cmd' && GREEK[v.v])) throw new ParseError(`\\${op} needs an index letter under it: \\${op}_{i=0}^{n-1}.`, v.pos, 'matho-sum-index'); + this.next(); + idx = v.t === 'id' ? v.v : '\\' + v.v; + if (this.isSym('=')) { this.next(); lo = this.expr(); } + else if (this.isCmd('in')) { this.next(); over = this.expr(); } + if (braced) this.expectSym('}', 'to close the index'); + } else if (this.isSym('^')) { + this.next(); + hi = this.scriptArg(); + } + } + if (!idx) throw new ParseError(`\\${op} needs an index letter under it: \\${op}_{i=0}^{n-1}.`, pos, 'matho-sum-index'); + if (hi && !lo && !over) throw new ParseError(`\\${op}_{${idx}} has an upper bound but no lower one: write \\${op}_{${idx}=0}^{...}.`, pos, 'matho-sum-bounds'); + const body = this.term(); + return { k: 'big', op, idx, lo, hi, over, body, pos }; + } +} + +function isVector(e: Expr): boolean { return e.k === 'var' && (e.decor === 'vec' || e.decor === 'bold') && !e.sub; } + +/* ------------------------------------------------------------------ */ +/* Lowering */ +/* ------------------------------------------------------------------ */ + +type PType = 'INTEGER' | 'FLOAT' | 'LIST OF FLOAT' | 'LIST OF LIST OF FLOAT'; + +interface Param { key: string; name: string; type: PType; roles: Set<'index' | 'bound' | 'list' | 'matrix' | 'vector' | 'plain'> } + +interface Def { key: string; fnName: string; params: Param[]; } + +const RESERVED = new Set(reservedNames().map((r) => r.term.toUpperCase())); + +/** A variable's key: its base name plus decoration plus a non-index subscript. */ +function varKey(v: Extract<Expr, { k: 'var' }>, indexSub: boolean): string { + return `${v.decor ?? ''}:${v.name}:${indexSub ? '' : (v.subText ?? '')}`; +} + +function baseWord(name: string): string { + if (name.startsWith('\\')) return GREEK[name.slice(1)] ?? name.slice(1); + if (name.length === 1) return name.toUpperCase(); + return name.split(/[_\s]+/).map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join(''); +} + +/** `x` → X, `\sigma` → Sigma, `\bar{x}` → XBar, `v_{max}` → VMax, `x_0` → X0. */ +function pseudoName(v: Extract<Expr, { k: 'var' }>, indexSub: boolean): string { + let s = baseWord(v.name); + if (v.decor === 'bar') s += 'Bar'; + else if (v.decor === 'hat') s += 'Hat'; + else if (v.decor === 'tilde') s += 'Tilde'; + if (!indexSub && v.subText) { + const sub = v.subText.replace(/^\\/, ''); + s += GREEK[sub] ?? (sub.charAt(0).toUpperCase() + sub.slice(1)); + } + if (!/^[A-Za-z_]\w*$/.test(s)) s = s.replace(/\W/g, '_'); + if (RESERVED.has(s.toUpperCase())) s += 'Val'; + return s; +} + +/** The function name an equation's left side gives: `f` → F, `\bar{x}` → X_BAR, `E_k` → E_K, `\sigma^2` → SIGMA_SQUARED. */ +function functionName(lhs: Expr): string { + let squared = false; + let e = lhs; + if (e.k === 'pow' && e.e.k === 'num' && e.e.v === '2') { squared = true; e = e.b; } + if (e.k === 'fact' && e.e.k === 'var') return 'FACTORIAL'; + if (e.k === 'call' && Object.values(FUNCS).includes(e.fn)) return [e.fn, ...e.args.map((a) => (a.k === 'var' ? baseWord(a.name).toUpperCase() : 'OF'))].join('_'); + if (e.k === 'call') e = { k: 'var', name: e.fn, decor: null, sub: null, subText: null, pos: e.pos }; + if (e.k !== 'var') return 'VALUE'; + const parts = [baseWord(e.name).toUpperCase()]; + if (e.decor === 'bar') parts.push('BAR'); else if (e.decor === 'hat') parts.push('HAT'); else if (e.decor === 'tilde') parts.push('TILDE'); + if (e.subText) parts.push((GREEK[e.subText.replace(/^\\/, '')] ?? e.subText).toUpperCase()); + if (squared) parts.push('SQUARED'); + let s = parts.join('_').replace(/\W/g, '_'); + if (RESERVED.has(s)) s += '_FN'; + return s; +} + +class Lowerer { + diagnostics: Diagnostic[] = []; + defs = new Map<string, Def>(); + lines: string[] = []; + lineMap: number[] = []; + outDefs: MathoProgram['defs'] = []; + + private note(line: number, id: string, message: string): void { this.diagnostics.push({ lang: 'matho', severity: 'note', id, line, message }); } + private warn(line: number, id: string, message: string): void { this.diagnostics.push({ lang: 'matho', severity: 'warning', id, line, message }); } + + lower(eqs: Equation[]): void { + eqs.forEach((eq, k) => { + const start = this.lines.length; + this.one(eq, k); + for (let i = start; i < this.lines.length; i++) this.lineMap[i] = k; + }); + } + + private one(eq: Equation, k: number): void { + const { lhs, rhs } = eq; + // --- the function's own name and explicit arguments + let fnName = lhs ? functionName(lhs) : (k === 0 ? 'VALUE' : `VALUE_${k + 1}`); + const explicit: Array<Extract<Expr, { k: 'var' }>> = []; + let lhsIndex: Extract<Expr, { k: 'var' }> | null = null; + const lhsIsBuiltin = lhs?.k === 'call' && Object.values(FUNCS).includes(lhs.fn); + if (lhs?.k === 'call' && !lhsIsBuiltin) for (const a of lhs.args) { if (a.k !== 'var' || a.sub) { this.warn(eq.line, 'matho-lhs-args', `The arguments on the left side should be plain names, as in f(x, y); ${describeExpr(a)} was ignored.`); continue; } explicit.push(a); } + if (lhs?.k === 'var' && lhs.sub && lhs.sub.k === 'var' && mentionsSub(rhs, lhs.sub.name)) { lhsIndex = lhs.sub; fnName = functionName({ ...lhs, sub: null, subText: null }); } + if (lhs && lhs.k !== 'var' && lhs.k !== 'call' && !(lhs.k === 'pow' && lhs.b.k === 'var') && !(lhs.k === 'fact' && lhs.e.k === 'var')) this.warn(eq.line, 'matho-lhs', `The left side ${describeExpr(lhs)} is an expression, not a name, so the function is called ${fnName}.`); + if (this.defs.has(fnName)) { let n = 2; while (this.defs.has(`${fnName}_${n}`)) n++; fnName = `${fnName}_${n}`; } + + // --- parameters: explicit first, then the free names of the right side + const params: Param[] = []; + const byKey = new Map<string, Param>(); + const bound = new Set<string>(); // index names bound by sums in scope while walking + const indexNames = new Set<string>(); // every index letter in this equation (sums + lhs index) + collectIndices(rhs, indexNames); + if (lhsIndex) indexNames.add(lhsIndex.name); + const add = (v: Extract<Expr, { k: 'var' }>, role: 'index' | 'bound' | 'plain'): Param => { + const indexSub = !!v.sub && isIndexExpr(v.sub, indexNames); + const key = varKey(v, indexSub); + let p = byKey.get(key); + if (!p) { + p = { key, name: pseudoName(v, indexSub), type: 'FLOAT', roles: new Set() }; + let n = 2; const base = p.name; while ([...byKey.values()].some((q) => q.name === p!.name) || p.name.toUpperCase() === fnName) p.name = `${base}${n++}`; + if (p.name !== base) this.note(eq.line, 'matho-case', `${describeExpr(v)} and ${p.name.toUpperCase() === fnName || base.toUpperCase() === fnName ? `the function ${fnName}` : base} are different symbols in mathematics but would be the same name in pseudo, so ${describeExpr(v)} became ${p.name}.`); + byKey.set(key, p); params.push(p); + } + p.roles.add(role); + if (indexSub && v.sub) p.roles.add(subDepth(v.sub) === 2 ? 'matrix' : 'list'); + if (isVector(v)) p.roles.add('vector'); + return p; + }; + if (lhsIndex) { const p = add(lhsIndex, 'index'); p.type = 'INTEGER'; } + for (const a of explicit) add(a, 'plain'); + const walk = (e: Expr, inBound: boolean, inIndex: boolean): void => { + switch (e.k) { + case 'var': + if (bound.has(e.name) && !e.sub && !e.decor) return; + if (e.name === 'e' && !e.sub && !e.decor) { /* Euler's e on its own is a name; a note is given at emit time */ } + if (this.defs.has(functionName(e)) && !e.sub) { // a reference to an earlier equation: its parameters join ours + const d = this.defs.get(functionName(e))!; + for (const q of d.params) { const p = byKey.get(q.key) ?? (byKey.set(q.key, { ...q, roles: new Set(q.roles) }).get(q.key)!); if (!params.includes(p)) params.push(p); } + return; + } + add(e, inBound ? 'bound' : inIndex ? 'index' : 'plain'); + if (e.sub) walk(e.sub, false, true); + return; + case 'num': case 'pi': return; + case 'bin': if (e.op === 'mod') { walk(e.l, true, false); walk(e.r, true, false); return; } walk(e.l, inBound, inIndex); walk(e.r, inBound, inIndex); return; + case 'dot': walk(e.l, false, false); walk(e.r, false, false); return; + case 'fact': walk(e.e, true, false); return; + case 'neg': case 'abs': case 'norm': case 'floor': case 'ceil': walk(e.e, inBound, inIndex); return; + case 'sqrt': walk(e.e, false, false); if (e.n) walk(e.n, true, false); return; + case 'pow': walk(e.b, inBound, inIndex); walk(e.e, inBound, inIndex); return; + case 'call': + if (this.defs.has(functionName({ k: 'var', name: e.fn, decor: null, sub: null, subText: null, pos: e.pos }))) { + const d = this.defs.get(functionName({ k: 'var', name: e.fn, decor: null, sub: null, subText: null, pos: e.pos }))!; + // explicit arguments cover the first parameters; the rest join ours + for (const q of d.params.slice(e.args.length)) { const p = byKey.get(q.key) ?? (byKey.set(q.key, { ...q, roles: new Set(q.roles) }).get(q.key)!); if (!params.includes(p)) params.push(p); } + } + for (const a of e.args) walk(a, false, false); + return; + case 'binom': walk(e.n, true, false); walk(e.m, true, false); return; + case 'big': + if (e.lo) walk(e.lo, true, false); + if (e.hi) walk(e.hi, true, false); + if (e.over) { if (e.over.k === 'var' && !e.over.sub) add(e.over, 'plain').roles.add('vector'); else walk(e.over, false, false); } + { const was = bound.has(e.idx); bound.add(e.idx); walk(e.body, false, false); if (!was) bound.delete(e.idx); } + return; + case 'cases': + for (const a of e.arms) { walk(a.value, false, false); if (a.cond) walkCond(a.cond); } + return; + } + }; + const walkCond = (c: Cond): void => { if (c.k === 'cmp') { walk(c.l, false, false); walk(c.r, false, false); } else if (c.k === 'not') walkCond(c.c); else { walkCond(c.l); walkCond(c.r); } }; + walk(rhs, false, false); + + // --- types from roles + for (const p of params) { + if (p.roles.has('matrix')) p.type = 'LIST OF LIST OF FLOAT'; + else if (p.roles.has('list') || p.roles.has('vector')) p.type = 'LIST OF FLOAT'; + else if (p.roles.has('index') || p.roles.has('bound')) p.type = 'INTEGER'; + else if (p.type !== 'INTEGER') p.type = 'FLOAT'; + } + // the explicit arguments keep their order; anything else the right side needs is appended, with a note + const explicitKeys = new Set(explicit.map((a) => varKey(a, false))); + if (lhs?.k === 'call' && !lhsIsBuiltin) { + const extra = params.filter((p) => !explicitKeys.has(p.key) && !lhsIndex); + if (extra.length) this.note(eq.line, 'matho-extra-params', `${extra.map((p) => p.name).join(', ')} ${extra.length === 1 ? 'is' : 'are'} used on the right side but ${extra.length === 1 ? 'is not an argument' : 'are not arguments'} of ${describeExpr(lhs)}, so ${extra.length === 1 ? 'it became an extra parameter' : 'they became extra parameters'} of ${fnName}.`); + } + for (const p of params) { + if (p.type === 'LIST OF FLOAT' && p.roles.has('list')) this.note(eq.line, 'matho-list', `${p.name} is indexed, so it is a LIST OF FLOAT, and the subscript is a position in it.`); + if (p.type === 'LIST OF LIST OF FLOAT') this.note(eq.line, 'matho-matrix', `${p.name} has two indices, so it is a LIST OF LIST OF FLOAT: a list of rows.`); + if (p.type === 'INTEGER' && p.roles.has('bound') && !p.roles.has('index')) this.note(eq.line, 'matho-bound', `${p.name} counts something (it bounds a sum or product, or sits in a binomial coefficient), so it is an INTEGER.`); + } + + // --- emit + const ctx: EmitCtx = { params: byKey, indexNames, defs: this.defs, decls: [], used: new Set(params.map((p) => p.name)), shifts: new Map(), shifted: new Set(), notes: [], line: eq.line, bound: new Map(), loopDepth: 0, hoisted: new Map(), prelude: [] }; + if (lhsIndex) ctx.bound.set(lhsIndex.name, byKey.get(varKey(lhsIndex, false))!.name); + const body: string[] = []; + let ret: string[] = []; + if (rhs.k === 'cases') { + ret = this.emitCases(rhs, ctx, body, 1); + } else { + const expr = emitExpr(rhs, ctx, body, 1); + ret = [` RETURN ${expr}`]; + } + const head = `FUNCTION ${fnName}(${params.map((p) => `${p.name} : ${p.type}`).join(', ')}) RETURNS FLOAT`; + this.lines.push(`// ${eq.source.replace(/\s+/g, ' ').trim()}`); + this.lines.push(head); + for (const d of ctx.decls) this.lines.push(` DECLARE ${d.name} : ${d.type}`); + this.lines.push(...ctx.prelude, ...body, ...ret, 'ENDFUNCTION', ''); + for (const n of ctx.notes) this.note(eq.line, n.id, n.message); + this.defs.set(fnName, { key: fnName, fnName, params }); + this.outDefs.push({ name: fnName, params: params.map((p) => ({ name: p.name, type: p.type })), line: eq.line }); + } + + private emitCases(e: Extract<Expr, { k: 'cases' }>, ctx: EmitCtx, out: string[], indent: number): string[] { + const pad = ' '.repeat(indent); + const lines: string[] = []; + e.arms.forEach((arm, i) => { + const last = i === e.arms.length - 1; + if (!last) { + const cond = emitCond(arm.cond!, ctx, out, indent); + lines.push(`${pad}${i === 0 ? 'IF' : 'ELSEIF'} ${cond} THEN`); + } else lines.push(`${pad}ELSE`); + const inner: string[] = []; + const v = emitExpr(arm.value, ctx, inner, indent + 1); + lines.push(...inner, `${pad} RETURN ${v}`); + }); + lines.push(`${pad}ENDIF`); + return lines; + } +} + +interface EmitCtx { + params: Map<string, Param>; + indexNames: Set<string>; + defs: Map<string, Def>; + decls: Array<{ name: string; type: string }>; + used: Set<string>; + /** index letter → how many to subtract when it indexes a list (a sum from 1) */ + shifts: Map<string, number>; + /** index letters whose shift was actually used on a subscript */ + shifted: Set<string>; + notes: Array<{ id: string; message: string }>; + line: number; + /** index letter → its pseudo name, while in scope */ + bound: Map<string, string>; + /** how many sum/product loops the emitter is inside */ + loopDepth: number; + /** calls of earlier equations that do not change inside a loop, computed once before it: call text → local name */ + hoisted: Map<string, string>; + /** the lines that compute the hoisted values, placed before the body */ + prelude: string[]; +} + +function fresh(ctx: EmitCtx, base: string): string { + const taken = (x: string) => ctx.used.has(x) || RESERVED.has(x.toUpperCase()) || ctx.defs.has(x.toUpperCase()); + let name = base; + if (taken(name) && !taken(`${base}Val`)) name = `${base}Val`; + let n = 2; + while (taken(name)) name = `${base}${n++}`; + ctx.used.add(name); + return name; +} + +function mentionsSub(e: Expr, idx: string): boolean { + let found = false; + const w = (x: Expr): void => { + if (found) return; + switch (x.k) { + case 'var': if (x.sub && subMentions(x.sub, idx)) found = true; return; + case 'bin': case 'dot': w(x.l); w(x.r); return; + case 'neg': case 'abs': case 'norm': case 'floor': case 'ceil': case 'fact': w(x.e); return; + case 'sqrt': w(x.e); return; + case 'pow': w(x.b); w(x.e); return; + case 'call': x.args.forEach(w); return; + case 'binom': w(x.n); w(x.m); return; + case 'big': w(x.body); return; + case 'cases': x.arms.forEach((a) => w(a.value)); return; + default: return; + } + }; + w(e); + return found; +} + +function subMentions(sub: Expr, idx: string): boolean { + switch (sub.k) { + case 'var': return !sub.sub && sub.name === idx; + case 'bin': return subMentions(sub.l, idx) || subMentions(sub.r, idx); + case 'neg': return subMentions(sub.e, idx); + default: return false; + } +} + +function collectIndices(e: Expr, out: Set<string>): void { + switch (e.k) { + case 'big': out.add(e.idx); collectIndices(e.body, out); if (e.lo) collectIndices(e.lo, out); if (e.hi) collectIndices(e.hi, out); return; + case 'bin': case 'dot': collectIndices(e.l, out); collectIndices(e.r, out); return; + case 'neg': case 'abs': case 'norm': case 'floor': case 'ceil': case 'fact': collectIndices(e.e, out); return; + case 'sqrt': collectIndices(e.e, out); return; + case 'pow': collectIndices(e.b, out); collectIndices(e.e, out); return; + case 'call': e.args.forEach((a) => collectIndices(a, out)); return; + case 'binom': collectIndices(e.n, out); collectIndices(e.m, out); return; + case 'cases': e.arms.forEach((a) => collectIndices(a.value, out)); return; + case 'var': if (e.sub) collectIndices(e.sub, out); return; + default: return; + } +} + +/** Is a subscript a position (built from index letters and numbers) rather than a label such as max or 0? */ +function isIndexExpr(e: Expr, indexNames: Set<string>): boolean { + switch (e.k) { + case 'var': return !e.sub && !e.decor && indexNames.has(e.name); + case 'num': return false; + case 'bin': return (e.op === '+' || e.op === '-' || e.op === '*') && (isIndexExpr(e.l, indexNames) || isIndexExpr(e.r, indexNames)) && (isIndexExpr(e.l, indexNames) || e.l.k === 'num') && (isIndexExpr(e.r, indexNames) || e.r.k === 'num'); + case 'neg': return isIndexExpr(e.e, indexNames); + default: return false; + } +} + +/** `a_{ij}` is two indices when the subscript is two index letters juxtaposed. */ +function subDepth(sub: Expr): number { + if (sub.k === 'bin' && sub.op === '*' && sub.l.k === 'var' && sub.r.k === 'var' && !sub.l.sub && !sub.r.sub) return 2; + return 1; +} + +function describeExpr(e: Expr): string { + switch (e.k) { + case 'num': return e.v; + case 'var': return (e.decor ? `\\${e.decor}{${e.name}}` : e.name) + (e.subText ? `_{${e.subText}}` : e.sub ? '_{...}' : ''); + case 'call': return `${e.fn}(${e.args.map(describeExpr).join(', ')})`; + case 'pow': return `${describeExpr(e.b)}^{${describeExpr(e.e)}}`; + case 'bin': return `${describeExpr(e.l)} ${e.op} ${describeExpr(e.r)}`; + default: return `a ${e.k}`; + } +} + +const PREC: Record<string, number> = { '+': 6, '-': 6, '*': 7, '/': 7, mod: 7, '^': 9 }; + +function emitExpr(e: Expr, ctx: EmitCtx, out: string[], indent: number): string { + const pad = ' '.repeat(indent); + const wrapIf = (s: string, child: Expr, parentPrec: number, right = false): string => { + if (child.k === 'bin') { const p = PREC[child.op]; if (p < parentPrec || (p === parentPrec && right)) return `(${s})`; } + if (child.k === 'neg' && parentPrec >= 7) return `(${s})`; + return s; + }; + switch (e.k) { + case 'num': return e.v; + case 'pi': return 'PI()'; + case 'var': { + if (!e.sub && !e.decor && ctx.bound.has(e.name)) return ctx.bound.get(e.name)!; + const fnName = functionName(e); + if (!e.sub && ctx.defs.has(fnName)) { const d = ctx.defs.get(fnName)!; return hoist(ctx, `${d.fnName}(${d.params.map((p) => ctx.params.get(p.key)?.name ?? p.name).join(', ')})`, pseudoName(e, false)); } + const indexSub = !!e.sub && isIndexExpr(e.sub, ctx.indexNames); + const p = ctx.params.get(varKey(e, indexSub)); + const name = p ? p.name : pseudoName(e, indexSub); + if (e.name === 'e' && !e.sub && !e.decor && !ctx.notes.some((n) => n.id === 'matho-euler')) ctx.notes.push({ id: 'matho-euler', message: 'A plain e is a name here (E), not Euler\'s number; write e^{x} for EXP(x), or EXP(1) for the constant.' }); + if (!indexSub || !e.sub) return name; + if (subDepth(e.sub) === 2 && e.sub.k === 'bin') return `${name}[${emitIndex(e.sub.l, ctx, out, indent)}][${emitIndex(e.sub.r, ctx, out, indent)}]`; + return `${name}[${emitIndex(e.sub, ctx, out, indent)}]`; + } + case 'bin': { + const p = PREC[e.op]; + const l = wrapIf(emitExpr(e.l, ctx, out, indent), e.l, p); + const r = wrapIf(emitExpr(e.r, ctx, out, indent), e.r, p, true); + return `${l} ${e.op === 'mod' ? 'MOD' : e.op} ${r}`; + } + case 'dot': return `DOT(${emitExpr(e.l, ctx, out, indent)}, ${emitExpr(e.r, ctx, out, indent)})`; + case 'neg': { const s = emitExpr(e.e, ctx, out, indent); return `-${e.e.k === 'bin' ? `(${s})` : s}`; } + case 'pow': { + const b = emitExpr(e.b, ctx, out, indent); + const x = emitExpr(e.e, ctx, out, indent); + const bs = e.b.k === 'bin' || e.b.k === 'neg' || e.b.k === 'pow' ? `(${b})` : b; + if (e.e.k === 'bin' && e.e.op === '/' && e.e.l.k === 'num' && e.e.l.v === '1' && e.e.r.k === 'num' && e.e.r.v === '2') return `SQRT(${b})`; + const xs = e.e.k === 'bin' || e.e.k === 'neg' ? `(${x})` : x; + return `${bs} ^ ${xs}`; + } + case 'call': { + const fnName = functionName({ k: 'var', name: e.fn, decor: null, sub: null, subText: null, pos: e.pos }); + if (ctx.defs.has(fnName)) { + const d = ctx.defs.get(fnName)!; + const args = e.args.map((a) => emitExpr(a, ctx, out, indent)); + const rest = d.params.slice(e.args.length).map((p) => ctx.params.get(p.key)?.name ?? p.name); + const text = `${d.fnName}(${[...args, ...rest].join(', ')})`; + const usesIndex = [...ctx.bound.values()].some((n) => new RegExp(`\\b${n}\\b`).test(args.join(','))); + return usesIndex ? text : hoist(ctx, text, baseWord(e.fn)); + } + if (e.fn === 'ATAN') return `ATAN2(${emitExpr(e.args[0], ctx, out, indent)}, 1)`; + return `${e.fn}(${e.args.map((a) => emitExpr(a, ctx, out, indent)).join(', ')})`; + } + case 'sqrt': { + const x = emitExpr(e.e, ctx, out, indent); + if (!e.n) return `SQRT(${x})`; + return `${e.e.k === 'bin' || e.e.k === 'neg' ? `(${x})` : x} ^ (1 / ${emitExpr(e.n, ctx, out, indent)})`; + } + case 'abs': return `ABS(${emitExpr(e.e, ctx, out, indent)})`; + case 'norm': return `NORM_OF(${emitExpr(e.e, ctx, out, indent)})`; + case 'floor': return `FLOOR(${emitExpr(e.e, ctx, out, indent)})`; + case 'ceil': return `CEIL(${emitExpr(e.e, ctx, out, indent)})`; + case 'binom': return `CHOOSE(${emitExpr(e.n, ctx, out, indent)}, ${emitExpr(e.m, ctx, out, indent)})`; + case 'fact': { + // n! as a product loop + const n = emitExpr(e.e, ctx, out, indent); + const acc = fresh(ctx, 'Factorial'); + const k = fresh(ctx, 'K'); + ctx.decls.push({ name: acc, type: 'FLOAT' }); + out.push(`${pad}${acc} <- 1.0`, `${pad}FOR ${k} <- 2 TO ${e.e.k === 'num' || e.e.k === 'var' ? n : `(${n})`} + 1`, `${pad} ${acc} <- ${acc} * ${k}`, `${pad}NEXT ${k}`); + if (!ctx.notes.some((x) => x.id === 'matho-factorial')) ctx.notes.push({ id: 'matho-factorial', message: `${describeExpr(e.e)}! became a product loop, ${acc}, run before the RETURN.` }); + return acc; + } + case 'big': { + const isSum = e.op === 'sum'; + const acc = fresh(ctx, isSum ? 'Total' : 'Product'); + ctx.decls.push({ name: acc, type: 'FLOAT' }); + const idxName = fresh(ctx, baseWord(e.idx)); + out.push(`${pad}${acc} <- ${isSum ? '0.0' : '1.0'}`); + const inner: string[] = []; + let shift = 0; + if (e.over) { + out.push(`${pad}FOR EACH ${idxName} IN ${emitExpr(e.over, ctx, out, indent)}`); + } else { + const lo = e.lo ? emitExpr(e.lo, ctx, out, indent) : '0'; + let hi: string; + if (!e.hi) { + // no upper bound: the length of the list the index walks + const list = listIndexedBy(e.body, e.idx, ctx); + if (!list) throw new ParseError(`\\${e.op}_{${e.idx}} has no upper bound and ${e.idx} indexes no list, so the reader cannot tell how many passes to make. Write \\${e.op}_{${e.idx}=0}^{n-1}.`, e.pos, 'matho-sum-bounds'); + hi = `LENGTH(${list})`; + ctx.notes.push({ id: 'matho-sum-length', message: `\\${e.op}_{${e.idx}} has no bounds, so the loop runs over every position of ${list}.` }); + } else hi = inclusiveToExclusive(e.hi, ctx, out, indent); + if (e.lo && e.lo.k === 'num' && e.lo.v === '1' && e.hi && (e.hi.k === 'var' || (e.hi.k === 'bin' && e.hi.op !== '-'))) shift = 1; + out.push(`${pad}FOR ${idxName} <- ${lo} TO ${hi}`); + } + const wasBound = ctx.bound.get(e.idx); + const wasShift = ctx.shifts.get(e.idx); + ctx.bound.set(e.idx, idxName); + if (shift) ctx.shifts.set(e.idx, shift); else ctx.shifts.delete(e.idx); + ctx.loopDepth++; + const body = emitExpr(e.body, ctx, inner, indent + 1); + ctx.loopDepth--; + if (shift && ctx.shifted.has(e.idx)) ctx.notes.push({ id: 'matho-one-based', message: `The ${isSum ? 'sum' : 'product'} runs from ${e.idx} = 1, and lists in pseudo start at position 0, so a subscript ${e.idx} became a position ${idxName} - 1.` }); + ctx.shifted.delete(e.idx); + if (wasBound === undefined) ctx.bound.delete(e.idx); else ctx.bound.set(e.idx, wasBound); + if (wasShift === undefined) ctx.shifts.delete(e.idx); else ctx.shifts.set(e.idx, wasShift); + const needsParens = e.body.k === 'bin' && (isSum ? e.body.op === '+' || e.body.op === '-' : e.body.op !== '*'); + out.push(...inner, `${pad} ${acc} <- ${acc} ${isSum ? '+' : '*'} ${needsParens ? `(${body})` : body}`, `${pad}NEXT ${idxName}`); + return acc; + } + case 'cases': throw new ParseError('\\begin{cases} can only be the whole right-hand side of an equation.', 0, 'matho-cases-place'); + } +} + +/** Inside a loop, a value that does not change from pass to pass is computed once, before the loop. */ +function hoist(ctx: EmitCtx, callText: string, nameHint: string): string { + if (ctx.loopDepth === 0) return callText; + const have = ctx.hoisted.get(callText); + if (have) return have; + const local = fresh(ctx, nameHint); + ctx.decls.push({ name: local, type: 'FLOAT' }); + ctx.prelude.push(` ${local} <- ${callText}`); + ctx.hoisted.set(callText, local); + ctx.notes.push({ id: 'matho-hoist', message: `${callText} does not change from one pass of the loop to the next, so it is computed once, as ${local}, before the loop.` }); + return local; +} + +/** A subscript as a list position, with the 1-based shift applied to bare index letters. */ +function emitIndex(e: Expr, ctx: EmitCtx, out: string[], indent: number): string { + const s = emitExpr(e, ctx, out, indent); + if (e.k === 'var' && !e.sub && ctx.shifts.get(e.name)) { ctx.shifted.add(e.name); return `${s} - ${ctx.shifts.get(e.name)}`; } + if (e.k === 'bin') { + // i+1 with a shift of 1 is just I + const shiftOf = (x: Expr): number => x.k === 'var' && !x.sub ? ctx.shifts.get(x.name) ?? 0 : 0; + const sh = shiftOf(e.l) || shiftOf(e.r); + if (sh) { + ctx.shifted.add(e.l.k === 'var' && shiftOf(e.l) ? e.l.name : (e.r as any).name); + if (e.op === '+' && e.r.k === 'num' && Number(e.r.v) === sh) return emitExpr(e.l, ctx, out, indent); + return `${s} - ${sh}`; + } + } + return s; +} + +/** The upper bound of a sum is inclusive; FOR ... TO is exclusive. `n-1` → N, `n` → N + 1, `5` → 6. */ +function inclusiveToExclusive(hi: Expr, ctx: EmitCtx, out: string[], indent: number): string { + if (hi.k === 'bin' && hi.op === '-' && hi.r.k === 'num' && hi.r.v === '1') return emitExpr(hi.l, ctx, out, indent); + if (hi.k === 'num' && /^\d+$/.test(hi.v)) return String(Number(hi.v) + 1); + const s = emitExpr(hi, ctx, out, indent); + return hi.k === 'bin' ? `${s} + 1` : `${s} + 1`; +} + +function listIndexedBy(body: Expr, idx: string, ctx: EmitCtx): string | null { + let found: string | null = null; + const w = (e: Expr): void => { + if (found) return; + switch (e.k) { + case 'var': if (e.sub && e.sub.k === 'var' && e.sub.name === idx) { const p = ctx.params.get(varKey(e, true)); found = p ? p.name : pseudoName(e, true); } return; + case 'bin': case 'dot': w(e.l); w(e.r); return; + case 'neg': case 'abs': case 'norm': case 'floor': case 'ceil': case 'fact': case 'sqrt': w(e.e); return; + case 'pow': w(e.b); w(e.e); return; + case 'call': e.args.forEach(w); return; + case 'big': w(e.body); return; + default: return; + } + }; + w(body); + return found; +} + +function emitCond(c: Cond, ctx: EmitCtx, out: string[], indent: number): string { + switch (c.k) { + case 'cmp': return `${emitExpr(c.l, ctx, out, indent)} ${c.op} ${emitExpr(c.r, ctx, out, indent)}`; + case 'and': return `${emitCond(c.l, ctx, out, indent)} AND ${emitCond(c.r, ctx, out, indent)}`; + case 'or': return `(${emitCond(c.l, ctx, out, indent)}) OR (${emitCond(c.r, ctx, out, indent)})`; + case 'not': return `NOT (${emitCond(c.c, ctx, out, indent)})`; + } +} + +/* ------------------------------------------------------------------ */ +/* Splitting a document into equations */ +/* ------------------------------------------------------------------ */ + +const WRAPPERS = /\\\[|\\\]|\\begin\{(?:equation|align|gather|eqnarray|math|displaymath)\*?\}|\\end\{(?:equation|align|gather|eqnarray|math|displaymath)\*?\}|\$\$|\\nonumber|\\notag/g; + +/** Lines that belong together: an equation continues while braces or a cases block are open, or the next line starts with an operator or `=`. */ +export function splitEquations(src: string): Array<{ line: number; text: string }> { + const lines = src.split('\n'); + const out: Array<{ line: number; text: string }> = []; + let cur: { line: number; text: string } | null = null; + let depth = 0; + let inCases = 0; + const flush = () => { if (cur && cur.text.trim()) out.push(cur); cur = null; }; + lines.forEach((raw, i) => { + let line = raw.replace(/%.*$/, '').replace(WRAPPERS, ' '); + if (!line.trim()) { if (depth <= 0 && inCases <= 0) flush(); return; } + // an unclosed brace keeps the equation open, unless the next line plainly starts a new one (a fresh `=` outside any cases) + const freshEquation = depth > 0 && inCases <= 0 && /^[^=&]*[A-Za-z}]\s*=/.test(line) && !/^\s*(?:&\s*)?=/.test(line); + const continues = !freshEquation && (depth > 0 || inCases > 0 || /^\s*(?:&\s*)?(?:=|\+|-|\\cdot|\\times)/.test(line) || (cur && /(?:=|\+|-|\\cdot|\\times|\\frac|\\left\(|\{)\s*$/.test(cur.text))); + if (freshEquation) depth = 0; + if (!continues) flush(); + if (inCases <= 0 && !/\\begin\{cases\}/.test(line)) line = line.replace(/&/g, ' '); // alignment marks outside cases + if (!cur) cur = { line: i + 1, text: line }; + else cur.text += '\n' + line; + for (const ch of line) { if (ch === '{') depth++; else if (ch === '}') depth--; } + inCases += (line.match(/\\begin\{cases\}/g) || []).length - (line.match(/\\end\{cases\}/g) || []).length; + }); + flush(); + // a trailing \\ ends an equation in align; drop it + for (const e of out) e.text = e.text.replace(/\\\\\s*$/, '').replace(/^\s*&/, ''); + return out; +} + +/* ------------------------------------------------------------------ */ +/* The service */ +/* ------------------------------------------------------------------ */ + +export function parseMatho(source: string): Parsed<MathoProgram> { + const diagnostics: Diagnostic[] = []; + const equations: Equation[] = []; + const known = new Set<string>(); + const chunks = splitEquations(source); + if (chunks.length > MAX_EQUATIONS) { + diagnostics.push({ lang: 'matho', severity: 'error', id: 'matho-too-long', line: chunks[MAX_EQUATIONS].line, message: `This pane has more than ${MAX_EQUATIONS} equations, which is more than this reader reads. Keep a pane to the equations of one computation.` }); + return { program: { equations: [], pseudo: '', lineMap: [], defs: [] }, diagnostics, ok: false }; + } + for (const chunk of chunks) { + const { toks, error } = tokenize(chunk.text); + if (error) diagnostics.push({ lang: 'matho', severity: 'error', id: 'matho-character', line: chunk.line, message: error }); + if (toks.length > MAX_TOKENS) { + diagnostics.push({ lang: 'matho', severity: 'error', id: 'matho-too-long', line: chunk.line, message: `This equation has more than ${MAX_TOKENS.toLocaleString('en-US')} symbols, which is more than this reader reads. Split it into smaller equations, each naming a part the next one uses.` }); + continue; + } + const p = new Parser(toks, known); + try { + const eq = p.equation(); + equations.push({ line: chunk.line, source: chunk.text, lhs: eq.lhs, rhs: eq.rhs }); + if (eq.lhs?.k === 'call') known.add(eq.lhs.fn); + if (eq.lhs?.k === 'var' && !eq.lhs.sub && !eq.lhs.decor && eq.lhs.name.length === 1) known.add(eq.lhs.name); + } catch (e: any) { + if (e instanceof ParseError) { + const before = chunk.text.slice(0, e.pos); + const line = chunk.line + (before.match(/\n/g) || []).length; + const col = before.length - before.lastIndexOf('\n'); + diagnostics.push({ lang: 'matho', severity: 'error', id: e.id, line, col, message: e.message, guide: e.id === 'matho-refused' ? 'matho-what-a-program-can-be' : undefined }); + } else throw e; + } + } + const low = new Lowerer(); + try { + low.lower(equations); + } catch (e: any) { + if (e instanceof ParseError) { + const eq = equations[Math.min(low.outDefs.length, equations.length - 1)]; + diagnostics.push({ lang: 'matho', severity: 'error', id: e.id, line: eq ? eq.line : 1, message: e.message }); + return { program: { equations, pseudo: '', lineMap: [], defs: [] }, diagnostics, ok: false }; + } + throw e; + } + diagnostics.push(...low.diagnostics); + if (low.lines.length > MAX_PSEUDO_LINES) { + diagnostics.push({ lang: 'matho', severity: 'error', id: 'matho-too-long', line: equations[0]?.line ?? 1, message: `These equations become more than ${MAX_PSEUDO_LINES.toLocaleString('en-US')} lines of pseudocode, which is more than this reader follows. Keep a pane to the equations of one computation.` }); + return { program: { equations, pseudo: '', lineMap: [], defs: [] }, diagnostics, ok: false }; + } + const pseudo = low.lines.join('\n').replace(/\n+$/, '\n'); + // a translation that does not read as pseudo is a bug in the reader; surface it rather than hide it + const st = studyOf(pseudo); + for (const d of st.diagnostics.filter((x) => x.severity === 'error')) { + const k = low.lineMap[d.line - 1] ?? 0; + diagnostics.push({ lang: 'matho', severity: 'error', id: 'matho-pseudo', line: equations[k]?.line ?? 1, message: `The pseudo this became does not read (line ${d.line}: ${d.message}). The translation is too literal here; open the pseudo pane and finish it by hand.` }); + } + return { program: { equations, pseudo, lineMap: low.lineMap, defs: low.outDefs }, diagnostics, ok: !diagnostics.some((d) => d.severity === 'error') && equations.length > 0 }; +} + +export function lowerMatho(parsed: Parsed<MathoProgram>, target: string): Lowered | null { + const p = parsed.program; + if (!p || !p.pseudo || target !== 'pseudo') return null; + return { + target: 'pseudo', + source: p.pseudo, + map: p.lineMap.map((k, i) => ({ from: p.equations[k]?.line ?? 0, to: i + 1 })).filter((m) => m.from > 0), + notes: [], + }; +} + +/** A call that fits the last function: lists of four values, n = 4, indices 1, other numbers 2.0. */ +export function defaultCallMatho(parsed: Parsed<MathoProgram>): string | null { + const p = parsed.program; + if (!p || !p.defs.length) return null; + const d = p.defs[p.defs.length - 1]; + const sample = (q: { name: string; type: string }): string => { + if (q.type === 'LIST OF FLOAT') return '[1.0, 2.0, 3.0, 4.0]'; + if (q.type === 'LIST OF LIST OF FLOAT') return '[[1.0, 2.0], [3.0, 4.0]]'; + if (q.type === 'INTEGER') return /^(N|M|Len|Count)/.test(q.name) ? '4' : '1'; + return /^(X|T|P|Q|Prob|Theta|Alpha)\d*$/.test(q.name) ? '0.5' : '2.0'; + }; + // when a matrix is passed, its bounds are 2 + const hasMatrix = d.params.some((q) => q.type === 'LIST OF LIST OF FLOAT'); + return `${d.name}(${d.params.map((q) => (hasMatrix && q.type === 'INTEGER' ? '2' : sample(q))).join(', ')})`; +} + +export function runMatho(parsed: Parsed<MathoProgram>, options: RunOptions): Trace { + const low = lowerMatho(parsed, 'pseudo'); + if (!low) throw new Error('The equations do not translate yet; see the problems view.'); + const st = studyOf(low.source); + if (!st.ok) throw new Error('The pseudo these equations became has an error: ' + (st.diagnostics.find((d) => d.severity === 'error')?.message || '')); + const tr = runPseudo(st, { ...options, call: options.call ?? defaultCallMatho(parsed) ?? pseudoDefaultCall(st.program) }); + const toMatho = new Map(low.map.map((m) => [m.to, m.from])); + const hits = new Map<number, number>(); + for (const [line, n] of tr.hits) { const e = toMatho.get(line); if (e) hits.set(e, (hits.get(e) || 0) + n); } + return { + ...tr, + lang: 'matho', + hits, + steps: tr.steps.map((s: any) => ({ ...s, pseudoLine: s.line, line: toMatho.get(s.line) ?? s.line })), + error: tr.error ? { ...tr.error, line: tr.error.line != null ? toMatho.get(tr.error.line) ?? tr.error.line : null } : null, + }; +} + +export function tokenizeMatho(line: string): Array<{ text: string; cls: string }> { + const out: Array<{ text: string; cls: string }> = []; + const re = /(%.*$)|(\\[A-Za-z]+|\\.)|(\d+(?:\.\d+)?)|([A-Za-z])|([{}])|(\s+)|(.)/g; + let m: RegExpExecArray | null; + while ((m = re.exec(line))) { + const [text, c, cmd, n, w, br] = m; + let cls = ''; + if (c) cls = 'tok-comment'; + else if (cmd) cls = REFUSED[cmd.slice(1)] ? 'tok-error' : GREEK[cmd.slice(1)] ? 'tok-name' : 'tok-keyword'; + else if (n) cls = 'tok-number'; + else if (w) cls = 'tok-name'; + else if (br) cls = 'tok-punct'; + out.push({ text, cls }); + if (c) break; + } + return out; +} + +export const mathoService: LanguageService<MathoProgram> = { + id: 'matho', + name: 'matho', + parse: parseMatho, + run: runMatho, + lower: lowerMatho, + defaultCall: defaultCallMatho, + tokenize: tokenizeMatho, +}; diff --git a/src/services/mojo.ts b/src/services/mojo.ts new file mode 100644 index 0000000..34b2975 --- /dev/null +++ b/src/services/mojo.ts @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The Mojo pane's language service: highlighting (src/mojo/tokenize), and what + * Programming+ can say before compiling (src/mojo/check: removed syntax, then + * the parser and the structure and name checks, each in the compiler's words, + * scored against godbolt in tests/mojo-check.test.mjs). Nothing runs here yet: + * Mojo runs on the real compiler through the Compile button (src/services/godbolt), + * and Programming+'s own stepping arrives with the interpreter in Stage 3. + */ +import type { LanguageService, Parsed } from '../common/types.js'; +import { tokenizeMojo, tokenizeMojoLine } from '../mojo/tokenize.js'; +import { checkMojo } from '../mojo/check.js'; +import type { Module } from '../mojo/parse.js'; + +export interface MojoProgram { source: string; module: Module } + +export function parseMojo(source: string): Parsed<MojoProgram> { + const { diagnostics, module } = checkMojo(source); + return { program: { source, module }, diagnostics, ok: !diagnostics.some((d) => d.severity === 'error') }; +} + +/** Whether the program reads standard input, which is when the input box is shown. */ +export function mojoReadsInput(source: string): boolean { + return /\binput\s*\(/.test(source.replace(/#.*$/gm, '')); +} + +export const mojoService: LanguageService<MojoProgram> = { + id: 'mojo', + name: 'Mojo', + parse: parseMojo, + tokenize: tokenizeMojoLine, + tokenizeLines: tokenizeMojo, +}; diff --git a/src/services/pseudo-study.ts b/src/services/pseudo-study.ts new file mode 100644 index 0000000..7aed12a --- /dev/null +++ b/src/services/pseudo-study.ts @@ -0,0 +1,300 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The app's reading of pseudo: the engine's `study()` (src/engines/pseudo/study.js), with one form added. + * + * DECLARE Total : INTEGER <- 0 + * + * declares a name and gives it its starting value on one line, the way Mojo writes `var total: Int = 0`. The + * two-line form (DECLARE, then an assignment) still reads as before. The engine's parser holds one statement per + * line and stays unchanged, so this module does what `study()` does, in its order (parse, + * check, analyze, coach), with one step between parse and check: + * + * 1. Each line that is `DECLARE <name> : <type> <- <value>` has its `<- <value>` blanked out with spaces, so + * the engine parses a plain DECLARE and every column on the line stays where it was. + * 2. The value is parsed on its own (`parseExpression`), its positions moved onto that line, and an ordinary + * `Assign` is put right after the `Declare` in the tree. It carries `withDeclare: true`, and the `Declare` + * carries it as `init`, so a translator can write the pair as one line (pseudo-to-mojo does). + * + * From then on everything (the checker, the data flow, the coach, the interpreter, the verifier, the fuzzer, the + * translators) sees a DECLARE and an assignment on the same line, and the run shows the declaration and the + * value it starts with as two steps of that line. + */ +import { parse, parseExpression } from '../engines/pseudo/parse.js'; +import { check } from '../engines/pseudo/check.js'; +import { analyze } from '../engines/pseudo/analyze.js'; +import { coach } from '../engines/pseudo/coach.js'; +import { libraryModule, libraryModules, generation as libraryGeneration } from '../library/registry.js'; +import type { LibraryRoutine } from '../library/registry.js'; + +const RANK: Record<string, number> = { error: 0, warning: 1, note: 2 }; + +/** One `DECLARE x : T <- value` line, as found in the source. */ +export interface DeclareInit { + line: number; + /** The names before the colon, as written. */ + names: string[]; + /** 1-based columns: the arrow, and the first character of the value. */ + arrowCol: number; + valueCol: number; + /** The value's text, without a trailing comment. */ + value: string; +} + +/** Where a `//` comment starts on a line, outside any string or character literal (-1 if none). */ +function commentStart(line: string): number { + let quote: string | null = null; + for (let i = 0; i < line.length; i++) { + const c = line[i]; + if (quote) { if (c === '\\') i++; else if (c === quote) quote = null; continue; } + if (c === '"' || c === "'") quote = c; + else if (c === '/' && line[i + 1] === '/') return i; + } + return -1; +} + +/** The first `<-` on a line outside literals, before `end` (-1 if none). */ +function arrowAt(line: string, end: number): number { + let quote: string | null = null; + for (let i = 0; i < end; i++) { + const c = line[i]; + if (quote) { if (c === '\\') i++; else if (c === quote) quote = null; continue; } + if (c === '"' || c === "'") quote = c; + else if (c === '<' && line[i + 1] === '-') return i; + } + return -1; +} + +/** + * The `DECLARE ... : type <- value` lines of a source, and the source with each `<- value` blanked out. A DECLARE + * with an arrow and no colon before it is left as written: the engine already explains that one. + */ +export function splitDeclareInits(source: string): { source: string; inits: DeclareInit[] } { + const lines = String(source ?? '').replace(/\r\n?/g, '\n').split('\n'); + const inits: DeclareInit[] = []; + lines.forEach((text, i) => { + const head = /^(\s*)DECLARE\b/i.exec(text); + if (!head) return; + const cut = commentStart(text); + const end = cut < 0 ? text.length : cut; + const arrow = arrowAt(text, end); + if (arrow < 0) return; + const colon = text.indexOf(':', head[0].length); + if (colon < 0 || colon > arrow) return; + const names = text.slice(head[0].length, colon).split(',').map((n) => n.trim()).filter(Boolean); + const raw = text.slice(arrow + 2, end); + const lead = raw.length - raw.trimStart().length; + inits.push({ line: i + 1, names, arrowCol: arrow + 1, valueCol: arrow + 3 + lead, value: raw.trim() }); + lines[i] = text.slice(0, arrow) + ' '.repeat(end - arrow) + text.slice(end); + }); + return { source: lines.join('\n'), inits }; +} + +/** Move a parsed expression from line 1 of its own text onto `line`, starting at column `col`. */ +function place(node: any, line: number, col: number): void { + if (!node || typeof node !== 'object') return; + if (Array.isArray(node)) { for (const x of node) place(x, line, col); return; } + for (const [k, v] of Object.entries(node)) { + if (k === 'line' && typeof v === 'number') node.line = line; + else if ((k === 'col' || k.endsWith('Col')) && typeof v === 'number') node[k] = v + col - 1; + else if (k === 'end' && typeof v === 'number') node.end = v + col - 1; + else if (v && typeof v === 'object') place(v, line, col); + } +} + +/** + * The Declare on each line and the statement list that holds it, from one walk of the tree: the first list in a + * depth-first walk that holds a Declare of that line directly. + */ +function declaresByLine(root: any): Map<number, { list: any[]; decl: any }> { + const out = new Map<number, { list: any[]; decl: any }>(); + const seen = new Set<any>(); + const walk = (node: any): void => { + if (!node || typeof node !== 'object' || seen.has(node)) return; + seen.add(node); + if (Array.isArray(node)) { + for (const s of node) if (s && s.kind === 'Declare' && !out.has(s.line)) out.set(s.line, { list: node, decl: s }); + for (const x of node) walk(x); + return; + } + for (const v of Object.values(node)) walk(v); + }; + walk(root); + return out; +} + +/** Put each assignment right after its Declare, rebuilding each statement list once. */ +function spliceAfterDeclares(after: Map<any[], Map<any, any>>): void { + for (const [list, adds] of after) { + const old = list.slice(); + list.length = 0; + for (const s of old) { list.push(s); if (adds.has(s)) list.push(adds.get(s)); } + } +} + +const diag = (severity: string, code: string, line: number, col: number, message: string, extra: object = {}) => + ({ severity, code, line, col, message, ...extra }); + +/** + * What the engine's `study(source, options)` returns, for a source that may also declare and assign on one line. + * `source` is the source as written; `parsed` is the engine's parse of the blanked source, with the assignments + * put in. + */ +/** Studied library routines, by generation, id and source, so a routine is checked once however many programs use it. */ +const routineCache = new Map<string, any>(); + +/** A library routine, studied as a program of its own (its own USE lines resolved the same way). */ +function studyRoutine(r: LibraryRoutine, stack: string[]): any { + const key = `${libraryGeneration}\u0000${r.id}\u0000${r.source}`; + let st = routineCache.get(key); + if (!st) { + st = studyPseudo(r.source, { coach: false, libraryStack: [...stack, r.id] }); + routineCache.set(key, st); + } + return st; +} + +/** + * The routines a program's USE lines bring in: UPPER name -> the checked routine (its own program's FUNCTION or + * PROCEDURE, marked with `lib`), where each came from, and what could not be found. + */ +export function resolveUses(body: any[], stack: string[] = []): { libs: Map<string, any>; origin: Map<string, { module: string; line: number }>; diags: any[] } { + const libs = new Map<string, any>(); + const origin = new Map<string, { module: string; line: number }>(); + const diags: any[] = []; + for (const u of body) { + if (u?.kind !== 'Use' || u.ieee) continue; + const m = libraryModule(u.module); + if (!m) { + const names = libraryModules().map((x) => x.name); + diags.push(diag('error', 'unknown-library', u.line, u.moduleCol ?? u.col, `No module of the library is called ${u.module}.${names.length ? ` The modules are ${names.join(', ')}.` : ' The library has no modules in this build.'}`, { guide: 'libraries' })); + continue; + } + let routines = m.routines; + if (u.routine) { + const r = m.routines.find((x) => x.name.toUpperCase() === String(u.routine).toUpperCase()); + if (!r) { diags.push(diag('error', 'unknown-routine', u.line, u.routineCol ?? u.col, `The module ${m.name} has no routine called ${u.routine}. Its routines are ${m.routines.map((x) => x.name).join(', ')}.`, { guide: 'libraries' })); continue; } + routines = [r]; + } + for (const r of routines) { + if (stack.includes(r.id)) { diags.push(diag('error', 'library-cycle', u.line, u.col, `${r.id} is already being brought in by ${stack[stack.length - 1]}, so the routines would use each other in a circle.`, { guide: 'libraries' })); continue; } + const st = studyRoutine(r, stack); + const sub = st.ok ? st.program.subs.get(r.name.toUpperCase()) : null; + if (!sub) { + const why = st.errors?.[0] ? `line ${st.errors[0].line}: ${st.errors[0].message}` : `it defines no routine called ${r.name}`; + diags.push(diag('error', 'library-broken', u.line, u.col, `The library routine ${r.id} does not check (${why}). That is a fault in the library, not in this program.`, { guide: 'libraries' })); + continue; + } + sub.lib = { module: m.name, id: r.id, program: st.program, source: r.source }; + const k = r.name.toUpperCase(); + const prior = origin.get(k); + if (prior && prior.module.toUpperCase() !== m.name.toUpperCase()) { + diags.push(diag('error', 'library-clash', u.line, u.col, `${r.name} comes from both ${prior.module} (line ${prior.line}) and ${m.name}. Bring in only the one you mean: USE ${m.name}.<Name>.`, { guide: 'libraries' })); + continue; + } + libs.set(k, sub); + origin.set(k, { module: m.name, line: u.line }); + } + } + return { libs, origin, diags }; +} + +/** + * The longest program read: 2,000 lines, as for the pseudocode lingo steps and matho equations become, and + * 100,000 characters (docs/limits.md). Past either, the lines before the one that passes it are read, and that line + * has one `too-long` error: the analysis of a routine grows with the square of its length (which definitions reach + * each use), and every pane, the prover and the fuzzer start from this reading. The corpus's longest program is 58 + * lines and 1,500 characters. + */ +export const MAX_PSEUDO_LINES = 2000; +export const MAX_PSEUDO_CHARS = 100000; + +/** The program a study reads: all of it, or the lines before the limit and the error that says where it is. */ +function withinLimits(original: string): { read: string; tooLong: any } { + // lines as the parser counts them (a lone CR ends one too); blank lines past a limit do not count + const lines = original.split(/\r\n?|\n/); + let chars = 0; + for (let i = 0; i < lines.length; i++) { + chars += lines[i].length + 1; + const pastLines = i >= MAX_PSEUDO_LINES; + if (!pastLines && chars <= MAX_PSEUDO_CHARS) continue; + if (!lines.slice(i).some((l) => l.trim())) break; + const what = pastLines ? `more than ${MAX_PSEUDO_LINES.toLocaleString('en-US')} lines long` : `longer than ${MAX_PSEUDO_CHARS.toLocaleString('en-US')} characters`; + return { + read: lines.slice(0, i).join('\n'), + tooLong: diag('error', 'too-long', i + 1, 1, `This program is ${what}, which is more than this tool reads: the lines from here on are not read, and the program is not run or proved. Keep a pane to one algorithm at a time.`), + }; + } + return { read: original, tooLong: null }; +} + +export function studyPseudo(source: string, options: { coach?: boolean; strict?: boolean; libraryStack?: string[] } = {}): any { + const original = String(source ?? ''); + const { read, tooLong } = withinLimits(original); + const { source: blanked, inits } = splitDeclareInits(read); + const parsed: any = parse(blanked); + const extra: any[] = []; + const blankedLines = inits.length ? blanked.split('\n') : []; + const declares = inits.length ? declaresByLine(parsed.ast) : new Map<number, { list: any[]; decl: any }>(); + const after = new Map<any[], Map<any, any>>(); + for (const it of inits) { + if (it.names.length !== 1) { + extra.push(diag('error', 'declare-init-names', it.line, it.arrowCol, `A DECLARE with a starting value names one variable, as in DECLARE ${it.names[0] || 'Total'} : INTEGER <- 0. Give ${it.names.slice(1).join(', ')} a line of its own.`, { guide: 'declaring' })); + continue; + } + if (!it.value) { + extra.push(diag('error', 'declare-init-empty', it.line, it.arrowCol, `The declaration of ${it.names[0]} has nothing on the right of its arrow. Write its starting value, as in DECLARE ${it.names[0]} : INTEGER <- 0, or leave the arrow off.`, { guide: 'declaring' })); + continue; + } + const p: any = parseExpression(it.value); + const errs = (p.diags as any[]).filter((d) => d.severity === 'error'); + if (errs.length || !p.expr) { + for (const d of errs.length ? errs : [diag('error', 'expected', 1, 1, 'The starting value is not an expression.')]) extra.push({ ...d, line: it.line, col: (d.col ?? 1) + it.valueCol - 1 }); + continue; + } + place(p.expr, it.line, it.valueCol); + const name = it.names[0]; + // the blanked source has the line endings the line numbers were counted with (a lone CR ends a line too) + const nameCol = (blankedLines[it.line - 1] ?? '').indexOf(name) + 1; + const at = declares.get(it.line); + if (at) { + const assign = { kind: 'Assign', line: it.line, col: nameCol, target: { kind: 'Name', name, line: it.line, col: nameCol }, value: p.expr, withDeclare: true }; + at.decl.init = assign; + if (!after.has(at.list)) after.set(at.list, new Map()); + after.get(at.list)!.set(at.decl, assign); + } + } + spliceAfterDeclares(after); + if (extra.length) parsed.diags = [...parsed.diags, ...extra].sort((a: any, b: any) => a.line - b.line || a.col - b.col); + + // the routines USE brings in, checked in their own files first + const uses = resolveUses(parsed.ast.body, options.libraryStack || []); + if (uses.diags.length) parsed.diags = [...parsed.diags, ...uses.diags].sort((a: any, b: any) => a.line - b.line || a.col - b.col); + + // from here on, what study() does (src/engines/pseudo/study.js), with the library's routines in scope + const checked = check(parsed, { libs: uses.libs, libOrigin: uses.origin }); + const program = checked.program; + // a block the cut leaves open is the cut's doing, and the too-long error says so + const diags = tooLong ? [...checked.diags.filter((d: any) => d.code !== 'missing-closer'), tooLong] : checked.diags; + const errors = diags.filter((d: any) => d.severity === 'error'); + let analysis = null; + let coached: any[] = []; + const structural = parsed.diags.some((d: any) => d.severity === 'error' && ['missing-closer', 'stray-closer', 'next-mismatch'].includes(d.code)); + if (!structural) { + try { + analysis = analyze(program); + if (options.coach !== false) coached = coach(program, analysis, options); + } catch (err) { + if (options.strict) throw err; + analysis = null; + } + } + const all = [...diags, ...coached].map(reword).sort((a: any, b: any) => a.line - b.line || RANK[a.severity] - RANK[b.severity] || (a.col || 0) - (b.col || 0)); + return { source: original, parsed, program, analysis, diags: all, errors: errors.map(reword), blanks: program.blanks, ok: errors.length === 0 }; +} + +/** The engine's advice that a declaration "gives no value" is out of date once one can: say what the form is. */ +function reword(d: any): any { + if (d.code === 'declare-assign') return { ...d, message: 'A declaration needs its type before a starting value: DECLARE Total : INTEGER <- 0, or DECLARE Total : INTEGER and then Total <- 0 on the next line.' }; + return d; +} diff --git a/src/services/pseudo.ts b/src/services/pseudo.ts new file mode 100644 index 0000000..6a4db8e --- /dev/null +++ b/src/services/pseudo.ts @@ -0,0 +1,220 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The pseudocode service: wraps the course's parse/check/coach/interp/ + * translate/vcgen/testing modules behind the LanguageService interface. + * The engine modules are vendored unchanged under ../engines/pseudo. + */ +import { studyPseudo as study } from './pseudo-study.js'; +import { run as runProgram, evaluateConstant, valueFits } from '../engines/pseudo/interp.js'; +import { parseExpression, sourceOf } from '../engines/pseudo/parse.js'; +import { verify } from '../engines/pseudo/vcgen.js'; +import { checkProperties } from '../engines/pseudo/testing.js'; +import { typeName, PArray, formatValue, KEYWORDS, BUILTINS } from '../engines/pseudo/lang.js'; +import type { Diagnostic, LanguageService, Parsed, ProofReport, PropertyReport, RunOptions, Trace } from '../common/types.js'; + +/* The engine's own shapes, typed loosely: they are JavaScript. */ +export type Study = any; +export type PseudoState = any; +export type PseudoStep = any; + +export function toDiagnostic(d: any, lang: 'pseudo' | 'lingo' = 'pseudo'): Diagnostic { + const out: Diagnostic = { + lang, + severity: d.severity, + id: d.code || d.id || 'diag', + line: d.line ?? 1, + col: d.col ?? 1, + message: d.message, + }; + if (d.guide) out.guide = d.guide; + if (d.fix && d.fix.col != null) out.fix = { label: 'Fix', line: d.fix.line ?? d.line, col: d.fix.col, endCol: d.fix.endCol ?? null, text: d.fix.text }; + if (d.question?.options?.length) { + out.question = { + text: d.question.prompt || d.question.text || 'Which did you mean?', + options: d.question.options.map((o: any) => { + const r = o.replace || o.insert || {}; + return { label: o.label, line: r.line ?? d.line, col: r.col ?? d.col ?? 1, endCol: o.insert ? r.col ?? d.col ?? 1 : (r.endCol ?? null), text: r.text ?? '' }; + }), + }; + } + return out; +} + +export function studyOf(source: string): Parsed<Study> { + const s = study(source, { coach: true }); + return { program: s, diagnostics: s.diags.map((d: any) => toDiagnostic(d)), ok: s.ok }; +} + +function resolveCall(program: any, call: string) { + const p = parseExpression(call); + if (p.diags.some((d: any) => d.severity === 'error') || p.expr.kind !== 'CallExpr') throw new Error(`"${call}" is not a call. Write the subroutine's name and its arguments, as in COUNT_BASE('A', "GATTACA").`); + const sub = program.subs.get(p.expr.name.toUpperCase()) || program.libs?.get(p.expr.name.toUpperCase()); + if (!sub) throw new Error(`The program has no FUNCTION or PROCEDURE called ${p.expr.name}.`); + if (p.expr.args.length !== sub.params.length) throw new Error(`${sub.name} takes ${sub.params.length} value(s), and the call gives ${p.expr.args.length}.`); + const args = p.expr.args.map((a: any, i: number) => { + let v = evaluateConstant(program, a); + const t = sub.params[i].type; + if (t.k === 'ARRAY' && Array.isArray(v)) { const arr = new PArray([v.length]); arr.data = v; v = arr; } + if (t.k === 'SET' && Array.isArray(v)) v = new Set(v); + if (!valueFits(t, v)) throw new Error(`Parameter ${sub.params[i].name} of ${sub.name} is ${typeName(t)}, and the call gives ${sourceOf(a)}.`); + return v; + }); + return { name: sub.name, args, text: call }; +} + +export function defaultCall(s: Study): string | null { + const program = s?.program; + if (!program) return null; + if (program.main?.body?.length) return null; + const subs = [...program.subs.values()] as any[]; + const sub = subs.find((x) => x.kind === 'Function') || subs[0]; + if (!sub) return null; + const sample = (t: any): string => { + switch (t?.k) { + case 'STRING': return '"GATTACA"'; + case 'CHAR': return "'A'"; + case 'INTEGER': return '3'; + case 'FLOAT': return '0.5'; + case 'BOOL': return 'TRUE'; + case 'LIST': return t.of?.k === 'STRING' ? '["GATTACA", "GGC"]' : t.of?.k === 'FLOAT' ? '[0.5, 1.5]' : '[12, 30, 25, 8]'; + default: return '[]'; + } + }; + return `${sub.name}(${sub.params.map((p: any) => sample(p.type)).join(', ')})`; +} + +export function runPseudo(parsed: Parsed<Study>, options: RunOptions): Trace<PseudoState, PseudoStep> { + const s = parsed.program; + if (!s || !parsed.ok) throw new Error('The program has errors; fix them before running.'); + const text = options.call ?? defaultCall(s); + const call = text ? resolveCall(s.program, text) : null; + const tr = runProgram(s.program, { call, input: options.input || [], maxSteps: options.maxSteps ?? 20000, ...(options.maxVisits != null ? { maxVisits: options.maxVisits } : {}), contracts: true, seed: 1 }); + const output = tr.output.slice(); + if (call && tr.result !== undefined && !tr.error) output.push(`${call.name} returned ${formatValue(tr.result, tr.resultType)}`); + return { + lang: 'pseudo', + steps: tr.steps, + states: tr.states, + count: tr.count, + output, + error: tr.error ? { kind: tr.error.kind, message: tr.error.message, line: tr.error.line } : null, + truncated: !!tr.truncated, + hits: tr.hits, + raw: tr, + }; +} + +/** + * The work (steps and quantifier positions, testing.js `maxWork`) the generated inputs that cross-check a proof may + * take for one routine and for a whole program. The corpus's routines take at most 350,000. + */ +export const CROSS_CHECK_ROUTINE = 600_000; +export const CROSS_CHECK_PROGRAM = 1_200_000; + +export function provePseudo(parsed: Parsed<Study>): ProofReport { + const s = parsed.program; + if (!s || !parsed.ok) return { units: [], counts: { proved: 0, broken: 0, open: 0, total: 0 } }; + const r = verify(s.program, { budgetMs: 3000 }); + // Cross-check unproved obligations against generated inputs: an obligation + // a run breaks is "broken", the rest stay "open". + const brokenKeys = new Set<string>(); + const brokenAt = new Set<string>(); + // the inputs share a budget of work (steps and quantifier positions), each routine's and the program's: a routine + // past it is not tried, and its unproved obligations stay open + let left = CROSS_CHECK_PROGRAM; + for (const sub of s.program.subs.values()) { + if (left <= 0) break; + try { + // and the same wall-clock budget the verifier has, per routine: a slow routine is tried on fewer inputs + const pr = checkProperties(s.program, sub.name, { count: 150, budgetMs: 3000, maxWork: Math.min(left, CROSS_CHECK_ROUTINE) }); + left -= (pr as { work?: number }).work ?? 0; + // a failure that says which half of a contract it broke (an INVARIANT before the first pass or after one, a + // DECREASES below 0 or not smaller) counts against that obligation only + for (const f of pr.failures) { brokenKeys.add(`${f.kind}@${f.line}`); brokenAt.add(`${f.kind}@${f.line}:${f.col}:${f.phase || ''}`); if (!f.phase) for (const ph of ['init', 'keep', 'bound', 'step']) brokenAt.add(`${f.kind}@${f.line}:${f.col}:${ph}`); } + } catch { /* a subroutine whose parameters cannot be generated */ } + } + // A run outranks the prover: an obligation marked proved that a generated + // input breaks at the same place is shown as broken. That would be a + // soundness bug, which tests/prover-soundness.test.mjs exists to catch first. + const statusOf = (o: any): 'proved' | 'broken' | 'open' => { + if (o.status === 'proved') return brokenAt.has(`${kindKey(o.kind)}@${o.line}:${o.col}:${PHASE_OF[o.kind] || ''}`) ? 'broken' : 'proved'; + return brokenKeys.has(`${kindKey(o.kind)}@${o.line}`) ? 'broken' : 'open'; + }; + const units = r.units.map((u: any) => ({ + name: u.name, kind: u.kind, line: u.line, partial: !!u.partial, + inferred: (u.inferred || []).flatMap((x: any) => (typeof x === 'string' ? [x] : (x.facts || []).map((f: string) => `line ${x.line}: ${f}`))), + notes: (u.notes || []).map((n: any) => n.text || String(n)), + obligations: u.obligations.map((o: any) => { + const status = statusOf(o); + return { + kind: o.kind, label: o.label, line: o.line, col: o.col, paths: o.paths, + status: u.partial && status === 'proved' ? 'open' : status, + facts: o.shown?.facts || [], goal: o.shown?.goal || '', reason: o.shown?.reason, + }; + }), + })); + const all = units.flatMap((u: any) => u.obligations); + return { + units, + counts: { + proved: all.filter((o: any) => o.status === 'proved').length, + broken: all.filter((o: any) => o.status === 'broken').length, + open: all.filter((o: any) => o.status === 'open').length, + total: all.length, + }, + }; +} + +/** The half of a contract an obligation is about, as a runtime error names it in `phase`. */ +const PHASE_OF: Record<string, string> = { 'invariant-init': 'init', 'invariant-keep': 'keep', 'decreases-bound': 'bound', 'decreases-step': 'step' }; + +/** The verifier's obligation kinds vs. the runtime error kinds `checkProperties` reports. */ +function kindKey(k: string): string { + return ({ index: 'index', key: 'key', slice: 'index', divide: 'divide', argument: 'value', require: 'require', ensure: 'ensure', assert: 'assert', 'invariant-init': 'invariant', 'invariant-keep': 'invariant', 'decreases-bound': 'decreases', 'decreases-step': 'decreases' } as Record<string, string>)[k] || k; +} + +export function propertiesPseudo(parsed: Parsed<Study>, name?: string): PropertyReport[] { + const s = parsed.program; + if (!s || !parsed.ok) return []; + const subs = [...s.program.subs.values()] as any[]; + const targets = name ? subs.filter((x) => x.name.toUpperCase() === name.toUpperCase()) : subs; + return targets.map((sub) => { + const pr = checkProperties(s.program, sub.name, { count: 300 }); + return { + name: sub.name, checked: pr.checked, problem: pr.problem, + failures: pr.failures.map((f: any) => ({ call: f.call, message: f.message, line: f.line, kind: f.kind })), + }; + }); +} + +const KW = new Set<string>(KEYWORDS as Iterable<string>); +const BI = new Set<string>(Object.keys(BUILTINS || {})); + +export function tokenizePseudo(line: string): Array<{ text: string; cls: string }> { + const out: Array<{ text: string; cls: string }> = []; + const re = /(\/\/.*$)|("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)?')|(\b\d+(?:\.\d+)?(?:e-?\d+)?\b)|(\b[A-Za-z_][A-Za-z0-9_]*\b)|(<-|==|!=|<=|>=|[-+*\/^&<>=()[\]{}:,.])|(\s+)|(.)/g; + let m: RegExpExecArray | null; + while ((m = re.exec(line))) { + const [text, c, s, n, w, op] = m; + let cls = ''; + if (c) cls = 'tok-comment'; + else if (s) cls = 'tok-string'; + else if (n) cls = 'tok-number'; + else if (w) cls = KW.has(w) ? 'tok-keyword' : BI.has(w) ? 'tok-builtin' : /^[A-Z][A-Z0-9_]*$/.test(w) && KW.has(w.toUpperCase()) ? 'tok-keyword' : 'tok-name'; + else if (op) cls = 'tok-op'; + out.push({ text, cls }); + } + return out; +} + +export const pseudoService: LanguageService<Study, PseudoState, PseudoStep> = { + id: 'pseudo', + name: 'pseudocode', + parse: studyOf, + run: runPseudo, + prove: provePseudo, + checkProperties: propertiesPseudo, + defaultCall: (p) => defaultCall(p.program), + tokenize: tokenizePseudo, +}; diff --git a/src/services/speco.ts b/src/services/speco.ts new file mode 100644 index 0000000..0819b33 --- /dev/null +++ b/src/services/speco.ts @@ -0,0 +1,430 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * speco: the specification language, as a standalone document. + * + * Today the course's contracts (REQUIRE, ENSURE, INVARIANT, DECREASES, + * ASSERT) can only live inside pseudocode. This service lets a student write + * them in their own pane, in math notation, and weaves them into the + * pseudocode before handing it to the verifier. It is the seam the review + * recommended: the spec is written before the algorithm, rendered as + * mathematics, and proved against whatever pseudocode is in the next pane. + * + * SPEC COUNT_BASE(B, T) + * ENSURE RESULT = |{ k ∈ [0, |T|) : T[k] = B }| + * LOOP FOR I + * INVARIANT Total = |{ k ∈ [0, I) : T[k] = B }| + * + * Math symbols are accepted and lowered to the notation's ASCII spelling: + * ∀ ∃ Σ ∈ ∧ ∨ ¬ ≤ ≥ ≠ = |…| → FORALL EXISTS SUM IN AND OR NOT <= >= != == LENGTH(…) + * and the reverse direction renders a pseudocode program's contracts as a + * speco document, so existing examples show up in the pane. + */ +import type { Diagnostic, LanguageService, Lowered, Parsed, ProofReport } from '../common/types.js'; +import { studyOf, provePseudo } from './pseudo.js'; +import { SPECO_SYMBOLS } from '../common/reference.js'; + +export interface SpecLoop { selector: string; /** 1-based line of the LOOP header in the speco source */ line?: number; lines: Array<{ kind: 'INVARIANT' | 'DECREASES'; text: string; line: number }> } +export interface SpecUnit { + name: string; + params: string[]; + line: number; + requires: Array<{ text: string; line: number }>; + ensures: Array<{ text: string; line: number }>; + decreases: { text: string; line: number } | null; + loops: SpecLoop[]; +} +export interface SpecDoc { units: SpecUnit[]; diagnostics: Diagnostic[] } + +/* ------------------------------------------------------------------ */ +/* Math notation <-> the notation's ASCII */ +/* ------------------------------------------------------------------ */ + +/** `|X|` → `LENGTH(X)`, with nesting. */ +function lowerBars(s: string): string { + // Treat | as an opening bar when the previous significant char is an operator or start, else closing. + // The last character that is not a space, and the word it ends, are kept as the text grows: trimming and + // searching the whole text at every bar made a long claim quadratic. + let out = ''; + let open = 0; + let prev = ''; // the last character of out that is not a space + let word = ''; // the letters, digits and _ that end out before any trailing space + let inWord = false; // the last character of out is one of those + const add = (t: string) => { + out += t; + for (const c of t) { + if (/\s/.test(c)) { inWord = false; continue; } + prev = c; + if (/\w/.test(c)) { word = inWord ? word + c : c; inWord = true; } else { word = ''; inWord = false; } + } + }; + for (let i = 0; i < s.length; i++) { + const ch = s[i]; + if (ch !== '|') { add(ch); continue; } + const afterOperator = prev === '' || /[(\[{,:=<>+\-*\/^&]/.test(prev) || /^(AND|OR|NOT|IN|FORALL|EXISTS|SUM)$/i.test(word); + if (afterOperator || open === 0) { add('LENGTH('); open++; } + else { add(')'); open--; } + } + while (open-- > 0) out += ')'; + return out; +} + +export function toAscii(expr: string): string { + let s = expr; + for (const sym of SPECO_SYMBOLS) { + if (sym.math.length !== 1 || sym.math === '=') continue; // single symbols; = and |X| and words are handled below + s = s.split(sym.math).join(` ${sym.ascii} `); + } + s = s.replace(/∑/g, ' SUM ').replace(/×/g, ' * ').replace(/÷/g, ' / ').replace(/→/g, ' -> '); + s = lowerBars(s); + s = s.replace(/\bresult\b/g, 'RESULT').replace(/([A-Za-z_]\w*)₀/g, 'OLD($1)'); + // a single = that is not part of ==, <=, >=, != becomes == + s = s.replace(/(^|[^=<>!])=(?!=)/g, '$1=='); + // a symbol written just inside a bracket, (∀ K ...), is padded like any other; the padding is not part of the claim + return s.replace(/\s+/g, ' ').replace(/\(\s+/g, '(').replace(/\s+\)/g, ')').trim(); +} + +/** How many LENGTH(...) one claim shows as |...| (toMath): far past any claim written by hand. */ +const MAX_BARS = 200; + +/** The notation's ASCII rendered as mathematics, for display. */ +export function toMath(expr: string): string { + let s = ' ' + expr + ' '; + // LENGTH(X) → |X|, matching the closing paren while skipping half-open ranges such as [0, N) + const closeOf = (text: string, open: number): number => { + const stack: string[] = []; + for (let j = open; j < text.length; j++) { + const ch = text[j]; + if (ch === '(' || ch === '[' || ch === '{') stack.push(ch); + else if (ch === ')' || ch === ']' || ch === '}') { + stack.pop(); + if (!stack.length) return j; + } + } + return -1; + }; + // at most MAX_BARS of them: each is found and matched from the start (past that the rest stay LENGTH(...)) + for (let n = 0; n < MAX_BARS; n++) { + const i = s.search(/\bLENGTH\s*\(/); + if (i < 0) break; + const open = s.indexOf('(', i); + const j = closeOf(s, open); + if (j < 0) break; + s = s.slice(0, i) + '|' + s.slice(open + 1, j) + '|' + s.slice(j + 1); + } + for (const sym of SPECO_SYMBOLS) { + if (sym.math.length !== 1) continue; + s = sym.ascii.match(/^[A-Z]+$/) ? s.replace(new RegExp(`\\b${sym.ascii}\\b`, 'g'), sym.math) : s.split(sym.ascii).join(sym.math); + } + return s.replace(/\bRESULT\b/g, 'result').replace(/\bOLD\(([^)]*)\)/g, '$1₀').replace(/\s+/g, ' ').trim(); +} + +/* ------------------------------------------------------------------ */ +/* Parsing a speco document */ +/* ------------------------------------------------------------------ */ + +export function parseSpec(source: string): SpecDoc { + const units: SpecUnit[] = []; + const diagnostics: Diagnostic[] = []; + const lines = source.replace(/\r\n?/g, '\n').split('\n'); + let unit: SpecUnit | null = null; + let loop: SpecLoop | null = null; + const say = (line: number, message: string, id = 'spec') => diagnostics.push({ lang: 'speco', severity: 'error', id, line, col: 1, message }); + lines.forEach((raw, i) => { + const line = i + 1; + const text = raw.replace(/\/\/.*$/, '').trim(); + if (!text) return; + let m: RegExpExecArray | null; + if ((m = /^SPEC\s+([A-Za-z_]\w*)\s*(?:\(([^)]*)\))?\s*$/i.exec(text))) { + unit = { name: m[1].toUpperCase(), params: (m[2] || '').split(',').map((p) => p.trim().split(/\s*:\s*/)[0]).filter(Boolean), line, requires: [], ensures: [], decreases: null, loops: [] }; + units.push(unit); loop = null; return; + } + if (!unit) { say(line, 'A speco document starts with SPEC <name>(<params>), naming the routine these claims are about.', 'no-spec'); return; } + if ((m = /^LOOP\s+(.+)$/i.exec(text))) { loop = { selector: m[1].trim(), line, lines: [] }; unit.loops.push(loop); return; } + if ((m = /^(REQUIRE|ENSURE|INVARIANT|DECREASES|ASSERT)\s+(.+)$/i.exec(text))) { + const kind = m[1].toUpperCase(); + const body = toAscii(m[2]); + if (kind === 'REQUIRE') { if (loop) say(line, 'REQUIRE belongs to the routine, above any LOOP.'); else unit.requires.push({ text: body, line }); } + else if (kind === 'ENSURE') { if (loop) say(line, 'ENSURE belongs to the routine, above any LOOP.'); else unit.ensures.push({ text: body, line }); } + else if (kind === 'INVARIANT') { if (!loop) say(line, 'INVARIANT needs a LOOP line above it saying which loop it describes: LOOP FOR I, LOOP WHILE, or LOOP 1.'); else loop.lines.push({ kind: 'INVARIANT', text: body, line }); } + else if (kind === 'DECREASES') { if (loop) loop.lines.push({ kind: 'DECREASES', text: body, line }); else unit.decreases = { text: body, line }; } + else say(line, 'ASSERT lives in the pseudocode, at the line it is about; speco holds REQUIRE, ENSURE, INVARIANT and DECREASES.'); + return; + } + say(line, `Not a speco line. Each line is SPEC, LOOP, REQUIRE, ENSURE, INVARIANT or DECREASES followed by a claim.`, 'unknown-line'); + }); + return { units, diagnostics }; +} + +/* ------------------------------------------------------------------ */ +/* Weaving a spec into pseudocode, and extracting one back out */ +/* ------------------------------------------------------------------ */ + +export interface Woven { source: string; map: Array<{ from: number; to: number }>; diagnostics: Diagnostic[] } + +export interface WeaveOptions { + /** + * Replace mode: for each SPEC unit, the routine's existing REQUIRE / ENSURE / DECREASES and the INVARIANT / + * DECREASES of every loop the unit selects are removed first, so the pseudocode says what the speco says and + * nothing else about those routines and loops. Without it (the default, used by the proof view) the claims are + * only inserted, and what is already there is kept and never duplicated. + */ + replace?: boolean; +} + +/** The line index of the loop a `LOOP <selector>` line names, among the loops of a routine, or -1. */ +function findLoop(lines: string[], loopLines: number[], selector: string): number { + let m: RegExpExecArray | null; + if ((m = /^(\d+)$/.exec(selector))) return loopLines[Number(m[1]) - 1] ?? -1; + if ((m = /^FOR\s+EACH\s+(\w+)/i.exec(selector))) return loopLines.find((i) => new RegExp(`^\\s*FOR\\s+EACH\\s+${m![1]}\\b`, 'i').test(lines[i])) ?? -1; + if ((m = /^FOR\s+(\w+)/i.exec(selector))) return loopLines.find((i) => new RegExp(`^\\s*FOR\\s+${m![1]}\\b`, 'i').test(lines[i])) ?? -1; + if (/^WHILE/i.test(selector)) return loopLines.find((i) => /^\s*WHILE\b/i.test(lines[i])) ?? -1; + if (/^REPEAT/i.test(selector)) return loopLines.find((i) => /^\s*REPEAT\b/i.test(lines[i])) ?? -1; + return -1; +} + +/** Where a routine starts and ends, and where its loops are, in a program's lines. */ +function locate(lines: string[], name: string): { head: number; end: number; loops: number[] } | null { + const head = lines.findIndex((l) => new RegExp(`^\\s*(FUNCTION|PROCEDURE)\\s+${name}\\b`, 'i').test(l)); + if (head < 0) return null; + let end = lines.length; + for (let i = head + 1; i < lines.length; i++) if (/^\s*(ENDFUNCTION|ENDPROCEDURE)\b/i.test(lines[i])) { end = i; break; } + const loops: number[] = []; + for (let i = head + 1; i < end; i++) if (/^\s*(FOR|WHILE|REPEAT)\b/i.test(lines[i])) loops.push(i); + return { head, end, loops }; +} + +/** Insert the spec's claims into the pseudocode under the routines and loops they name (or, in replace mode, put them in place of what is there). */ +export function weave(pseudo: string, spec: SpecDoc, options: WeaveOptions = {}): Woven { + let lines = pseudo.replace(/\r\n?/g, '\n').split('\n'); + const diagnostics: Diagnostic[] = []; + const inserts: Array<{ after: number; text: string; from: number }> = []; + if (options.replace) { + // Remove first, against the original lines, so the indices the insertions use are those of the program that is left. + const drop = new Set<number>(); + for (const u of spec.units) { + const at = locate(lines, u.name); + if (!at) continue; + for (let i = at.head + 1; i < at.end && /^\s*(REQUIRE|ENSURE|DECREASES)\b/i.test(lines[i]); i++) drop.add(i); + for (const lp of u.loops) { + const target = findLoop(lines, at.loops, lp.selector); + if (target < 0) continue; + for (let i = target + 1; i < at.end && /^\s*(INVARIANT|DECREASES)\b/i.test(lines[i]); i++) drop.add(i); + } + } + lines = lines.filter((_, i) => !drop.has(i)); + } + for (const u of spec.units) { + const at = locate(lines, u.name); + if (!at) { diagnostics.push({ lang: 'speco', severity: 'error', id: 'no-routine', line: u.line, col: 1, message: `The pseudocode has no FUNCTION or PROCEDURE called ${u.name}.` }); continue; } + const { head, end } = at; + const indent = (lines[head].match(/^\s*/)?.[0].length ?? 0) + 4; + const norm = (t: string) => t.replace(/\s+/g, '').toUpperCase(); + const has = (from: number, to: number, kind: string, text: string) => { + for (let i = from; i < to; i++) { const m = new RegExp(`^\\s*${kind}\\s+(.+)$`, 'i').exec(lines[i]); if (m && norm(m[1]) === norm(text)) return true; } + return false; + }; + // existing contracts directly under the header keep their order: REQUIRE, then ENSURE + let afterReq = head; let afterEns = head; + for (let i = head + 1; i < end; i++) { + if (/^\s*REQUIRE\b/i.test(lines[i])) { afterReq = i; afterEns = i; } + else if (/^\s*ENSURE\b/i.test(lines[i])) afterEns = i; + else if (/^\s*DECREASES\b/i.test(lines[i])) afterEns = i; + else break; + } + for (const r of u.requires) if (!has(head, end, 'REQUIRE', r.text)) inserts.push({ after: afterReq, text: ' '.repeat(indent) + 'REQUIRE ' + r.text, from: r.line }); + for (const e of u.ensures) if (!has(head, end, 'ENSURE', e.text)) inserts.push({ after: afterEns, text: ' '.repeat(indent) + 'ENSURE ' + e.text, from: e.line }); + if (u.decreases && !has(head, afterEns + 1, 'DECREASES', u.decreases.text)) inserts.push({ after: afterEns, text: ' '.repeat(indent) + 'DECREASES ' + u.decreases.text, from: u.decreases.line }); + // loops + for (const lp of u.loops) { + const target = findLoop(lines, at.loops, lp.selector); + if (target < 0) { diagnostics.push({ lang: 'speco', severity: 'error', id: 'no-loop', line: lp.line ?? lp.lines[0]?.line ?? u.line, col: 1, message: `${u.name} has no loop matching "LOOP ${lp.selector}". Write LOOP FOR I, LOOP WHILE, or LOOP <n> for the n-th loop.` }); continue; } + const li = (lines[target].match(/^\s*/)?.[0].length ?? 0) + 4; + // existing INVARIANT lines, then an existing DECREASES, directly under the loop header + let afterInv = target; let hasDec = -1; + for (let i = target + 1; i < end; i++) { + if (/^\s*INVARIANT\b/i.test(lines[i])) afterInv = i; + else if (/^\s*DECREASES\b/i.test(lines[i])) { hasDec = i; break; } + else break; + } + const invs = lp.lines.filter((l) => l.kind === 'INVARIANT' && !has(target, hasDec >= 0 ? hasDec : afterInv + 1, 'INVARIANT', l.text)); + const decs = lp.lines.filter((l) => l.kind === 'DECREASES'); + for (const l of invs) inserts.push({ after: afterInv, text: ' '.repeat(li) + 'INVARIANT ' + l.text, from: l.line }); + for (const l of decs) { + if (hasDec >= 0) { if (norm(lines[hasDec].replace(/^\s*DECREASES\s+/i, '')) !== norm(l.text)) diagnostics.push({ lang: 'speco', severity: 'warning', id: 'decreases-twice', line: l.line, col: 1, message: `This loop already has DECREASES in the pseudocode; the speco line is ignored.` }); continue; } + inserts.push({ after: afterInv, text: ' '.repeat(li) + 'DECREASES ' + l.text, from: l.line }); + } + } + } + // apply bottom-up so indices stay valid; keep insertion order among equal anchors + inserts.sort((a, b) => b.after - a.after); + const out = lines.slice(); + const origin: Array<number | null> = lines.map(() => null); + let i = 0; + while (i < inserts.length) { + let j = i; + while (j < inserts.length && inserts[j].after === inserts[i].after) j++; + const group = inserts.slice(i, j); // same anchor, in document order + out.splice(inserts[i].after + 1, 0, ...group.map((g) => g.text)); + origin.splice(inserts[i].after + 1, 0, ...group.map((g) => g.from)); + i = j; + } + const map = origin.map((from, idx) => (from ? { from, to: idx + 1 } : null)).filter((x): x is { from: number; to: number } => !!x); + return { source: out.join('\n'), map, diagnostics }; +} + +/** The contracts already inside a pseudocode program, as a speco document. */ +export function extractSpec(pseudo: string): string { + const lines = pseudo.replace(/\r\n?/g, '\n').split('\n'); + const out: string[] = []; + let inUnit = false; + let loopIndex = 0; + let pendingLoop: string | null = null; + for (const raw of lines) { + const t = raw.trim(); + let m: RegExpExecArray | null; + if ((m = /^(FUNCTION|PROCEDURE)\s+([A-Za-z_]\w*)\s*(?:\(([^)]*)\))?/i.exec(t))) { + if (inUnit && out[out.length - 1] !== '') out.push(''); + const params = (m[3] || '').split(',').map((p) => p.trim().replace(/^BY(REF|VAL)\s+/i, '').split(/\s*:\s*/)[0]).filter(Boolean); + out.push(`SPEC ${m[2].toUpperCase()}(${params.join(', ')})`); + inUnit = true; loopIndex = 0; pendingLoop = null; continue; + } + if (!inUnit) continue; + if (/^(ENDFUNCTION|ENDPROCEDURE)\b/i.test(t)) { inUnit = false; continue; } + if ((m = /^FOR\s+EACH\s+(\w+)/i.exec(t))) { loopIndex++; pendingLoop = `FOR EACH ${m[1]}`; continue; } + if ((m = /^FOR\s+(\w+)/i.exec(t))) { loopIndex++; pendingLoop = `FOR ${m[1]}`; continue; } + if (/^WHILE\b/i.test(t)) { loopIndex++; pendingLoop = loopIndex === 1 ? 'WHILE' : String(loopIndex); continue; } + if (/^REPEAT\b/i.test(t)) { loopIndex++; pendingLoop = loopIndex === 1 ? 'REPEAT' : String(loopIndex); continue; } + if ((m = /^(REQUIRE|ENSURE)\s+(.+)$/i.exec(t))) { out.push(` ${m[1].toUpperCase()} ${toMath(m[2])}`); continue; } + if ((m = /^(INVARIANT|DECREASES)\s+(.+)$/i.exec(t))) { + if (pendingLoop) { out.push(` LOOP ${pendingLoop}`); pendingLoop = null; out.push(` ${m[1].toUpperCase()} ${toMath(m[2])}`); } + else if (out.length && /^\s{8}(INVARIANT|DECREASES)/.test(out[out.length - 1])) out.push(` ${m[1].toUpperCase()} ${toMath(m[2])}`); + else out.push(` ${m[1].toUpperCase()} ${toMath(m[2])}`); + continue; + } + if (t && !/^\/\//.test(t)) pendingLoop = pendingLoop && /^(INVARIANT|DECREASES)/i.test(t) ? pendingLoop : null; + } + const text = out.join('\n').trim(); + // drop SPEC headers that gathered no claims + return text.split(/\n(?=SPEC )/).filter((block) => block.split('\n').length > 1).join('\n\n').trim(); +} + +/* ------------------------------------------------------------------ */ +/* Pseudo as the authority: the speco that follows it, orphans kept */ +/* ------------------------------------------------------------------ */ + +export interface Reconciled { source: string; diagnostics: Diagnostic[] } + +const KEPT = '// kept: '; + +/** + * The speco that follows a pseudocode program: `extractSpec(pseudo)`, plus every claim the previous speco made + * about a routine or loop the pseudocode no longer has. Those are kept (never dropped) under a `// kept:` + * comment that says why, and reported on their line in the speco returned. They stay until the routine or loop + * comes back (the claims then belong to it again, even when the pseudocode has not written them yet) or the + * student deletes them. A claim about something the pseudocode does have is the pseudocode's: changing or + * deleting a contract in the pseudocode changes the speco. + * + * Fixed point: `reconcileSpec(pseudo, reconcileSpec(pseudo, previous).source).source` is the same text. + */ +export function reconcileSpec(pseudo: string, previous: string): Reconciled { + const lines = pseudo.replace(/\r\n?/g, '\n').split('\n'); + const prev = parseSpec(previous); + const prevLines = previous.replace(/\r\n?/g, '\n').split('\n'); + const wasKept = (line: number | undefined) => line !== undefined && /^\s*\/\/ kept:/.test(prevLines[line - 2] ?? ''); + interface Row { t: string; note?: string } + const row = (t: string, note?: string): Row => ({ t, note }); + const fresh = extractSpec(pseudo); + const blocks: Array<{ name: string; rows: Row[] }> = fresh + ? fresh.split(/\n(?=SPEC )/).map((b) => ({ name: /^SPEC\s+(\w+)/.exec(b)![1].toUpperCase(), rows: b.split('\n').map((t) => row(t)) })) + : []; + const head = (u: SpecUnit): Row[] => { + const o = [row(`SPEC ${u.name}(${u.params.join(', ')})`)]; + for (const r of u.requires) o.push(row(` REQUIRE ${toMath(r.text)}`)); + for (const e of u.ensures) o.push(row(` ENSURE ${toMath(e.text)}`)); + if (u.decreases) o.push(row(` DECREASES ${toMath(u.decreases.text)}`)); + return o; + }; + const loopRows = (lp: SpecLoop): Row[] => [row(` LOOP ${lp.selector}`), ...lp.lines.map((l) => row(` ${l.kind} ${toMath(l.text)}`))]; + const orphans: Row[][] = []; + for (const u of prev.units) { + const at = locate(lines, u.name); + if (!at) { + const why = `The pseudocode has no FUNCTION or PROCEDURE called ${u.name}, so these claims have nothing to be about. They are kept here until it comes back.`; + const [first, ...rest] = [...head(u), ...u.loops.flatMap(loopRows)]; + orphans.push([row(`${KEPT}the pseudocode has no FUNCTION or PROCEDURE called ${u.name}`), { t: first.t, note: why }, ...rest]); + continue; + } + let block = blocks.find((b) => b.name === u.name); + if (!block) { + // no contracts in the pseudocode for it: they were deleted there, unless this spec was only waiting for the routine to return + // (or only a loop of it was: then the unit comes back with just that loop) + if (wasKept(u.line)) block = { name: u.name, rows: head(u) }; + else if (u.loops.some((lp) => wasKept(lp.line))) block = { name: u.name, rows: [row(`SPEC ${u.name}(${u.params.join(', ')})`)] }; + else continue; + blocks.push(block); + } + // the loops the pseudocode has, as the extracted block names them + const claimed = new Set<number>(); + for (const r of block.rows) { const m = /^\s{4}LOOP (.+)$/.exec(r.t); if (m) claimed.add(findLoop(lines, at.loops, m[1].trim())); } + for (const lp of u.loops) { + const target = findLoop(lines, at.loops, lp.selector); + if (target < 0) { + const why = `${u.name} has no loop matching "LOOP ${lp.selector}", so these claims have nothing to be about. They are kept here until the loop comes back.`; + const [first, ...rest] = loopRows(lp); + block.rows.push(row(` ${KEPT}${u.name} has no loop matching "LOOP ${lp.selector}"`), { t: first.t, note: why }, ...rest); + } else if (!claimed.has(target) && wasKept(lp.line)) { + block.rows.push(...loopRows(lp)); + claimed.add(target); + } + } + } + const out: Row[] = []; + [...blocks.map((b) => b.rows), ...orphans].forEach((b, i) => { if (i) out.push(row('')); out.push(...b); }); + const diagnostics: Diagnostic[] = []; + out.forEach((r, i) => { if (r.note) diagnostics.push({ lang: 'speco', severity: 'warning', id: 'spec-orphan', line: i + 1, col: 1, message: r.note }); }); + return { source: out.map((r) => r.t).join('\n'), diagnostics }; +} + +/* ------------------------------------------------------------------ */ +/* The service */ +/* ------------------------------------------------------------------ */ + +export interface SpecoProgram { spec: SpecDoc; source: string } + +export function parseSpeco(source: string): Parsed<SpecoProgram> { + const spec = parseSpec(source); + return { program: { spec, source }, diagnostics: spec.diagnostics, ok: spec.diagnostics.every((d) => d.severity !== 'error') }; +} + +/** Prove a pseudocode program against this spec. Obligation lines are reported in speco's numbering where a claim came from speco. */ +export function proveAgainst(parsed: Parsed<SpecoProgram>, pseudo: string): { report: ProofReport; woven: Woven; study: Parsed<any> } { + const woven = weave(pseudo, parsed.program!.spec); + const st = studyOf(woven.source); + const report = provePseudo(st); + const back = new Map(woven.map.map((m) => [m.to, m.from])); + for (const u of report.units) for (const o of u.obligations) { + const from = back.get(o.line); + (o as any).pseudoLine = o.line; + (o as any).specoLine = from ?? null; + } + return { report, woven, study: st }; +} + +export function tokenizeSpeco(line: string): Array<{ text: string; cls: string }> { + const out: Array<{ text: string; cls: string }> = []; + const re = /(\/\/.*$)|("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)?')|(\b\d+(?:\.\d+)?\b)|(\b(?:SPEC|LOOP|REQUIRE|ENSURE|INVARIANT|DECREASES|FORALL|EXISTS|SUM|IN|AND|OR|NOT|RESULT|OLD|LENGTH|SORTED|FOR|EACH|WHILE|REPEAT)\b)|([∀∃Σ∑∈∧∨¬≤≥≠|])|(\b[A-Za-z_]\w*\b)|(\s+)|(.)/g; + let m: RegExpExecArray | null; + while ((m = re.exec(line))) { + const [text, c, s, n, kw, sym, w] = m; + out.push({ text, cls: c ? 'tok-comment' : s ? 'tok-string' : n ? 'tok-number' : kw ? 'tok-keyword' : sym ? 'tok-op' : w ? 'tok-name' : '' }); + } + return out; +} + +export const specoService: LanguageService<SpecoProgram> = { + id: 'speco', + name: 'speco', + parse: parseSpeco, + tokenize: tokenizeSpeco, +}; diff --git a/src/sync/compare.ts b/src/sync/compare.ts new file mode 100644 index 0000000..1faabea --- /dev/null +++ b/src/sync/compare.ts @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Comparing what two programs print, by value rather than by text, for the + * tests that hold a translation to the program it came from (pseudo run vs. + * the generated Mojo run, pseudo vs. the pseudo that lingo or matho lowers + * to). DOM-free, and only the tests import it. + * + * The notation and Mojo print the same value differently: TRUE and True, a + * string with or without quotes, 3 and 3.0, a map's keys smallest first or in + * insertion order, `infinity` and `not a number` where Mojo prints `inf` and + * `nan`. `canonical` erases exactly those differences and no others. + * + * Two outputs can also differ only in the last digits of a number: the + * compiled program's std.math is not the interpreter's (exp, log2, cosh, a + * fractional **), and a program built with Mojo's default `--fp-mode + * contract=fast` fuses `a * b + c` into one rounding when the operands arrive + * at run time. Programming+ builds with contraction off (library/mojo/pin.json), + * so the second does not show in its own builds. `outputDistance` measures the + * difference, so a caller can tell rounding from a wrong answer and say which + * it found. + */ +import { PArray, PRecord } from '../engines/pseudo/lang.js'; + +/** A value from the pseudo interpreter, as Mojo's print shows it. */ +export function formatAsMojo(v: any): string { + if (v === undefined || v === null) return 'None'; + if (typeof v === 'boolean') return v ? 'True' : 'False'; + if (typeof v === 'number') return Number.isInteger(v) ? String(v) : String(v); + if (typeof v === 'string') return v; + if (Array.isArray(v)) return `[${v.map(formatAsMojo).join(', ')}]`; + if (v instanceof PArray) { + if (v.dims.length === 2) { + const rows: string[] = []; + for (let r = 0; r < v.dims[0]; r++) rows.push(`[${v.data.slice(r * v.dims[1], (r + 1) * v.dims[1]).map(formatAsMojo).join(', ')}]`); + return `[${rows.join(', ')}]`; + } + return `[${v.data.map(formatAsMojo).join(', ')}]`; + } + if (v instanceof Set) return `{${[...v].map(formatAsMojo).join(', ')}}`; + if (v instanceof Map) return `{${[...v].map(([k, x]) => `${formatAsMojo(k)}: ${formatAsMojo(x)}`).join(', ')}}`; + if (v instanceof PRecord) return `${v.typeName}(${[...v.fields].map(([, x]) => formatAsMojo(x)).join(', ')})`; + return String(v); +} + +/** Text with the print differences between the notation and Mojo erased. */ +export function canonical(text: string): string { + return text + .replace(/\r/g, '') + .replace(/["']/g, '') + .replace(/\bTRUE\b/g, 'True') + .replace(/\bFALSE\b/g, 'False') + .replace(/\binfinity\b/g, 'inf') + .replace(/\bnot a number\b/g, 'nan') + .replace(/\{([^{}[\]]*)\}/g, (_m, inner: string) => `{${inner.split(', ').sort().join(', ')}}`) + .replace(/-?\d+(?:\.\d+)?(?:e[+-]?\d+)?/gi, (m) => String(Number(m))) + .split('\n').map((l) => l.trimEnd()).join('\n') + .trim(); +} + +/** Do two printed outputs say the same thing? */ +export function sameOutput(a: string, b: string): boolean { return canonical(a) === canonical(b); } + +/** + * How far apart two printed outputs are: 0 when they say the same thing, the largest relative difference between + * two numbers in the same place when everything else is the same, and Infinity when they differ in any other way + * (other text, another count of lines or numbers, a number against a word). + */ +export function outputDistance(a: string, b: string): number { + const x = canonical(a); const y = canonical(b); + if (x === y) return 0; + const NUM = /-?\d+(?:\.\d+)?(?:e[+-]?\d+)?/gi; + if (x.replace(NUM, '#') !== y.replace(NUM, '#')) return Infinity; + const xs = x.match(NUM) ?? []; const ys = y.match(NUM) ?? []; + let worst = 0; + for (let i = 0; i < xs.length; i++) { + const p = Number(xs[i]); const q = Number(ys[i]); + if (p === q) continue; + const scale = Math.max(Math.abs(p), Math.abs(q)); + worst = Math.max(worst, scale === 0 || !Number.isFinite(scale) ? Infinity : Math.abs(p - q) / scale); + } + return worst; +} diff --git a/src/sync/header.ts b/src/sync/header.ts new file mode 100644 index 0000000..5200e47 --- /dev/null +++ b/src/sync/header.ts @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The first line of every generated Mojo source (restructure plan, section 2.4). It travels with the + * code, so copying the text out keeps the warning. It lives in its own module so the translator + * (which never applies best-practice rewrites) has no reason to mention them. + */ +export const GENERATED_HEADER = '# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices.'; diff --git a/src/sync/hub.ts b/src/sync/hub.ts new file mode 100644 index 0000000..7a56434 --- /dev/null +++ b/src/sync/hub.ts @@ -0,0 +1,387 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The sync hub (restructure plan, section 2.3). Pseudo is the hub: lingo and matho lower to it, lingo and + * matho are generated from it, contracts live in it and speco is the lens on them, and Mojo is produced from + * it and never the other way. The hub is pure: no DOM, no localStorage, a function of the texts it is given. + * + * The rules, each pinned by tests/sync-hub.test.mjs: + * 1. The pane you edit is the authority. An edit produces a new pseudo (directly, or by lowering) and the + * other linked panes are regenerated from it. The pane that was edited is never rewritten. + * 2. Contracts ride along. Lingo and matho cannot say them, so the speco is woven into the pseudo that an + * edit in English or equations produces; when the authority is speco, its contracts replace the pseudo's + * and lingo and matho (which ignore contracts) follow. + * 3. An authority with errors changes nothing: every other pane keeps its text and is marked stale. + * 4. A language that cannot say the program keeps its last text, marked stale with the reason. + * 5. Mojo comes only from pseudo, as a literal translation, and is computed here, not stored. + * 6. Opening a problem rewrites nothing: `fill` generates only the linked panes that are empty. + * 7. Line maps compose through the hub: `relate` answers "which lines of that pane match this line here". + */ +import type { Diagnostic, LangId } from '../common/types.js'; +import { readLingo, lowerLingo } from '../services/lingo.js'; +import { parseMatho, lowerMatho } from '../services/matho.js'; +import { studyOf, defaultCall } from '../services/pseudo.js'; +import { parseSpec, weave, extractSpec, reconcileSpec } from '../services/speco.js'; +import { pseudoToLingo } from './pseudo-to-lingo.js'; +import { pseudoToMatho } from './pseudo-to-matho.js'; +import { pseudoToMojo } from './pseudo-to-mojo.js'; +import type { LineMap, NameMap } from './types.js'; + +/** The four languages that are kept in step with each other. Mojo and x86-64 are not linked. */ +export const LINKED = ['lingo', 'matho', 'pseudo', 'speco'] as const; +export type Linked = (typeof LINKED)[number]; +export const isLinked = (l: string): l is Linked => (LINKED as readonly string[]).includes(l); + +/** What `PlayState.sync` holds: which pane the others follow, and why any pane is not in step. */ +export interface SyncInfo { + authority: Linked; + stale?: Partial<Record<Linked, string>>; + names?: NameMap; +} + +export type Sources = Partial<Record<LangId, string>>; + +/** Line maps, kept per pane. `up[l][i]` is the line of pane `l` that pseudo line `i+1` was lowered from; `down[l][i]` is the pseudo line that line `i+1` of pane `l` was generated from. */ +export interface HubMaps { + up: Partial<Record<Linked, LineMap>>; + down: Partial<Record<Linked | 'mojo', LineMap>>; +} + +export interface MojoDerived { source: string; map: LineMap; diagnostics: Diagnostic[] } + +export interface HubResult { + sources: Sources; + authority: Linked; + stale: Partial<Record<Linked, string>>; + names: NameMap; + maps: HubMaps; + /** The literal Mojo for the pseudo now in play. */ + mojo: MojoDerived; + /** Panes that changed text because of this edit (never the one that was edited). */ + changed: Linked[]; +} + +const nonBlank = (s: string | undefined): boolean => !!(s ?? '').trim(); +const firstError = (ds: Diagnostic[]): Diagnostic | undefined => ds.find((d) => d.severity === 'error'); +const reasonOf = (lang: Linked, d: Diagnostic | undefined): string => { + if (!d) return `${lang} does not read yet`; + const where = lang === 'speco' ? `${lang} line ${d.line} has a problem` : `${lang} line ${d.line} does not read yet`; + return `${where} (${d.message.replace(/\s+/g, ' ').trim()})`; +}; +const unsupportedReason = (lang: Linked, ds: Diagnostic[]): string => (ds[0]?.message ?? `${lang} has no form for this program`).replace(/\s+/g, ' ').trim(); + +/** + * The last line of defense: a reader or a translator that throws (a fault in Programming+, past every reader's own + * limits) leaves the panes as they were, marked stale with this reason, and never throws out of the hub + * (tests/reader-limits.test.mjs). The id is the one pplus-web's Session gives the same failure. + */ +export const FAULT_ID = 'engine-stop'; +const faultReason = (e: unknown): string => `Programming+ could not follow this change (${String((e as Error)?.message ?? e).slice(0, 200)}). That is a fault in Programming+, not in the program; the other panes keep their text.`; + +/** A `{from,to}` list as a LineMap over `n` output lines. */ +function pairsToMap(pairs: Array<{ from: number; to: number }>, n: number): LineMap { + const out: LineMap = Array.from({ length: n }, () => null); + for (const p of pairs) if (p.to >= 1 && p.to <= n && p.from > 0) out[p.to - 1] = p.from; + return out; +} + +/** + * The student's spelling of equation names, so a round trip through pseudo restores `\overline{x}` rather than + * the default `\bar{x}`. Keyed by the pseudo name the reader gave the function. Only a bare name on the left + * side is recorded; a function written f(x, y) is spelled by the generator itself. + */ +export function namesFromMatho(source: string): NameMap { + let parsed: ReturnType<typeof parseMatho>; + try { parsed = parseMatho(source); } catch { return {}; } // a reader's fault: no spellings to keep + const p = parsed.program; + const names: NameMap = {}; + if (!p) return names; + for (const def of p.defs) { + const eq = p.equations.find((e) => e.line === def.line); + if (!eq) continue; + const at = eq.source.indexOf('='); + const lhs = (at > 0 ? eq.source.slice(0, at) : '').trim(); + if (lhs && !lhs.includes('(') && /^\\|^[A-Za-z]/.test(lhs)) names[def.name.toUpperCase()] = lhs; + } + return names; +} + +/** Carry a map made against a program without contracts across a weave that inserted `inserted` lines (1-based, in the woven program). */ +function shiftUp(map: LineMap | undefined, inserted: Set<number>, wovenLines: number): LineMap | undefined { + if (!map) return map; + const out: LineMap = []; + let k = 0; + for (let i = 1; i <= wovenLines; i++) out.push(inserted.has(i) ? null : (map[k++] ?? null)); + return out; +} + +export class SyncHub { + private sources: Sources; + private info: SyncInfo | undefined; + private maps: HubMaps = { up: {}, down: {} }; + private call: string | null; + /** The study of the last pseudo an edit checked, so the call that becomes `main` does not study it again. */ + private studied: { source: string; study: ReturnType<typeof studyOf> } | null = null; + + /** `sources` are the texts as the state holds them; `info` is what the state remembers of earlier edits. */ + constructor(sources: Sources, info?: SyncInfo, opts: { call?: string | null } = {}) { + this.sources = { ...sources }; + this.info = info ? { authority: info.authority, stale: { ...(info.stale ?? {}) }, names: { ...(info.names ?? {}) } } : undefined; + this.call = opts.call ?? null; + } + + /* ---------- opening: rule 6 ---------- */ + + /** + * The panes as they should be shown when a problem opens: what is written stays as written, and a linked pane + * that is empty is generated from the pseudo (which is itself lowered from the English or equations when it is + * missing). Nothing that has text is touched, so an authored exercise is shown exactly as its author wrote it. + */ + fill(): { sources: Sources; stale: Partial<Record<Linked, string>>; maps: HubMaps } { + try { return this.fillPanes(); } catch (e) { + // nothing is generated: every empty linked pane says why + const stale: Partial<Record<Linked, string>> = {}; + for (const l of LINKED) if (!nonBlank(this.sources[l])) stale[l] = faultReason(e); + return { sources: { ...this.sources }, stale, maps: { up: {}, down: {} } }; + } + } + + private fillPanes(): { sources: Sources; stale: Partial<Record<Linked, string>>; maps: HubMaps } { + const out: Sources = { ...this.sources }; + const stale: Partial<Record<Linked, string>> = {}; + const maps: HubMaps = { up: {}, down: {} }; + if (!nonBlank(out.pseudo)) { + const low = this.lowerOne('lingo', out.lingo) ?? this.lowerOne('matho', out.matho); + if (low && 'pseudo' in low) { out.pseudo = low.pseudo; maps.up[low.lang] = low.map; } + } + const P = out.pseudo ?? ''; + if (nonBlank(P)) { + if (!nonBlank(out.lingo)) { + const r = pseudoToLingo(P); + if ('unsupported' in r) stale.lingo = unsupportedReason('lingo', r.unsupported); + else { out.lingo = r.source; maps.down.lingo = r.map; } + } + if (!nonBlank(out.matho)) { + const r = this.generateMatho(P, namesFromMatho(out.matho ?? '')); + if ('unsupported' in r) stale.matho = unsupportedReason('matho', r.unsupported); + else { out.matho = r.source; maps.down.matho = r.map; } + } + if (!nonBlank(out.speco)) { + const s = extractSpec(P); + if (s) out.speco = s; + } + } + return { sources: out, stale, maps }; + } + + private lowerOne(lang: 'lingo' | 'matho', text: string | undefined): { lang: 'lingo' | 'matho'; pseudo: string; map: LineMap } | null { + if (!nonBlank(text)) return null; + if (lang === 'lingo') { + const parsed = readLingo(text!); + const low = parsed.ok ? lowerLingo(parsed, 'pseudo') : null; + return low ? { lang, pseudo: low.source, map: pairsToMap(low.map, low.source.split('\n').length) } : null; + } + const parsed = parseMatho(text!); + const low = parsed.ok ? lowerMatho(parsed, 'pseudo') : null; + return low ? { lang, pseudo: low.source, map: pairsToMap(low.map, low.source.split('\n').length) } : null; + } + + private generateMatho(P: string, names: NameMap) { + const merged = { ...(this.info?.names ?? {}), ...names }; + const r = pseudoToMatho(P, { names: merged }); + // a spelling that does not lower back to the function is not trusted; fall back to the default spelling + if ('unsupported' in r && Object.keys(merged).length) return pseudoToMatho(P); + return r; + } + + /* ---------- an edit: rules 1 to 5 ---------- */ + + edit(lang: Linked, text: string): HubResult { + const before: Sources = { ...this.sources }; + const info = this.info; + try { return this.editPanes(lang, text); } catch (e) { + // rule 3, as if the authority had an error: it keeps its text, and every other pane keeps its own, stale + const stale: Partial<Record<Linked, string>> = { ...(info?.stale ?? {}) }; + delete stale[lang]; + for (const l of LINKED) if (l !== lang) stale[l] = faultReason(e); + const sources: Sources = { ...before, [lang]: text }; + this.sources = { ...sources }; + this.info = { authority: lang, stale: { ...stale }, names: { ...(info?.names ?? {}) } }; + return { + sources, authority: lang, stale, names: { ...(info?.names ?? {}) }, maps: { up: {}, down: {} }, + mojo: { source: '', map: [], diagnostics: [{ lang: 'mojo', severity: 'error', id: FAULT_ID, line: 1, col: 1, message: faultReason(e) }] }, + changed: [], + }; + } + } + + private editPanes(lang: Linked, text: string): HubResult { + const prior: Sources = { ...this.sources }; + const stale: Partial<Record<Linked, string>> = { ...(this.info?.stale ?? {}) }; + delete stale[lang]; + const names: NameMap = { ...(this.info?.names ?? {}) }; + this.sources[lang] = text; + this.info = { authority: lang, stale, names }; + // the maps of the authority are about to be remade; the others' stay until they are regenerated + delete this.maps.down[lang]; + delete this.maps.up[lang]; + + const others = LINKED.filter((l) => l !== lang); + const failWith = (reason: string): HubResult => { + for (const l of others) stale[l] = reason; + return this.result(prior); + }; + + // 1. the pseudo this edit comes to + let P: string; + let upMap: LineMap | undefined; + let specoMap: Array<{ from: number; to: number }> = []; + if (!nonBlank(text)) { + P = ''; + } else if (lang === 'pseudo') { + const st = studyOf(text); + this.studied = { source: text, study: st }; + if (!st.ok) return failWith(reasonOf('pseudo', firstError(st.diagnostics))); + P = text; + } else if (lang === 'lingo') { + const parsed = readLingo(text); + const low = parsed.ok ? lowerLingo(parsed, 'pseudo') : null; + if (!low) return failWith(reasonOf('lingo', firstError(parsed.diagnostics))); + P = low.source; upMap = pairsToMap(low.map, P.split('\n').length); + } else if (lang === 'matho') { + const parsed = parseMatho(text); + const low = parsed.ok ? lowerMatho(parsed, 'pseudo') : null; + if (!low) return failWith(reasonOf('matho', firstError(parsed.diagnostics))); + P = low.source; upMap = pairsToMap(low.map, P.split('\n').length); + const nm = namesFromMatho(text); + for (const k of Object.keys(names)) delete names[k]; + Object.assign(names, nm); + } else { + // speco: its contracts replace the pseudo's, in the pseudo the other panes follow + const spec = parseSpec(text); + const bad = firstError(spec.diagnostics); + if (bad) return failWith(reasonOf('speco', bad)); + const base = this.sources.pseudo ?? ''; + if (nonBlank(base)) { const w = weave(base, spec, { replace: true }); P = w.source; specoMap = w.map; } else P = base; + } + + // 2. contracts ride along: an English or equation edit weaves the speco into the pseudo it produced + if (lang === 'lingo' || lang === 'matho') { + const spec = nonBlank(this.sources.speco) ? parseSpec(this.sources.speco!) : null; + if (spec && !firstError(spec.diagnostics) && spec.units.length && nonBlank(P)) { + const w = weave(P, spec); + if (w.source !== P) { + const inserted = new Set(w.map.map((m) => m.to)); + upMap = shiftUp(upMap, inserted, w.source.split('\n').length); + P = w.source; + } + } + } + + // 3. regenerate the other panes from P + this.sources.pseudo = lang === 'pseudo' ? text : P; + if (lang !== 'pseudo') { delete stale.pseudo; } + if (upMap) this.maps.up[lang] = upMap; + if (lang === 'speco' && specoMap.length) this.maps.up.speco = pairsToMap(specoMap.map((m) => ({ from: m.to, to: m.from })), (text || '').split('\n').length); + const empty = !nonBlank(P); + if (lang !== 'lingo') { + if (empty) { this.sources.lingo = ''; delete stale.lingo; delete this.maps.down.lingo; } + else { + const r = pseudoToLingo(P); + if ('unsupported' in r) stale.lingo = unsupportedReason('lingo', r.unsupported); + else { this.sources.lingo = r.source; delete stale.lingo; this.maps.down.lingo = r.map; } + } + } + if (lang !== 'matho') { + if (empty) { this.sources.matho = ''; delete stale.matho; delete this.maps.down.matho; } + else { + const r = this.generateMatho(P, {}); + if ('unsupported' in r) stale.matho = unsupportedReason('matho', r.unsupported); + else { this.sources.matho = r.source; delete stale.matho; this.maps.down.matho = r.map; } + } + } + if (lang !== 'speco') { + // the contracts the pseudo holds, with every claim about something it no longer has kept (stage 7) + const r = reconcileSpec(P, this.sources.speco ?? ''); + this.sources.speco = r.source; + delete stale.speco; + } + delete this.maps.down.pseudo; + return this.result(prior); + } + + private result(prior: Sources): HubResult { + const stale = this.info!.stale ?? {}; + const P = this.sources.pseudo ?? ''; + const mojo = this.mojo(P); + if (!('unsupported' in mojo)) this.maps.down.mojo = mojo.map; else delete this.maps.down.mojo; + const changed = LINKED.filter((l) => l !== this.info!.authority && (this.sources[l] ?? '') !== (prior[l] ?? '')); + return { + sources: { ...this.sources }, authority: this.info!.authority, stale: { ...stale }, names: { ...(this.info!.names ?? {}) }, + maps: { up: { ...this.maps.up }, down: { ...this.maps.down } }, + mojo: mojoDerived(mojo), changed, + }; + } + + private mojo(P: string) { + if (!nonBlank(P)) return { source: '', map: [], notes: [] as Diagnostic[] }; + const known = this.studied?.source === P ? this.studied.study : undefined; + this.studied = null; + return pseudoToMojo(P, { call: callFor(P, this.call, known) }); + } + + /* ---------- rule 7: line maps compose through the hub ---------- */ + + /** The lines of pane `to` that match line `line` of pane `from`, going through the pseudo. */ + static relate(maps: HubMaps, from: LangId, line: number, to: LangId): number[] { + if (from === to) return [line]; + const pseudoLines = toPseudoLines(maps, from, line); + if (to === 'pseudo') return pseudoLines; + const out = new Set<number>(); + for (const p of pseudoLines) for (const l of fromPseudoLines(maps, to, p)) out.add(l); + return [...out].sort((a, b) => a - b); + } +} + +function mojoDerived(r: ReturnType<typeof pseudoToMojo> | { source: string; map: LineMap; notes: Diagnostic[] }): MojoDerived { + return 'unsupported' in r ? { source: '', map: [], diagnostics: r.unsupported } : { source: r.source, map: r.map, diagnostics: r.notes }; +} + +/** Pseudo lines for a line of pane `from`. */ +function toPseudoLines(maps: HubMaps, from: LangId, line: number): number[] { + if (from === 'pseudo') return [line]; + if (from !== 'lingo' && from !== 'matho' && from !== 'speco' && from !== 'mojo') return []; + const out: number[] = []; + const up = from === 'mojo' ? undefined : maps.up[from]; + const down = maps.down[from]; + if (up) up.forEach((src, i) => { if (src === line) out.push(i + 1); }); + else if (down) { const p = down[line - 1]; if (p) out.push(p); } + return out; +} + +/** Lines of pane `to` that were made from, or lowered into, pseudo line `p`. */ +function fromPseudoLines(maps: HubMaps, to: LangId, p: number): number[] { + if (to === 'pseudo') return [p]; + if (to !== 'lingo' && to !== 'matho' && to !== 'speco' && to !== 'mojo') return []; + const out: number[] = []; + const down = maps.down[to]; + const up = to === 'mojo' ? undefined : maps.up[to]; + if (down) down.forEach((src, i) => { if (src === p) out.push(i + 1); }); + else if (up) { const s = up[p - 1]; if (s) out.push(s); } + return out; +} + +/** Literal Mojo for a pseudo program, for callers outside an edit (the session's memo). */ +export function mojoFromPseudo(pseudo: string, call: string | null): MojoDerived { + if (!nonBlank(pseudo)) return { source: '', map: [], diagnostics: [] }; + try { return mojoDerived(pseudoToMojo(pseudo, { call: callFor(pseudo, call) })); } catch (e) { + return { source: '', map: [], diagnostics: [{ lang: 'mojo', severity: 'error', id: FAULT_ID, line: 1, col: 1, message: faultReason(e) }] }; + } +} + +/** The call that becomes `main`: the one the student set, else the one the pseudo's own routines suggest. */ +function callFor(pseudo: string, call: string | null, studied?: ReturnType<typeof studyOf>): string | null { + if (call && call.trim()) return call; + const st = studied ?? studyOf(pseudo); + return st.program ? defaultCall(st.program) : null; +} diff --git a/src/sync/pseudo-to-lingo.ts b/src/sync/pseudo-to-lingo.ts new file mode 100644 index 0000000..d9e3a03 --- /dev/null +++ b/src/sync/pseudo-to-lingo.ts @@ -0,0 +1,545 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Pseudo to lingo: the English steps a student could have written to get this pseudo. + * + * The generator writes only the sentence forms the reader (src/engines/pseudo/english.js, + * `readSteps`; the table is SENTENCE_FORMS) lowers back to the same program: + * + * FUNCTION COUNT_BASE(B, T) To count base for B and T: + * Total <- 0 Start with Total at 0. + * Total <- Total + 1 Add 1 to Total. + * FOR I <- 0 TO N For each i from 0 up to but not including N: + * FOR EACH C IN T For each letter c of T: + * IF ... ELSEIF ... ELSE If ...: / Otherwise, if ...: / Otherwise: + * RETURN V The answer is V. (Stop and answer V. when not last) + * + * It never writes English that reads as something else. Whatever the generator wrote is read + * back with `readSteps` before it is returned; if the reader does not accept it, or finds a + * different routine than the pseudo has, the answer is `unsupported`, with the reader's own + * message on the pseudo line it concerns. Anything without an English form (records, PARALLEL + * FOR, REPEAT, steps outside a routine, most built-ins) is `unsupported` too, one diagnostic + * per construct, naming it and its line. + * + * Contracts (REQUIRE, ENSURE, INVARIANT, DECREASES) are not in English; they ride along in + * speco (section 2.3 of the plan). `//` comments are carried across on their lines; the + * lowering strips them, so they are the one thing a round trip does not bring back. + */ +import { studyPseudo as study } from '../services/pseudo-study.js'; +import { readSteps } from '../engines/pseudo/english.js'; +import { walkStatements, walkExpr, statementExprs } from '../engines/pseudo/analyze.js'; +import { toDiagnostic } from '../services/pseudo.js'; +import type { Diagnostic } from '../common/types.js'; +import type { LineMap } from './types.js'; + +export interface LingoResult { source: string; map: LineMap; notes: Diagnostic[] } + +/** A construct English has no sentence for. Collected per statement; the generator goes on to find the rest. */ +class NoForm extends Error { + constructor(public what: string, public line: number) { super(what); } +} + +interface Line { text: string; from: number | null } + +/** A type as the reader's "X is a ..." sentence says it, or null when English has no words for it. */ +function kindWords(t: any): string | null { + const one: Record<string, string> = { INTEGER: 'a whole number', FLOAT: 'a decimal number', BOOL: 'yes or no', CHAR: 'a letter', STRING: 'a text' }; + const many: Record<string, string> = { INTEGER: 'whole numbers', FLOAT: 'decimal numbers', CHAR: 'letters', STRING: 'texts' }; + if (!t) return null; + if (one[t.k]) return one[t.k]; + if ((t.k === 'LIST' || t.k === 'ARRAY') && t.dims !== 2 && many[t.of?.k]) return `a list of ${many[t.of.k]}`; + return null; +} + +/** A type reduced to what has to agree between the pseudo's and the reader's idea of an input. */ +function typeKey(t: any): string { + if (!t || t.k === 'ANY') return 'ANY'; + if (t.k === 'LIST' || t.k === 'ARRAY') return `LIST<${typeKey(t.of)}>`; + if (t.k === 'SET') return `SET<${typeKey(t.of)}>`; + if (t.k === 'HASHMAP') return `MAP<${typeKey(t.key)},${typeKey(t.val)}>`; + if (t.k === 'LETTER') return 'CHAR'; + return t.k; +} + +const up = (s: unknown): string => String(s).toUpperCase(); + +/* Precedence levels, loosest first. Open phrases (ones that read an open-ended value after them) get their own level. */ +const OR = 0, AND = 1, NOT = 2, CMP = 3, ADD = 4, MUL = 5, NEG = 6, ATOM = 7; + +interface Text { s: string; p: number; open?: boolean } + +class Gen { + out: Line[] = []; + problems: Diagnostic[] = []; + notes: Diagnostic[] = []; + srcLines: string[]; + comments = new Map<number, any>(); + usedComments = new Set<number>(); + /** Lower-case names already introduced in the routine being written. */ + known = new Set<string>(); + curLine = 1; + subs: Map<string, any>; + + constructor(private program: any, private source: string, private withKinds = false) { + this.srcLines = source.split('\n'); + for (const c of program.comments || []) this.comments.set(c.line, c); + this.subs = program.subs; + } + + fail(what: string, line = this.curLine): never { throw new NoForm(what, line); } + unsupported(what: string, line: number): void { + this.problems.push({ lang: 'lingo', severity: 'warning', id: 'lingo-no-form', line, message: `English has no sentence for ${what}, so the lingo pane keeps its last good text.` }); + } + + /* ---------- names ---------- */ + /** A pseudo name as the words a student would write: Word_Count -> "Word Count". */ + nameText(n: string): string { return String(n).split('_').filter(Boolean).join(' '); } + + emit(indent: number, text: string, from: number | null): void { this.out.push({ text: `${' '.repeat(indent)}${text}`, from }); } + + leadingComments(line: number, indent: number): void { + for (const [l, c] of this.comments) { + if (l >= line || this.usedComments.has(l) || !c.standalone) continue; + this.usedComments.add(l); + this.emit(indent, `// ${c.text}`, l); + } + } + trailing(line: number): string { + const c = this.comments.get(line); + if (!c || c.standalone || this.usedComments.has(line)) return ''; + this.usedComments.add(line); + return ` // ${c.text}`; + } + + /* ---------- program ---------- */ + run(): LingoResult | null { + const p = this.program; + for (const c of p.constants.values()) this.unsupported('a CONSTANT', c.line); + for (const d of p.defines.values()) this.unsupported('a DEFINE set', d.line); + for (const t of p.types.values()) this.unsupported('a record type', t.node.line); + if (p.main.body.length) this.unsupported('steps outside a routine (every English step belongs to a routine)', p.main.body[0].line); + let first = true; + for (const sub of p.subs.values()) { + if (!first) this.emit(0, '', null); + first = false; + this.routine(sub); + } + this.leadingComments(Infinity, 0); + if (this.problems.length) return null; + while (this.out.length && !this.out[this.out.length - 1].text.trim()) this.out.pop(); + return { source: this.out.map((l) => l.text).join('\n') + '\n', map: this.out.map((l) => l.from), notes: this.notes }; + } + + routine(sub: any): void { + const s = sub.node; + this.curLine = s.line; + this.leadingComments(s.line, 0); + this.known = new Set(); + try { + const params = sub.params.map((p: any) => { + if (p.mode === 'BYREF') this.fail('a BYREF parameter', s.line); + if (!/^[A-Za-z]$/.test(p.name) && !/^[A-Z][a-z0-9]+$/.test(p.name)) this.fail(`the parameter ${p.name}, which English can only take as a single letter or a Capitalized word`, s.line); + this.known.add(p.name.toLowerCase()); + return p.name; + }); + const words = String(sub.name).split('_').filter(Boolean).map((w: string) => w.toLowerCase()); + if (!words.length) this.fail('a routine with no name', s.line); + const list = params.length > 1 ? `${params.slice(0, -1).join(', ')} and ${params[params.length - 1]}` : params[0]; + this.emit(0, `To ${words.join(' ')}${params.length ? ` for ${list}` : ''}:${this.trailing(s.line)}`, s.line); + } catch (e) { + if (!(e instanceof NoForm)) throw e; + this.unsupported(e.what, e.line); + return; + } + this.earlyInits(sub); + this.block(s.body, 1, { top: true, loops: 0 }, sub); + if (this.withKinds) this.kindLines(sub); + } + + /** + * English names a value where a step first gives it one, and the name is known only inside the block of + * that step. A value the pseudo declares up top, first sets inside a branch or loop, and uses outside it + * therefore gets a starting value at the top of the routine ("Start with Partner at ''."), so the later + * sentences can name it. It is the one place the English holds a value the pseudo leaves unset. + */ + earlyInits(sub: any): void { + const declared = new Map<string, { type: any; line: number; name: string }>(); + walkStatements(sub.node.body, (st: any) => { + if (st.kind === 'Declare' && ['INTEGER', 'FLOAT', 'BOOL', 'STRING', 'CHAR'].includes(st.resolved?.k)) for (const n of st.names) declared.set(up(n.name), { type: st.resolved, line: st.line, name: n.name }); + return undefined; + }); + if (!declared.size) return; + const first = new Map<string, string>(); // name -> the block path of its first assignment + const outside = new Set<string>(); + const readsOf = (e: any, path: string) => walkExpr(e, (x: any) => { if (x.kind === 'Name') note(up(x.name), path); return undefined; }); + const note = (name: string, path: string) => { + const f = first.get(name); + if (f === undefined) return; + if (!path.startsWith(f)) outside.add(name); + }; + const visit = (body: any[], path: string) => { + body.forEach((st: any, i: number) => { + const here = `${path}/${i}`; + const inner = (b: any[], k: string) => visit(b, `${here}:${k}`); + for (const e of statementExprs(st)) readsOf(e, path); + if ((st.kind === 'Assign' || st.kind === 'Input') && st.target.kind === 'Name') { + const n = up(st.target.name); + if (declared.has(n) && !first.has(n)) first.set(n, path); + } + if (st.kind === 'If') { st.branches.forEach((b: any, k: number) => inner(b.body, `b${k}`)); if (st.elseBody) inner(st.elseBody, 'e'); } + else if (st.kind === 'Case') { st.arms.forEach((a: any, k: number) => inner(a.body, `a${k}`)); if (st.otherwise) inner(st.otherwise, 'o'); } + else if (['For', 'ForEach', 'While', 'Repeat'].includes(st.kind)) inner(st.body, 'l'); + }); + }; + visit(sub.node.body, 'r'); + for (const [name, at] of first) { + if (at === 'r' || !outside.has(name)) continue; + const info = declared.get(name)!; + const value = info.type.k === 'INTEGER' ? '0' : info.type.k === 'FLOAT' ? '0.0' : info.type.k === 'BOOL' ? 'false' : info.type.k === 'STRING' ? '""' : "' '"; + this.emit(1, `Start with ${this.nameText(info.name)} at ${value}.`, info.line); + this.notes.push({ lang: 'lingo', severity: 'note', id: 'lingo-early-start', line: info.line, message: `${info.name} is first set inside a branch or loop and used after it, and English can name a value only where a step first gives it one. The English starts it at ${value} at the top of the routine; the pseudo leaves it unset until then.` }); + this.known.add(String(info.name).toLowerCase()); + } + } + + /** "X is a whole number." after the routine: what each input is, when its steps do not settle it. */ + kindLines(sub: any): void { + const lines: string[] = []; + for (const p of sub.params) { + const k = kindWords(p.type); + if (!k) { this.unsupported(`the kind of the input ${p.name} (${p.type?.k})`, sub.node.line); return; } + const have = this.kindOf.get(String(p.name).toLowerCase()); + if (have && have !== k) { this.unsupported(`the input ${p.name}, which two routines give different kinds (English keeps one kind per name)`, sub.node.line); return; } + this.kindOf.set(String(p.name).toLowerCase(), k); + lines.push(`${p.name} is ${k}.`); + } + if (!lines.length) return; + this.emit(0, '', null); + for (const l of lines) this.emit(0, l, sub.node.line); + } + + kindOf = new Map<string, string>(); + + block(body: any[], d: number, ctx: { top: boolean; loops: number }, sub: any): void { + const before = this.out.length; + body.forEach((st, i) => { + this.curLine = st.line ?? this.curLine; + this.leadingComments(st.line, d); + try { this.stmt(st, d, ctx, sub, i === body.length - 1); } catch (e) { + if (!(e instanceof NoForm)) throw e; + this.unsupported(e.what, e.line); + } + }); + void before; + } + + /* ---------- statements ---------- */ + stmt(s: any, d: number, ctx: { top: boolean; loops: number }, sub: any, last: boolean): void { + const tr = () => this.trailing(s.line); + const E = (e: any) => this.ex(e).s; + switch (s.kind) { + case 'Declare': { + const t = s.resolved; + for (const n of s.names) { + if (t.k === 'LIST' || t.k === 'ARRAY') { this.emit(d, `Start with an empty list called ${this.nameText(n.name)}.${tr()}`, s.line); if (t.k === 'ARRAY') this.fail('a fixed-length ARRAY', s.line); } + else if (t.k === 'SET') this.emit(d, `Start with an empty set called ${this.nameText(n.name)}.${tr()}`, s.line); + else if (t.k === 'HASHMAP') { + if (t.val?.k !== 'INTEGER') this.fail('a HASHMAP whose values are not counts', s.line); + this.emit(d, `Start with an empty tally called ${this.nameText(n.name)}.${tr()}`, s.line); + } else if (t.k === 'RECORD') this.fail('a record', s.line); + // a number, text, or yes-or-no value is introduced by the step that first gives it a value + if (['LIST', 'ARRAY', 'SET', 'HASHMAP'].includes(t.k)) this.known.add(String(n.name).toLowerCase()); + } + return; + } + case 'Assign': this.assign(s, d, ctx, tr()); return; + case 'Output': { + // "Show a, b" prints a, a space, then b: the pseudo's OUTPUT a, " ", b. Values with nothing between them have no English. + const vals: any[] = []; + s.values.forEach((v: any, i: number) => { + if (i % 2 === 1) { if (!(v.kind === 'Str' && v.value === ' ')) this.fail('an OUTPUT of several values with no single space between them (English puts one space between the values it shows)', s.line); } else vals.push(v); + }); + if (s.values.length % 2 === 0) this.fail('an OUTPUT that ends with a space', s.line); + this.emit(d, `Show ${vals.map(E).join(', ')}.${tr()}`, s.line); + return; + } + case 'Input': this.fail('INPUT', s.line); return; + case 'If': { + s.branches.forEach((b: any, i: number) => { + this.emit(d, `${i ? 'Otherwise, if' : 'If'} ${E(b.cond)}:${i ? this.trailing(b.line) : tr()}`, b.line); + this.block(b.body, d + 1, { ...ctx, top: false }, sub); + }); + if (s.elseBody) { this.emit(d, 'Otherwise:', s.elseLine); this.block(s.elseBody, d + 1, { ...ctx, top: false }, sub); } + return; + } + case 'For': { + const v = this.nameText(s.var); + const lo = E(s.start); const hi = E(s.stop); + const step = s.step ? this.stepOf(s.step) : 1; + if (s.parallel) this.fail('PARALLEL FOR', s.line); + this.known.add(String(s.var).toLowerCase()); + let range: string; + if (step === 1) range = `from ${lo} up to but not including ${hi}`; + else if (step > 1) range = `from ${lo} up to but not including ${hi}, counting by ${step}`; + else if (step === -1) range = `from ${lo} down to but not including ${hi}`; + else range = `from ${lo} down to but not including ${hi}, counting by ${-step}`; + this.emit(d, `For each ${v} ${range}:${tr()}`, s.line); + this.block(s.body, d + 1, { top: false, loops: ctx.loops + 1 }, sub); + return; + } + case 'ForEach': { + const it = s.iterable; + const v = this.nameText(s.var); + this.known.add(String(s.var).toLowerCase()); + if (it.type?.k === 'STRING') this.emit(d, `For each letter ${v} of ${this.closed(it)}:${tr()}`, s.line); + else if (it.type?.k === 'LIST' || it.type?.k === 'ARRAY') this.emit(d, `For each item ${v} of ${this.closed(it)}:${tr()}`, s.line); + else this.fail('FOR EACH over this kind of collection', s.line); + this.block(s.body, d + 1, { top: false, loops: ctx.loops + 1 }, sub); + return; + } + case 'While': + this.emit(d, `Repeat while ${E(s.cond)}:${tr()}`, s.line); + this.block(s.body, d + 1, { top: false, loops: ctx.loops + 1 }, sub); + return; + case 'Repeat': this.fail('REPEAT ... UNTIL, which English reads as a loop that tests first', s.line); return; + case 'Case': { + if (!s.arms.length) this.fail('a CASE with only an OTHERWISE', s.line); + const subj = this.ex(s.subject); + const subjS = this.wrap(subj, ADD); + s.arms.forEach((a: any, i: number) => { + const conds = a.labels.map((l: any) => (l.value ? `${subjS} is ${this.wrap(this.ex(l.value), ADD)}` : `${subjS} is at least ${this.wrap(this.ex(l.low), ADD)} and ${subjS} is less than ${this.wrap(this.ex(l.high), ADD)}`)); + this.emit(d, `${i ? 'Otherwise, if' : 'If'} ${conds.join(' or ')}:`, a.line); + this.block(a.body, d + 1, { ...ctx, top: false }, sub); + }); + if (s.otherwise) { this.emit(d, 'Otherwise:', s.otherwiseLine); this.block(s.otherwise, d + 1, { ...ctx, top: false }, sub); } + return; + } + case 'Break': this.emit(d, `Stop the loop.${tr()}`, s.line); return; + case 'Continue': this.emit(d, `Skip to the next position.${tr()}`, s.line); return; + case 'Return': { + if (!s.value) { + if (ctx.loops > 0) this.fail('a RETURN with no value inside a loop', s.line); + this.emit(d, `Stop.${tr()}`, s.line); + return; + } + const late = ctx.top && last && ctx.loops === 0; + this.emit(d, `${late ? 'The answer is' : 'Stop and answer'} ${E(s.value)}.${tr()}`, s.line); + return; + } + case 'Call': this.emit(d, `${this.routineText(s)}.${tr()}`, s.line); return; + case 'BuiltinCall': this.builtinStep(s, d, tr()); return; + case 'Assert': this.emit(d, `Check that ${E(s.expr)}.${tr()}`, s.line); return; + case 'BlankStmt': this.fail('a blank to fill in', s.line); return; + default: this.fail(`a ${s.kind} step`, s.line); + } + } + + stepOf(e: any): number { + if (e.kind === 'Num' && Number.isInteger(e.value) && e.value !== 0) return e.value; + if (e.kind === 'Unary' && e.op === '-' && e.arg.kind === 'Num' && Number.isInteger(e.arg.value) && e.arg.value > 0) return -e.arg.value; + return this.fail('a loop step that is not a whole number written out', e.line ?? this.curLine); + } + + sameName(a: any, b: any): boolean { return a?.kind === 'Name' && b?.kind === 'Name' && up(a.name) === up(b.name); } + + assign(s: any, d: number, ctx: { top: boolean; loops: number }, tr: string): void { + const t = s.target; + const v = s.value; + const E = (e: any) => this.ex(e).s; + if (t.kind === 'Name') { + const name = this.nameText(t.name); + const key = String(t.name).toLowerCase(); + if (v.kind === 'Binary' && this.sameName(v.left, t) && v.op === '+' && this.known.has(key) && t.type?.k !== 'STRING' && t.type?.k !== 'CHAR') { this.emit(d, `Add ${E(v.right)} to ${name}.${tr}`, s.line); return; } + if (v.kind === 'Binary' && this.sameName(v.left, t) && v.op === '-' && this.known.has(key)) { this.emit(d, `Subtract ${E(v.right)} from ${name}.${tr}`, s.line); return; } + if (v.kind === 'Binary' && this.sameName(v.left, t) && v.op === '&' && this.known.has(key)) { this.emit(d, `Add ${E(v.right)} to the end of ${name}.${tr}`, s.line); return; } + if (v.kind === 'ListLit' && v.items.length === 0) { this.emit(d, `Set ${name} to an empty list.${tr}`, s.line); this.known.add(key); return; } + const fresh = !this.known.has(key); + this.known.add(key); + if (t.type?.k === 'FLOAT' && v.type?.k === 'INTEGER' && v.kind !== 'Num') this.fail('a whole-number value given to a FLOAT, which English would keep a whole number', s.line); + const value = t.type?.k === 'FLOAT' && v.kind === 'Num' && !/[.e]/.test(String(v.value)) ? `${v.value}.0` : E(v); + this.emit(d, fresh && ctx.top ? `Start with ${name} at ${value}.${tr}` : `Set ${name} to ${value}.${tr}`, s.line); + return; + } + if (t.kind === 'Index' && t.indices.length === 1 && t.target.kind === 'Name') { + const tt = t.target.type?.k; + if (tt === 'LIST' || tt === 'ARRAY') { this.emit(d, `Set the item of ${this.nameText(t.target.name)} at ${E(t.indices[0])} to ${E(v)}.${tr}`, s.line); return; } + if (tt === 'HASHMAP') { this.emit(d, `Set the count for ${E(t.indices[0])} in ${this.nameText(t.target.name)} to ${E(v)}.${tr}`, s.line); return; } + } + this.fail('a change to this part of a value', s.line); + } + + builtinStep(s: any, d: number, tr: string): void { + const [a, b] = s.args; + const E = (e: any) => this.ex(e).s; + switch (s.name) { + case 'APPEND': if (a.kind === 'Name') { this.emit(d, `Add ${E(b)} to ${this.nameText(a.name)}.${tr}`, s.line); return; } break; + case 'ADD': if (a.kind === 'Name') { this.emit(d, `Add ${E(b)} to ${this.nameText(a.name)}.${tr}`, s.line); return; } break; + case 'SORT': if (a.kind === 'Name') { this.emit(d, `Sort ${this.nameText(a.name)}.${tr}`, s.line); return; } break; + default: break; + } + this.fail(`${s.name}(...)`, s.line); + } + + /** A routine call as a step or value: the routine's own words, with the arguments where its inputs stand. */ + routineText(c: any): string { + const sub = this.subs.get(up(c.name)); + if (!sub) this.fail(`a call of ${c.name}`, this.curLine); + const words = String(sub.name).split('_').filter(Boolean).map((w: string) => w.toLowerCase()); + const args = c.args.map((a: any) => this.closed(a)); + const list = args.length > 1 ? `${args.slice(0, -1).join(', ')} and ${args[args.length - 1]}` : args[0]; + return `${words.join(' ')}${args.length ? ` for ${list}` : ''}`; + } + + /* ---------- expressions ---------- */ + /** An expression as a primary: wrapped in parentheses unless it already is one. */ + closed(e: any): string { + const t = this.ex(e); + return t.p >= ATOM && !t.open ? t.s : `(${t.s})`; + } + + wrap(t: Text, min: number): string { return t.p >= min && !(t.open && min > CMP) ? t.s : `(${t.s})`; } + + /** + * English for an expression. `p` is how tightly the text binds; `open` marks a phrase that reads a + * value after it ("the letter of T at i + 1"), which must be parenthesized before anything follows. + */ + ex(e: any, arg = false): Text { + void arg; + switch (e.kind) { + case 'Num': { const t = String(e.value); return { s: e.float && !/[.e]/.test(t) ? `${t}.0` : t, p: ATOM }; } + case 'Str': return { s: `"${String(e.value).replace(/"/g, '')}"`, p: ATOM }; + case 'Char': return { s: `'${e.value}'`, p: ATOM }; + case 'Bool': return { s: e.value ? 'true' : 'false', p: ATOM }; + case 'Name': return { s: this.nameText(e.name), p: ATOM }; + case 'Unary': + if (e.op === 'NOT' && e.arg.kind === 'Binary' && e.arg.op === 'IN') return { s: this.membership(e.arg, true), p: CMP }; + if (e.op === 'NOT') return { s: `not ${this.wrap(this.ex(e.arg), ATOM)}`, p: NOT }; + return { s: `-${this.wrap(this.ex(e.arg), ATOM)}`, p: NEG }; + case 'Binary': return this.binary(e); + case 'Chain': { + // a < b < c: each pair is a test + const parts: string[] = []; + e.operands.slice(1).forEach((o: any, i: number) => parts.push(this.binary({ kind: 'Binary', op: e.ops[i], left: e.operands[i], right: o }).s)); + return { s: parts.map((x) => `(${x})`).join(' and '), p: AND }; + } + case 'Index': return this.index(e); + case 'CallExpr': return this.callExpr(e); + case 'Quant': { + if (e.q !== 'FORALL' || e.domain.kind !== 'Range') this.fail(`${e.q} written this way`, this.curLine); + const v = this.nameText(e.var); + return { s: `for every ${v} from ${this.ex(e.domain.lo).s} up to but not including ${this.ex(e.domain.hi).s}, ${this.ex(e.body).s}`, p: OR - 1, open: true }; + } + default: return this.fail(`a ${e.kind} value`, this.curLine); + } + } + + binary(e: any): Text { + const op = e.op; + const l = this.ex(e.left); const r = this.ex(e.right); + switch (op) { + case 'AND': return { s: `${this.wrap(l, AND)} and ${this.wrap(r, AND + 1)}`, p: AND }; + case 'OR': return { s: `${this.wrap(l, OR)} or ${this.wrap(r, OR + 1)}`, p: OR }; + case '==': case '!=': case '<': case '<=': case '>': case '>=': { + const word = { '==': 'is', '!=': 'is not', '<': 'is less than', '<=': 'is at most', '>': 'is more than', '>=': 'is at least' }[op as string]; + return { s: `${this.wrap(l, ADD)} ${word} ${this.wrap(r, ADD)}`, p: CMP }; + } + case 'IN': return { s: this.membership(e, false), p: CMP }; + case '+': case '-': return { s: `${this.operand(l, ADD)} ${op} ${this.operand(r, ADD + 1)}`, p: ADD }; + case '&': return { s: `${this.operand(l, ADD)} joined with ${this.operand(r, ADD + 1)}`, p: ADD }; + case '*': return { s: `${this.operand(l, MUL)} * ${this.operand(r, MUL + 1)}`, p: MUL }; + case '/': return { s: `${this.operand(l, MUL)} / ${this.operand(r, MUL + 1)}`, p: MUL }; + case 'DIV': return { s: `(${this.operand(l, MUL)} / ${this.operand(r, MUL + 1)} rounded down)`, p: ATOM }; + case 'MOD': return { s: `${this.operand(l, MUL)} % ${this.operand(r, MUL + 1)}`, p: MUL }; + case '^': return this.fail('a power', this.curLine); + default: return this.fail(`the operator ${op}`, this.curLine); + } + } + + /** x is in S, x is a key of M, x is not in S, and a value inside a range. */ + membership(e: any, negated: boolean): string { + const x = this.wrap(this.ex(e.left), ADD); + if (e.right.kind === 'Range') { + const lo = this.wrap(this.ex(e.right.lo), ADD); const hi = this.wrap(this.ex(e.right.hi), ADD); + return negated ? `(${x} is less than ${lo} or ${x} is at least ${hi})` : `(${x} is at least ${lo} and ${x} is less than ${hi})`; + } + const k = e.right.type?.k; + if (k !== 'SET' && k !== 'HASHMAP' && k !== 'LIST') this.fail('IN on this kind of value', this.curLine); + return `${x} is ${negated ? 'not ' : ''}${k === 'HASHMAP' ? 'a key of' : 'in'} ${this.closed(e.right)}`; + } + + /** An operand of arithmetic: open phrases are parenthesized so they cannot swallow what follows. */ + operand(t: Text, min: number): string { return t.open || t.p < min ? `(${t.s})` : t.s; } + + index(e: any): Text { + const t = e.target; + if (e.indices.length === 1) { + const k = this.ex(e.indices[0]); + const idx = k.p >= ADD && !k.open ? k.s : `(${k.s})`; + if (t.type?.k === 'STRING') return { s: `the letter of ${this.closed(t)} at ${idx}`, p: ATOM, open: true }; + if (t.type?.k === 'LIST' || t.type?.k === 'ARRAY') return { s: `the item of ${this.closed(t)} at ${idx}`, p: ATOM, open: true }; + if (t.type?.k === 'HASHMAP') return { s: `the count for ${idx} in ${this.closed(t)}`, p: ATOM }; + } + return this.fail('this kind of position', this.curLine); + } + + callExpr(e: any): Text { + if (!e.builtin) return { s: `the result of ${this.routineText(e)}`, p: ATOM, open: true }; + const [a, b, c] = e.args; + // the reader takes "of A and B" and "of X" as one value each, so anything with an operator in it is parenthesized + const arg = (x: any) => this.closed(x); + const sum = (x: any) => { const t = this.ex(x); return t.p >= ADD && !t.open ? t.s : `(${t.s})`; }; + switch (e.builtin) { + case 'LENGTH': return { s: `the length of ${this.closed(a)}`, p: ATOM }; + case 'MIN': if (e.args.length === 2) return { s: `the smaller of ${arg(a)} and ${arg(b)}`, p: ATOM, open: true }; break; + case 'MAX': if (e.args.length === 2) return { s: `the larger of ${arg(a)} and ${arg(b)}`, p: ATOM, open: true }; break; + case 'ABS': return { s: `the absolute value of ${arg(a)}`, p: ATOM, open: true }; + case 'SQRT': return { s: `the square root of ${arg(a)}`, p: ATOM, open: true }; + case 'SLICE': if (a.type?.k === 'STRING') return { s: `the part of ${this.closed(a)} from ${sum(b)} up to but not including ${sum(c)}`, p: ATOM, open: true }; break; + default: break; + } + return this.fail(`the built-in ${e.builtin}`, this.curLine); + } +} + +/** + * Generate lingo from pseudo. `unsupported` when the pseudo has errors, when part of it has no English + * form, or when what was written does not read back as the same routines. + */ +export function pseudoToLingo(pseudo: string): LingoResult | { unsupported: Diagnostic[] } { + const st: any = study(pseudo, { coach: false }); + if (!st.ok || !st.program) { + const errors = (st.diags as any[]).filter((d) => d.severity === 'error').map((d) => toDiagnostic(d)); + return { unsupported: errors.length ? errors : [{ lang: 'pseudo', severity: 'error', id: 'lingo-no-program', line: 1, message: 'The pseudo does not read yet, so there is nothing to put into English.' }] }; + } + const program = st.program; + if (!program.subs.size && !program.main.body.length) return { unsupported: [{ lang: 'lingo', severity: 'warning', id: 'lingo-no-form', line: 1, message: 'There is no routine to put into English.' }] }; + const wanted = [...program.subs.values()] as any[]; + let last: { unsupported: Diagnostic[] } | null = null; + // First the terse English (the steps settle what each input is), then, if the reader read an input as + // something else, with a sentence saying what each input is. + for (const withKinds of [false, true]) { + const gen = new Gen(program, pseudo, withKinds); + const result = gen.run(); + if (!result) return { unsupported: gen.problems }; + const checked = readsBack(result, wanted); + if (!('unsupported' in checked)) return result; + last = checked; + } + return last!; +} + +/** The guarantee: what was written reads, and reads as the same routines with the same kinds of input. */ +function readsBack(result: LingoResult, wanted: any[]): { ok: true } | { unsupported: Diagnostic[] } { + const back: any = readSteps(result.source); + const bad = (back.diags as any[]).filter((d) => d.severity === 'error'); + const sameShape = back.routines.length === wanted.length && back.routines.every((r: any, i: number) => r.params.length === wanted[i].params.length + && r.params.every((p: any, j: number) => up(String(p.display).replace(/ /g, '_')) === up(wanted[i].params[j].name) && typeKey(p.type) === typeKey(wanted[i].params[j].type))); + if (back.ok && !bad.length && sameShape) return { ok: true }; + const problems: Diagnostic[] = bad.slice(0, 3).map((d: any) => ({ + lang: 'lingo' as const, severity: 'warning' as const, id: 'lingo-no-form', line: result.map[(d.line ?? 1) - 1] ?? 1, + message: `English cannot say this the way the reader needs it (${String(d.message).replace(/\s+/g, ' ').slice(0, 160)}), so the lingo pane keeps its last good text.`, + })); + if (!problems.length) problems.push({ lang: 'lingo', severity: 'warning', id: 'lingo-no-form', line: 1, message: 'The English written for this pseudo does not read back as the same routines and inputs, so the lingo pane keeps its last good text.' }); + return { unsupported: problems }; +} + diff --git a/src/sync/pseudo-to-matho.ts b/src/sync/pseudo-to-matho.ts new file mode 100644 index 0000000..71c0520 --- /dev/null +++ b/src/sync/pseudo-to-matho.ts @@ -0,0 +1,625 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Pseudo to matho: the equations a program says, written in LaTeX, for the programs that are equations. + * + * The matho reader (src/services/matho.ts) lowers an equation to one FUNCTION: a single RETURN, a sum or + * product as an accumulator loop, a piecewise definition as an IF chain, a reference to an earlier equation + * as a call. This generator recognizes exactly those shapes and writes the equation that lowers to them: + * + * FUNCTION X_BAR(N, X) \bar{x} = \frac{1}{n} \sum_{i=0}^{n-1} x_i + * Total <- 0.0 ; FOR I <- 0 TO N ... + * RETURN 1 / N * Total + * + * Anything else (a WHILE loop, text, a record, output, a loop that does more than add up) is `unsupported`, + * with the construct and its line: "matho has no form for a WHILE loop (line 7)". Matho is a notation for + * values, so a program that prints or reads input is not one of its equations. + * + * It never writes an equation that reads as something else. What was written is lowered again by the + * matho reader and held to the pseudo it came from: the same functions, taking the same kinds of input, giving + * the same values on sample inputs (and stopping the same way where the pseudo stops). A program that fails + * that check is `unsupported` too. + * + * Comments are not carried across: the lowering writes the equation itself as a comment above each function, + * and carrying that back would put a stale copy of the equation above the new one. + * + * Names. The reader turns `\bar{x}` into X_BAR and `\sigma` into SIGMA, so the student's spelling is not + * in the pseudo. The `NameMap` hint carries it (pseudo name, upper case, to the student's LaTeX); without a + * hint the default rules apply: SIGMA is `\sigma`, X_BAR is `\bar{x}`, a single letter is lower case, a longer + * name is `\text{...}`. + */ +import { studyPseudo as study } from '../services/pseudo-study.js'; +import { walkExpr } from '../engines/pseudo/analyze.js'; +import { studyOf, runPseudo, toDiagnostic } from '../services/pseudo.js'; +import { parseMatho } from '../services/matho.js'; +import type { Diagnostic } from '../common/types.js'; +import type { LineMap, NameMap } from './types.js'; + +export interface MathoResult { source: string; map: LineMap; notes: Diagnostic[] } + +class NoForm extends Error { + constructor(public what: string, public line: number) { super(what); } +} + +const up = (s: unknown): string => String(s).toUpperCase(); + +/* The Greek names the reader knows, by the pseudo name it gives them (src/services/matho.ts, GREEK). */ +const GREEK_LOWER: Record<string, string> = { + ALPHA: 'alpha', BETA: 'beta', GAMMA: 'gamma', DELTA: 'delta', EPSILON: 'epsilon', ZETA: 'zeta', ETA: 'eta', THETA: 'theta', IOTA: 'iota', + KAPPA: 'kappa', LAMBDA: 'lambda', MU: 'mu', NU: 'nu', XI: 'xi', RHO: 'rho', SIGMA: 'sigma', TAU: 'tau', UPSILON: 'upsilon', PHI: 'phi', + CHI: 'chi', PSI: 'psi', OMEGA: 'omega', +}; +const GREEK_UPPER: Record<string, string> = { + GAMMAU: 'Gamma', DELTAU: 'Delta', THETAU: 'Theta', LAMBDAU: 'Lambda', XIU: 'Xi', PIU: 'Pi', SIGMAU: 'Sigma', PHIU: 'Phi', PSIU: 'Psi', OMEGAU: 'Omega', +}; + +/** What a name looks like once the reader has made it a pseudo name (Word_Count -> WordCount): the form we spell. */ +function normal(name: string): string { + const s = String(name); + if (/^[A-Za-z]$/.test(s)) return s.toUpperCase(); + return s.split('_').filter(Boolean).map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join(''); +} + +/** LaTeX for a variable called `name` in pseudo. `hint` is the student's own spelling, when it is known. */ +function spellVar(name: string, hint: string | undefined, vec: boolean, noSub = false): string { + let t: string; + const n = String(name); + if (hint) t = hint; + else if (GREEK_LOWER[up(n)]) t = `\\${GREEK_LOWER[up(n)]}`; + else if (GREEK_UPPER[up(n)]) t = `\\${GREEK_UPPER[up(n)]}`; + else { + const m = /^([A-Za-z])(Bar|Hat|Tilde)$/.exec(n) ?? /^([A-Za-z])_(BAR|HAT|TILDE)$/i.exec(n); + if (m) t = `\\${m[2].toLowerCase()}{${m[1] === 'E' ? '\\text{E}' : m[1].toLowerCase()}}`; + else if (/^[A-Za-z]$/.test(n)) t = n === 'E' || n === 'e' ? '\\text{E}' : n.toLowerCase(); + else if (!noSub && /^[A-Za-z][0-9]+$/.test(n)) t = `${n[0].toLowerCase()}_{${n.slice(1)}}`; + else t = `\\text{${normal(n)}}`; + } + return vec ? `\\vec{${t}}` : t; +} + +/** The left side of an equation for a function: a name the reader turns back into this FUNCTION name. */ +function spellFunction(fn: string, params: string[], hint: string | undefined, spell: (p: string) => string): { lhs: string; call: boolean } { + if (hint) return { lhs: hint, call: false }; + const parts = String(fn).split('_').filter(Boolean); + if (parts.length === 1) { + const w = parts[0]; + if (GREEK_LOWER[w] || GREEK_UPPER[w]) return { lhs: `\\${GREEK_LOWER[w] ?? GREEK_UPPER[w]}`, call: false }; + const base = w.length === 1 ? w.toLowerCase() : `\\text{${w.toLowerCase()}}`; + return { lhs: `${base === 'e' ? '\\text{e}' : base}(${params.map(spell).join(', ')})`, call: true }; + } + // X_BAR, SIGMA_SQUARED, COUNT_BASE: a name with a mark or a subscript, written bare + let rest = parts.slice(); + let squared = false; + if (rest.length > 1 && rest[rest.length - 1] === 'SQUARED') { squared = true; rest = rest.slice(0, -1); } + const head = rest[0]; + const base = GREEK_LOWER[head] ? `\\${GREEK_LOWER[head]}` : head.length === 1 ? head.toLowerCase() : `\\text{${head.toLowerCase()}}`; + let t = base; + const mark = rest.length === 2 && /^(BAR|HAT|TILDE)$/.test(rest[1]) && (head.length === 1 || GREEK_LOWER[head]); + if (mark) t = `\\${rest[1].toLowerCase()}{${base}}`; + else if (rest.length > 1) t = `${base}_{\\text{${rest.slice(1).map((x) => x.toLowerCase()).join(' ')}}}`; + if (squared) t = `${mark ? t : t}^2`; + return { lhs: t, call: false }; +} + +/* Precedence: how tightly a piece of LaTeX binds. A sum or product written with its bounds reads everything + after it as its body, so it is parenthesized wherever something follows it. */ +const BIG = 0, ADD = 1, NEG = 2, MUL = 3, POW = 4, ATOM = 5; +interface T { s: string; p: number } +const wrap = (t: T, min: number): string => (t.p < min ? `(${t.s})` : t.s); + +interface Scope { + /** name (upper case) -> what stands for it in the LaTeX */ + names: Map<string, T>; + /** loop variables in scope, by name: the letter, and the lower bound as written */ + loops: Map<string, { letter: string; shifted: boolean; lo: any }>; + /** the lists a loop variable indexes, in the order the reader will find them */ + letters: Set<string>; +} + +const CMP_TEX: Record<string, string> = { '<': '<', '<=': '\\le', '>': '>', '>=': '\\ge', '==': '=', '!=': '\\ne' }; +const ONE_ARG: Record<string, string> = { SIN: '\\sin', COS: '\\cos', TAN: '\\tan', EXP: '\\exp', LOG: '\\ln', LOG10: '\\log' }; + +class Gen { + problems: Diagnostic[] = []; + lines: Array<{ text: string; from: number | null }> = []; + notes: Diagnostic[] = []; + /** Functions already written, with how a call of them is written. */ + written = new Map<string, { call: boolean; lhs: string; params: string[] }>(); + curLine = 1; + vecs = new Set<string>(); + paramNames = new Set<string>(); + + constructor(private program: any, private hints: NameMap) { + } + + fail(what: string, line = this.curLine): never { throw new NoForm(what, line); } + unsupported(what: string, line: number): void { + this.problems.push({ lang: 'matho', severity: 'warning', id: 'matho-no-form', line, message: `matho has no form for ${what} (line ${line}), so the matho pane keeps its last good text.` }); + } + + hint(name: string): string | undefined { return this.hints[up(name)]; } + /** A function written with its inputs listed (f(x, y)) cannot list a subscripted input, so such inputs are written as plain names there. */ + noSub = false; + spell(name: string): string { return spellVar(name, this.hint(name), this.vecs.has(up(name)), this.noSub); } + + run(): MathoResult | null { + const p = this.program; + for (const c of p.constants.values()) this.unsupported('a CONSTANT', c.line); + for (const d of p.defines.values()) this.unsupported('a DEFINE set', d.line); + for (const t of p.types.values()) this.unsupported('a record type', t.node.line); + if (p.main.body.length) this.unsupported('steps outside a function (an equation is a function)', p.main.body[0].line); + let first = true; + for (const sub of p.subs.values()) { + const at = this.lines.length; + try { + const eq = this.routine(sub); + if (!first) this.lines.push({ text: '', from: null }); + first = false; + for (const l of eq) this.lines.push(l); + } catch (e) { + if (!(e instanceof NoForm)) throw e; + this.lines.length = at; + this.unsupported(e.what, e.line); + } + } + if (this.problems.length) return null; + const out = this.lines; + while (out.length && !out[out.length - 1].text.trim()) out.pop(); + return { source: out.map((l) => l.text).join('\n') + '\n', map: out.map((l) => l.from), notes: this.notes }; + } + + /* ---------- one function ---------- */ + routine(sub: any): Array<{ text: string; from: number | null }> { + const s = sub.node; + this.curLine = s.line; + if (sub.kind !== 'Function' || !sub.returns || !['INTEGER', 'FLOAT'].includes(sub.returns.k)) this.fail(sub.kind === 'Function' ? `a function that answers ${sub.returns?.k ?? 'nothing'} (equations answer numbers)` : 'a PROCEDURE', s.line); + const params: string[] = []; + this.paramNames = new Set(); + for (const p of sub.params) { + if (p.mode === 'BYREF') this.fail('a BYREF parameter', s.line); + const t = p.type; + const num = (x: any) => ['INTEGER', 'FLOAT'].includes(x?.k); + const ok = num(t) || ((t?.k === 'LIST' || t?.k === 'ARRAY') && (num(t.of) || (t.of?.k === 'LIST' && num(t.of.of)))); + if (!ok) this.fail(`the input ${p.name}, which is not a number or a list of numbers`, s.line); + if (t.k === 'ARRAY') this.fail('a fixed-length ARRAY', s.line); + params.push(p.name); + this.paramNames.add(up(p.name)); + } + // inputs used as vectors (DOT, NORM_OF) are written \vec{a} everywhere + this.vecs = new Set(); + const body = s.body.filter((st: any) => st.kind !== 'Declare'); + const scanVec = (st: any) => { + for (const e of exprsOf(st)) walkExpr(e, (x: any) => { + if (x.kind === 'CallExpr' && (x.builtin === 'DOT' || x.builtin === 'NORM_OF')) for (const a of x.args) if (a.kind === 'Name' && this.paramNames.has(up(a.name))) this.vecs.add(up(a.name)); + return undefined; + }); + }; + eachStatement(s.body, scanVec); + + this.noSub = !this.hint(sub.name) && String(sub.name).split('_').filter(Boolean).length === 1; + const scope: Scope = { names: new Map(), loops: new Map(), letters: new Set() }; + for (const p of params) scope.names.set(up(p), { s: this.spell(p), p: ATOM }); + const callee = spellFunction(sub.name, params, this.hint(sub.name), (p) => this.spell(p)); + const result = this.block(body, scope, sub, true); + this.written.set(up(sub.name), { call: callee.call, lhs: callee.lhs, params }); + const out: Array<{ text: string; from: number | null }> = []; + if (result.kind === 'value') { + out.push({ text: `${callee.lhs} = ${result.t.s}`, from: s.line }); + } else { + out.push({ text: `${callee.lhs} = \\begin{cases}`, from: s.line }); + result.arms.forEach((a, i) => { + const last = i === result.arms.length - 1; + out.push({ text: ` ${a.value.s} & ${a.cond ?? '\\text{otherwise}'}${last ? '' : ' \\\\'}`, from: a.line }); + }); + out.push({ text: '\\end{cases}', from: s.line }); + } + return out; + } + + /** + * The value a block of steps comes to. Steps before the RETURN give names to values (substituted where + * they are used) or run an accumulator loop (a sum or product). The block must end by returning, either + * one value or an IF chain whose every branch returns. + */ + block(body: any[], scope: Scope, sub: any, top: boolean): { kind: 'value'; t: T } | { kind: 'cases'; arms: Array<{ value: T; cond: string | null; line: number }> } { + const names = new Map(scope.names); + const local: Scope = { names, loops: scope.loops, letters: scope.letters }; + const assigned = new Set<string>(); + for (let i = 0; i < body.length; i++) { + const st = body[i]; + this.curLine = st.line ?? this.curLine; + switch (st.kind) { + case 'Declare': break; + case 'Assign': { + const nxt = body[i + 1]; + if (st.target.kind !== 'Name') this.fail('a change to part of a value', st.line); + const n = up(st.target.name); + if (this.paramNames.has(n) || scope.loops.has(n)) this.fail(`a change to the input ${st.target.name} (an equation does not change its inputs)`, st.line); + if (assigned.has(n)) this.fail(`${st.target.name} set twice (an equation names a value once)`, st.line); + assigned.add(n); + if (nxt && (nxt.kind === 'For' || nxt.kind === 'ForEach') && this.accumulates(nxt, n)) { + const init = st.value; + const lit = init.kind === 'Num' ? Number(init.value) : NaN; + const op = this.accumulatorOp(nxt, n); + if (!((op === '+' && lit === 0) || (op === '*' && lit === 1))) this.fail(`${st.target.name} starting at ${init.kind === 'Num' ? init.value : 'a value that is not 0 (sum) or 1 (product)'}`, st.line); + names.set(n, this.bigOp(nxt, n, op!, local, sub)); + i++; + break; + } + names.set(n, this.ex(st.value, local)); + break; + } + case 'Return': { + if (!st.value) this.fail('a RETURN with no value', st.line); + if (i !== body.length - 1) this.fail('steps after a RETURN', body[i + 1].line); + return { kind: 'value', t: this.ex(st.value, local) }; + } + case 'If': { + if (!top) this.fail('a nested IF (a piecewise definition has one level of cases)', st.line); + const arms: Array<{ value: T; cond: string | null; line: number }> = []; + const rest = body.slice(i + 1); + for (const b of st.branches) { + const r = this.block(b.body, local, sub, false); + if (r.kind !== 'value') this.fail('a nested IF', b.line); + arms.push({ value: r.t, cond: this.cond(b.cond, local), line: b.line }); + } + let other: any[] | null = null; + if (st.elseBody) { if (rest.length) this.fail('steps after an IF that already has an ELSE', rest[0].line); other = st.elseBody; } + else if (rest.length) other = rest; + if (!other) this.fail('an IF with no ELSE and no RETURN after it (a piecewise definition has a value for every input)', st.line); + const r = this.block(other, local, sub, false); + if (r.kind !== 'value') this.fail('a nested IF', st.line); + arms.push({ value: r.t, cond: null, line: st.elseLine ?? other[0]?.line ?? st.line }); + return { kind: 'cases', arms }; + } + case 'For': case 'ForEach': + this.fail(`a loop that is not adding up or multiplying one value (${st.kind === 'For' ? 'FOR' : 'FOR EACH'})`, st.line); + break; + case 'While': this.fail('a WHILE loop', st.line); break; + case 'Repeat': this.fail('a REPEAT loop', st.line); break; + case 'Output': this.fail('OUTPUT (an equation has a value, not output)', st.line); break; + case 'Input': this.fail('INPUT', st.line); break; + case 'Case': this.fail('a CASE (write the IF chain, which becomes the cases)', st.line); break; + case 'Call': case 'BuiltinCall': this.fail('a step that only calls (an equation has a value)', st.line); break; + case 'Assert': this.fail('an ASSERT', st.line); break; + default: this.fail(`a ${st.kind} step`, st.line); + } + } + return this.fail('a function that does not end by returning a value', this.curLine); + } + + /** Is this loop `V <- V op t` and nothing that disturbs V? */ + accumulates(loop: any, v: string): boolean { return this.accumulatorOp(loop, v) !== null; } + accumulatorOp(loop: any, v: string): '+' | '*' | null { + if (loop.parallel) return null; + const body = loop.body.filter((s: any) => s.kind !== 'Declare'); + const last = body[body.length - 1]; + if (!last || last.kind !== 'Assign' || last.target.kind !== 'Name' || up(last.target.name) !== v) return null; + const x = last.value; + if (x.kind !== 'Binary' || !(x.op === '+' || x.op === '*') || x.left.kind !== 'Name' || up(x.left.name) !== v) return null; + return x.op; + } + + /** A sum or product: the loop, written with its bounds. */ + bigOp(loop: any, v: string, op: '+' | '*', scope: Scope, sub: any): T { + this.curLine = loop.line; + if (loop.kind === 'For' && loop.step) this.fail('a loop with a STEP', loop.line); + const name = up(loop.var); + if (this.paramNames.has(name) || scope.loops.has(name) || scope.names.has(name)) this.fail(`the loop counter ${loop.var}, which is already a name here`, loop.line); + const letter = this.indexLetter(loop.var); + if (!letter) this.fail(`the loop counter ${loop.var} (matho writes it as a single letter)`, loop.line); + if (scope.letters.has(letter)) this.fail(`two loops that both count with ${loop.var}`, loop.line); + const inner: Scope = { names: new Map(scope.names), loops: new Map(scope.loops), letters: new Set(scope.letters) }; + let under: string; + let shifted = false; + let lo: any = null; + if (loop.kind === 'ForEach') { + const it = loop.iterable; + if (it.kind !== 'Name' || !this.paramNames.has(up(it.name)) || !['LIST'].includes(it.type?.k)) this.fail('FOR EACH over anything but an input list', loop.line); + under = `_{${letter} \\in ${this.ex(it, scope).s}}`; + inner.names.set(name, { s: letter, p: ATOM }); + } else { + lo = loop.start; + const loT = this.ex(lo, scope); + const hiS = this.upper(loop.stop, scope); + const lit1 = lo.kind === 'Num' && Number(lo.value) === 1 && !lo.float; + const bare = hiS.bare; // LENGTH(L) with no stated bounds + if (bare) { + if (!(lo.kind === 'Num' && Number(lo.value) === 0)) this.fail('a loop over LENGTH that does not start at 0', loop.line); + under = `_{${letter}}`; + } else { + under = `_{${letter}=${loT.s}}^{${hiS.t.s}}`; + shifted = lit1 && (hiS.kind === 'var' || hiS.kind === 'bin'); + } + inner.names.set(name, { s: letter, p: ATOM }); + inner.loops.set(name, { letter, shifted, lo }); + if (bare) { + // the reader runs the loop over the first list the body indexes by this letter; it has to be the one named + inner.loops.get(name)!.lo = { bareList: hiS.list }; + } + } + inner.letters.add(letter); + const bodyAll = loop.body.filter((s: any) => s.kind !== 'Declare'); + const res = this.block([...bodyAll.slice(0, -1), { kind: 'Return', value: bodyAll[bodyAll.length - 1].value.right, line: bodyAll[bodyAll.length - 1].line }], inner, sub, false); + if (res.kind !== 'value') this.fail('a loop whose step chooses', loop.line); + if (loop.kind === 'For') { + const l = inner.loops.get(name)!; + if (l.lo?.bareList && this.firstIndexed(res.t.s, letter) !== l.lo.bareList) this.fail(`a loop over LENGTH(${l.lo.bareList}) whose body does not index ${l.lo.bareList} first`, loop.line); + } + const cmd = op === '+' ? '\\sum' : '\\prod'; + const needs = op === '+' ? res.t.p < MUL && res.t.p !== BIG : res.t.p < MUL && res.t.p !== BIG; + return { s: `${cmd}${under} ${needs ? `(${res.t.s})` : res.t.s}`, p: BIG }; + } + + indexLetter(v: string): string | null { + // the reader adds Val to a one-letter name that is a reserved word (K -> KVal); undo that + const n = /^[A-Za-z]Val$/.test(String(v)) ? String(v)[0] : String(v); + if (/^[A-Za-z]$/.test(n)) return n.toLowerCase(); + if (GREEK_LOWER[up(n)]) return `\\${GREEK_LOWER[up(n)]}`; + return null; + } + + /** The first list a body subscripts by `letter`, as the reader will find it: left to right in the LaTeX. */ + firstIndexed(tex: string, letter: string): string | null { + const esc = letter.replace(/\\/g, '\\\\'); + const m = new RegExp(`(\\\\text\\{[^}]*\\}|\\\\[A-Za-z]+(?:\\{[^}]*\\})?|[A-Za-z])_(?:\\{${esc}[^}]*\\}|${esc})`).exec(tex); + if (!m) return null; + const tok = m[1]; + for (const p of this.paramNames) if (spellVar(p, this.hints[p], this.vecs.has(p), this.noSub) === tok) return p; + return null; + } + + /** The inclusive upper bound a FOR ... TO (exclusive) comes from: N + 1 -> N, 5 -> 4, anything else -> that - 1. */ + upper(stop: any, scope: Scope): { t: T; kind: string; bare?: boolean; list?: string } { + if (stop.kind === 'CallExpr' && stop.builtin === 'LENGTH' && stop.args[0].kind === 'Name' && this.paramNames.has(up(stop.args[0].name))) { + return { t: { s: '', p: ATOM }, kind: 'length', bare: true, list: up(stop.args[0].name) }; + } + if (stop.kind === 'Binary' && stop.op === '+' && stop.right.kind === 'Num' && Number(stop.right.value) === 1) { + const t = this.ex(stop.left, scope); + return { t, kind: stop.left.kind === 'Name' ? 'var' : stop.left.kind === 'Binary' ? 'bin' : 'other' }; + } + if (stop.kind === 'Num' && Number.isInteger(Number(stop.value)) && Number(stop.value) >= 1) return { t: { s: String(Number(stop.value) - 1), p: ATOM }, kind: 'num' }; + const t = this.ex(stop, scope); + return { t: { s: `${wrap(t, ADD)}-1`, p: ADD }, kind: 'bin-' }; + } + + /* ---------- conditions ---------- */ + cond(e: any, scope: Scope): string { + const x = this.condPart(e, scope); + return x.s; + } + condPart(e: any, scope: Scope): { s: string; or: boolean } { + if (e.kind === 'Binary' && e.op === 'AND') { + const l = this.condPart(e.left, scope); const r = this.condPart(e.right, scope); + if (l.or || r.or) this.fail('an AND of an OR (matho conditions have no parentheses)', e.line ?? this.curLine); + return { s: `${l.s} \\land ${r.s}`, or: false }; + } + if (e.kind === 'Binary' && e.op === 'OR') { + const l = this.condPart(e.left, scope); const r = this.condPart(e.right, scope); + return { s: `${l.s} \\lor ${r.s}`, or: true }; + } + if (e.kind === 'Binary' && CMP_TEX[e.op]) { + if (e.left.type?.k === 'BOOL' || e.left.type?.k === 'STRING' || e.left.type?.k === 'CHAR') this.fail('a comparison of something that is not a number', e.line ?? this.curLine); + return { s: `${this.ex(e.left, scope).s} ${CMP_TEX[e.op]} ${this.ex(e.right, scope).s}`, or: false }; + } + if (e.kind === 'Chain') { + if (e.ops.length !== 2 || e.ops.some((o: string) => !CMP_TEX[o] || o === '==' || o === '!=')) this.fail('a chain of comparisons that matho cannot write', e.line ?? this.curLine); + const [a, b, c] = e.operands.map((o: any) => this.ex(o, scope).s); + return { s: `${a} ${CMP_TEX[e.ops[0]]} ${b} ${CMP_TEX[e.ops[1]]} ${c}`, or: false }; + } + return this.fail('a condition that is not a comparison of numbers (NOT, a yes-or-no value, or a test of membership)', e.line ?? this.curLine); + } + + /* ---------- expressions ---------- */ + ex(e: any, scope: Scope): T { + switch (e.kind) { + case 'Num': return { s: String(e.float && Number.isInteger(Number(e.value)) && !/[.e]/.test(String(e.value)) ? `${e.value}.0` : e.value), p: ATOM }; + case 'Name': { + const t = scope.names.get(up(e.name)); + if (!t) this.fail(`the name ${e.name}, which is not an input or a value named above`, e.line ?? this.curLine); + return t; + } + case 'Unary': { + if (e.op !== '-') this.fail(`the operator ${e.op}`, e.line ?? this.curLine); + const a = this.ex(e.arg, scope); + return { s: `-${wrap(a, POW)}`, p: NEG }; + } + case 'Binary': return this.binary(e, scope); + case 'Index': return this.index(e, scope); + case 'CallExpr': return this.call(e, scope); + default: return this.fail(`a ${e.kind === 'Str' ? 'text' : e.kind === 'Bool' ? 'yes-or-no' : e.kind} value`, e.line ?? this.curLine); + } + } + + binary(e: any, scope: Scope): T { + const l = this.ex(e.left, scope); const r = this.ex(e.right, scope); + switch (e.op) { + case '+': case '-': return { s: `${wrap(l, ADD)} ${e.op} ${wrap(r, ADD + 1)}`, p: ADD }; + case '*': return { s: `${wrap(l, MUL)} \\cdot ${wrap(r, MUL + 1)}`, p: MUL }; + case '/': return { s: `\\frac{${l.s}}{${r.s}}`, p: ATOM }; + case 'MOD': return { s: `${wrap(l, MUL)} \\bmod ${wrap(r, MUL + 1)}`, p: MUL }; + case '^': return { s: `${wrap(l, ATOM)}^{${r.s}}`, p: POW }; + case 'DIV': return this.fail('DIV (whole-number division)', e.line ?? this.curLine); + default: return this.fail(`the operator ${e.op}`, e.line ?? this.curLine); + } + } + + /** L[I] is l_i; with a sum that starts at 1, L[I - 1] is l_i, since the reader shifts the subscript. */ + index(e: any, scope: Scope): T { + const line = e.line ?? this.curLine; + // A[I][J] is an index of an index; read it as one two-index position + if (e.target.kind === 'Index' && e.target.indices.length === 1 && e.indices.length === 1) e = { ...e, target: e.target.target, indices: [e.target.indices[0], e.indices[0]] }; + if (e.target.kind !== 'Name' || !this.paramNames.has(up(e.target.name))) this.fail('an index into something that is not an input list', line); + const list = this.ex(e.target, scope); + const loopOf = (x: any) => (x.kind === 'Name' ? scope.loops.get(up(x.name)) : undefined); + if (e.indices.length === 2) { + // A[I][J] is A_{ij}, with both counters plain letters of loops that start where the reader's sums do + const [a, b] = e.indices.map(loopOf); + if (!a || !b || e.indices.some((x: any) => x.kind !== 'Name')) this.fail('a two-dimensional position that is not two loop counters', line); + if (a.shifted || b.shifted || a.letter.length !== 1 || b.letter.length !== 1) this.fail('a two-dimensional position in a loop that starts at 1', line); + return { s: `${list.s}_{${a.letter}${b.letter}}`, p: ATOM }; + } + if (e.indices.length !== 1) this.fail('a position with more than two indices', line); + const ix = e.indices[0]; + const sub = (s: string): T => ({ s: `${list.s}_{${s}}`, p: ATOM }); + // L[I] + if (ix.kind === 'Name') { + const lp = loopOf(ix); + if (!lp) this.fail(`the position ${ix.name}, which is not a loop counter (matho subscripts are loop letters)`, line); + if (lp.shifted) this.fail(`L[${ix.name}] in a loop that starts at 1 (matho counts a sum from 1 as positions 0, 1, ...; write L[${ix.name} - 1])`, line); + return sub(lp.letter); + } + if (ix.kind === 'Binary' && (ix.op === '+' || ix.op === '-') && ix.left.kind === 'Name' && ix.right.kind === 'Num' && Number.isInteger(Number(ix.right.value))) { + const lp = loopOf(ix.left); + if (!lp) this.fail('a position that is not built from a loop counter', line); + const k = Number(ix.right.value); + if (lp.shifted) { + if (ix.op === '-' && k === 1) return sub(lp.letter); + this.fail(`a position other than ${ix.left.name} - 1 in a loop that starts at 1`, line); + } + return sub(`${lp.letter}${ix.op}${k}`); + } + return this.fail('a position that is not a loop counter plus or minus a number', line); + } + + call(e: any, scope: Scope): T { + const line = e.line ?? this.curLine; + const args = () => e.args.map((a: any) => this.ex(a, scope)); + if (!e.builtin) { + const callee = this.written.get(up(e.name)); + if (!callee) this.fail(`a call of ${e.name}, which is not a function written above it`, line); + const a = args(); + if (callee.call) return { s: `${callee.lhs.replace(/\(.*$/, '')}(${a.map((x: T) => x.s).join(', ')})`, p: ATOM }; + // a function written bare is referred to by its name alone, and takes this equation's own inputs + const same = e.args.length === callee.params.length && e.args.every((x: any, i: number) => x.kind === 'Name' && up(x.name) === up(callee.params[i])); + if (!same) this.fail(`a call of ${e.name} with other values than its own inputs (matho refers to ${e.name} by name only when it is called with the same inputs)`, line); + return { s: callee.lhs, p: ATOM }; + } + const a = args(); + switch (e.builtin) { + case 'SQRT': return { s: `\\sqrt{${a[0].s}}`, p: ATOM }; + case 'ABS': return { s: `\\left|${a[0].s}\\right|`, p: ATOM }; + case 'FLOOR': return { s: `\\lfloor ${a[0].s} \\rfloor`, p: ATOM }; + case 'CEIL': return { s: `\\lceil ${a[0].s} \\rceil`, p: ATOM }; + case 'CHOOSE': return { s: `\\binom{${a[0].s}}{${a[1].s}}`, p: ATOM }; + case 'PI': return { s: '\\pi', p: ATOM }; + case 'LOG2': return { s: `\\log_{2}(${a[0].s})`, p: ATOM }; + case 'MAX': case 'MIN': return { s: `\\${e.builtin.toLowerCase()}(${a.map((x: T) => x.s).join(', ')})`, p: ATOM }; + case 'DOT': return { s: `${a[0].s} \\cdot ${a[1].s}`, p: MUL }; + case 'NORM_OF': return { s: `\\lVert ${a[0].s} \\rVert`, p: ATOM }; + default: + if (ONE_ARG[e.builtin] && a.length === 1) return { s: `${ONE_ARG[e.builtin]}(${a[0].s})`, p: ATOM }; + return this.fail(`the built-in ${e.builtin}`, line); + } + } +} + +/** Every expression a statement holds directly. */ +function exprsOf(st: any): any[] { + switch (st.kind) { + case 'Assign': return [st.value, ...(st.target?.kind === 'Index' ? [st.target] : [])]; + case 'Return': return st.value ? [st.value] : []; + case 'If': return st.branches.map((b: any) => b.cond); + case 'For': return [st.start, st.stop, ...(st.step ? [st.step] : [])]; + case 'ForEach': return [st.iterable]; + default: return []; + } +} + +function eachStatement(body: any[], f: (s: any) => void): void { + for (const st of body) { + f(st); + if (st.kind === 'If') { st.branches.forEach((b: any) => eachStatement(b.body, f)); if (st.elseBody) eachStatement(st.elseBody, f); } + else if (st.body) eachStatement(st.body, f); + } +} + +/* ---------- the check: what was written is lowered again and held to the pseudo ---------- */ + +type PType = any; +const numeric = (t: PType): string => (t?.k === 'LIST' || t?.k === 'ARRAY' ? `LIST<${numeric(t.of)}>` : t?.k === 'INTEGER' || t?.k === 'FLOAT' ? 'NUM' : String(t?.k)); + +/** Inputs for the sample runs: whole numbers small enough to index a list of four, decimals, and lists. */ +function samples(types: PType[]): string[][] { + const row = (ints: boolean, v: number[]) => `[${v.map((x) => (ints ? String(x) : x.toFixed(2).replace(/0$/, ''))).join(', ')}]`; + const make = (ints: number[][], floats: number[][], k: number) => (t: PType, i: number): string => { + const lists = (x: PType) => (x.of?.k === 'INTEGER' ? ints : floats); + if (t.k === 'INTEGER') return String([3, 2, 4, 1][(i + k) % 4]); + if (t.k === 'FLOAT') return String([2.5, 1.5, 3.25, 0.5][(i + k) % 4]); + if (t.of?.k === 'LIST') { const f = lists(t.of); return `[${[0, 1, 2, 3].map((r) => row(t.of.of?.k === 'INTEGER', f[(r + k) % f.length])).join(', ')}]`; } + return row(t.of?.k === 'INTEGER', lists(t)[k % lists(t).length]); + }; + const ints = [[4, 1, 3, 2], [1, 5, 2, 6], [3, 3, 3, 3]]; + const floats = [[2.5, 1, 4, 3.5], [0.5, 1.5, 2, 7], [1, 1, 1, 1]]; + return [0, 1, 2].map((k) => types.map(make(ints, floats, k))); +} + +function sameValue(a: string, b: string): boolean { + if (a === b) return true; + const x = Number(a); const y = Number(b); + if (Number.isNaN(x) || Number.isNaN(y)) return false; + return Math.abs(x - y) <= 1e-9 * Math.max(1, Math.abs(x), Math.abs(y)); +} + +/** The quantifier positions one sample run of the read-back may visit (docs/limits.md). */ +const READ_BACK_VISITS = 100000; + +function readsBack(latex: string, studied: any, map: LineMap): { ok: true } | { unsupported: Diagnostic[] } { + const program = studied.program; + const why = (message: string, line = 1): { unsupported: Diagnostic[] } => ({ unsupported: [{ lang: 'matho', severity: 'warning', id: 'matho-no-form', line, message: `matho cannot say this as equations the way the reader needs it (${message}), so the matho pane keeps its last good text.` }] }); + const back = parseMatho(latex); + const errs = back.diagnostics.filter((d) => d.severity === 'error'); + if (errs.length || !back.ok) return why(errs[0]?.message.replace(/\s+/g, ' ').slice(0, 160) ?? 'the equations do not read', map[(errs[0]?.line ?? 1) - 1] ?? 1); + const wanted = [...program.subs.values()] as any[]; + const defs = back.program!.defs; + if (defs.length !== wanted.length) return why('the reader found a different number of functions'); + const st = studyOf(back.program!.pseudo); + if (!st.ok) return why('the pseudo the equations become does not read'); + const lowered = st.program.program.subs; + for (let i = 0; i < wanted.length; i++) { + const w = wanted[i]; const d = defs[i]; + if (up(d.name) !== up(w.name)) return why(`the function ${w.name} comes back as ${d.name}`, w.node.line); + if (d.params.length !== w.params.length) return why(`${w.name} comes back with ${d.params.length} inputs instead of ${w.params.length}`, w.node.line); + for (let j = 0; j < w.params.length; j++) { + const have = lowered.get(up(d.name))?.params[j]?.type; + const got = numeric(have); const want = numeric(w.params[j].type); + if (got !== want) return why(`the input ${w.params[j].name} of ${w.name} comes back as ${have?.k === 'LIST' ? 'a list' : 'a number'}; the reader decides what each input is from how the equation uses it`, w.node.line); + } + } + // the same values: run each function with sample inputs, in the pseudo and in what the equations became. Each run is + // allowed READ_BACK_VISITS quantifier positions: an ENSURE of 30 nested FORALLs, checked on every run, took 2 seconds + // of every edit at the run's own allowance (100 a step), and two programs that are the same stop at the same place + const orig = { program: studied, diagnostics: [], ok: true }; + for (const w of wanted) { + const types = w.params.map((p: any) => p.type); + for (const args of samples(types)) { + const call = `${w.name}(${args.join(', ')})`; + const run = (p: any): string => { + try { const r = runPseudo(p, { call, input: [], maxSteps: 50000, maxVisits: READ_BACK_VISITS }); return r.error ? `error:${r.error.kind}` : String((r.raw as any).result); } catch (e: any) { return `cannot run: ${String(e.message).slice(0, 80)}`; } + }; + const va = run(orig); const vb = run(st); + if (!sameValue(va, vb)) return why(`${call} gives ${va} in the pseudo and ${vb} as equations`, w.node.line); + } + } + return { ok: true }; +} + +/** + * Generate matho from pseudo. `unsupported` when the pseudo has errors, when part of it is not an equation + * (with the construct and its line), or when what was written does not read back as the same functions. + */ +export function pseudoToMatho(pseudo: string, hints?: { names?: NameMap }): MathoResult | { unsupported: Diagnostic[] } { + const st: any = study(pseudo, { coach: false }); + if (!st.ok || !st.program) { + const errors = (st.diags as any[]).filter((d) => d.severity === 'error').map((d) => toDiagnostic(d)); + return { unsupported: errors.length ? errors : [{ lang: 'pseudo', severity: 'error', id: 'matho-no-program', line: 1, message: 'The pseudo does not read yet, so there is nothing to write as equations.' }] }; + } + const program = st.program; + if (!program.subs.size && !program.main.body.length) return { unsupported: [{ lang: 'matho', severity: 'warning', id: 'matho-no-form', line: 1, message: 'There is no function to write as an equation.' }] }; + const gen = new Gen(program, hints?.names ?? {}); + const result = gen.run(); + if (!result) return { unsupported: gen.problems }; + const checked = readsBack(result.source, st, result.map); + if ('unsupported' in checked) return checked; + return result; +} diff --git a/src/sync/pseudo-to-mojo.ts b/src/sync/pseudo-to-mojo.ts new file mode 100644 index 0000000..5d60a89 --- /dev/null +++ b/src/sync/pseudo-to-mojo.ts @@ -0,0 +1,1552 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * Pseudo to Mojo: the one translator in the sync model that only goes one way. + * + * This is an app-owned port of the Mojo half of src/engines/pseudo/translate.js + * (the engine file is left untouched; this fork is where the app's Mojo is + * decided). What changed from the engine's version: + * + * - it writes `def main()` itself, from the program's own lines or from the + * call the student is running (COUNT_BASE('A', "GATTACA") becomes + * print(count_base("A", "GATTACA"))); + * - `raises` is inferred, transitively, so the result passes the checker; + * - imports come from the harvested standard-library index; + * - it never writes a construct the interpreter declines when a verified + * spelling exists (a fixed-length ARRAY is an append loop, not List(length=)); + * - it never invents Mojo. What has no faithful spelling (a statistic the + * standard library lacks, a blank to fill in) becomes + * `raise Error("not translated: ...")` with a diagnostic, never a value + * that looks right and is not; + * - FORALL, EXISTS, SUM, set-builder, KEYS, SORTED, SPLIT, FILL and the + * other built-ins with no one-line spelling become a short loop written + * just before the statement that uses them. + * + * What it writes is held to the pinned compiler, not only to core's own + * interpreter: `pixi run mojo-local` builds the Mojo made from every exercise, + * regression program and library routine and compares what it prints with the + * pseudo run, and `pixi run mojo-fuzz` does the same for generated programs. + * Several spellings below exist because the compiler said so there: a list + * literal is written in brackets where a declared type receives it (`List[T](1, + * 2)` is not a constructor), a whole-number literal left of a Float64 in a + * comparison is written as a float, ROUND spells out its halves (Mojo's round() + * goes to the even neighbor), and -0.0 keeps its sign. + * + * The translation stays literal on purpose (`var n: Int` then `n = ...`, + * `total = total + 1`, an index loop over codepoints). Idiomatic rewrites are + * left to the optional Mojo helper under src/mojo/, so a student can see the + * difference. Nothing here may import that helper. + */ +import { studyPseudo as study } from '../services/pseudo-study.js'; +import { sourceOf, parseExpression } from '../engines/pseudo/parse.js'; +import { walkStatements, walkExpr, statementExprs, rootName, childBlocks } from '../engines/pseudo/analyze.js'; +import { toDiagnostic } from '../services/pseudo.js'; +import { MOJO_STD_EXPORTS } from '../mojo/generated/stdlib.js'; +import { MOJO_STD_STRUCTS, MOJO_STD_TRAITS } from '../mojo/generated/types.js'; +import type { Diagnostic } from '../common/types.js'; +import type { LineMap } from './types.js'; + +import { GENERATED_HEADER } from './header.js'; +export { GENERATED_HEADER }; + +export interface MojoOptions { + /** The call to run, such as COUNT_BASE('A', "GATTACA"); becomes the body of `main` when the pseudo has no lines of its own. */ + call?: string | null; + /** Internal: write the definitions only, no imports and no main (a library routine). */ + defsOnly?: boolean; + /** Internal: the library routines' Mojo, written before the program's own definitions. */ + libraryBlocks?: Array<{ id: string; lines: string[]; imports: string[] }>; +} + +export interface MojoResult { source: string; map: LineMap; notes: Diagnostic[] } + +/** Every standard-library name the translator can write, with the module it comes from. A test holds each to the harvested index. */ +export const USED_STDLIB: Readonly<Record<string, string>> = { + Set: 'std.collections', sort: 'std.builtin.sort', + sqrt: 'std.math', exp: 'std.math', log: 'std.math', log2: 'std.math', log10: 'std.math', + sin: 'std.math', cos: 'std.math', tan: 'std.math', atan2: 'std.math', pi: 'std.math', + floor: 'std.math', ceil: 'std.math', comb: 'std.math', + asin: 'std.math', acos: 'std.math', atan: 'std.math', sinh: 'std.math', cosh: 'std.math', tanh: 'std.math', + hypot: 'std.math', cbrt: 'std.math', expm1: 'std.math', log1p: 'std.math', trunc: 'std.math', copysign: 'std.math', + fma: 'std.math', isnan: 'std.math', isinf: 'std.math', isfinite: 'std.math', inf: 'std.math', nan: 'std.math', + ulp: 'std.math', nextafter: 'std.math', gcd: 'std.math', lcm: 'std.math', + random_float64: 'std.random', random_si64: 'std.random', +}; +/** Names the translator writes that Mojo provides without an import. */ +export const USED_PRELUDE: readonly string[] = ['Int', 'Float64', 'String', 'Bool', 'List', 'Dict', 'print', 'range', 'len', 'min', 'max', 'abs', 'round', 'input', 'atol', 'atof', 'chr', 'ord', 'Error', + 'Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Float32', 'DType']; + +/** The Mojo type for a pseudo number type: INT8 is Int8, FLOAT32 is Float32, INTEGER is Int, FLOAT and FLOAT64 are Float64. */ +export function mojoNumber(t: any): string | null { + if (!t) return null; + if (t.k === 'INTEGER') return t.bits ? `${t.signed ? 'Int' : 'UInt'}${t.bits}` : 'Int'; + if (t.k === 'FLOAT') return t.bits === 32 ? 'Float32' : 'Float64'; + return null; +} +const isSized = (t: any): boolean => !!t && ((t.k === 'INTEGER' && !!t.bits) || (t.k === 'FLOAT' && t.bits === 32)); + +const up = (s: unknown): string => String(s).toUpperCase(); + +/** GCCount -> gc_count, MaxOf -> max_of, I -> i, COUNT_BASE -> count_base. */ +export function snake(name: string): string { + if (/^[A-Z0-9_]+$/.test(name)) return name.toLowerCase(); + return name + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1_$2') + .replace(/([a-z0-9])([A-Z])/g, '$1_$2') + .toLowerCase(); +} + +/** Words a pseudo name must not become in Mojo: keywords, builtins the translation itself calls, and imported names. */ +const RESERVED = new Set([ + 'def', 'fn', 'struct', 'nonlocal', 'await', 'async', 'trait', 'comptime', 'alias', 'var', 'ref', 'mut', 'out', 'read', 'owned', 'inout', 'deinit', 'raises', 'raise', 'try', 'except', 'finally', 'with', 'as', 'import', 'from', + 'and', 'or', 'not', 'in', 'is', 'if', 'elif', 'else', 'for', 'while', 'break', 'continue', 'return', 'pass', 'del', 'assert', 'class', 'lambda', 'global', 'yield', 'match', 'case', 'self', + 'len', 'min', 'max', 'abs', 'round', 'print', 'range', 'input', 'sum', 'type', 'list', 'str', 'int', 'float', 'set', 'dict', 'chr', 'ord', 'pow', 'sort', 'sorted', 'copy', 'main', + 'atol', 'atof', 'floor', 'ceil', 'sqrt', 'exp', 'log', 'log2', 'log10', 'sin', 'cos', 'tan', 'atan2', 'pi', 'comb', 'true', 'false', 'none', + 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64', 'dtype', + ...Object.keys(USED_STDLIB), +]); + +/** Type and trait names Mojo has without an import, and those the translation imports or declares. A struct may not take one. */ +const MOJO_NAMES = new Set<string>(['Self', 'None', 'True', 'False', ...Object.keys(MOJO_STD_STRUCTS), ...Object.keys(MOJO_STD_TRAITS), ...USED_PRELUDE, ...Object.keys(USED_STDLIB)]); + +const COLLECTIONS = new Set(['LIST', 'ARRAY', 'HASHMAP', 'SET', 'RECORD']); +const isCollection = (t: any): boolean => !!t && COLLECTIONS.has(t.k); + +/** A statement or expression with no faithful Mojo spelling. Caught at the statement, which becomes a visible `raise`. */ +class Gap extends Error { } + +interface Line { text: string; from: number | null } +interface Pending { d: number; text: string } +/** The routine being written, and the innermost loop around the statement (a CONTINUE in a REPEAT tests its UNTIL first). */ +interface Ctx { sub: any | null; owner: string; loop?: any } + +/** An expression made only of numeric literals: Mojo folds these at compile time. */ +function litOnly(e: any): boolean { + if (!e) return false; + if (e.kind === 'Num') return true; + if (e.kind === 'Unary') return e.op !== 'NOT' && litOnly(e.arg); + if (e.kind === 'Binary') return ['+', '-', '*'].includes(e.op) && litOnly(e.left) && litOnly(e.right); + return false; +} +/** A list literal whose items are literals all the way down: its value cannot change between two evaluations. */ +function constantOnly(e: any): boolean { + if (!e) return false; + if (e.kind === 'Num' || e.kind === 'Str' || e.kind === 'Char' || e.kind === 'Bool') return true; + if (e.kind === 'Unary') return e.op === '-' && constantOnly(e.arg); + if (e.kind === 'ListLit') return e.items.every(constantOnly); + return false; +} +/** A number Mojo knows at compile time: literals, PI() and E(), and arithmetic on them. */ +function comptimeOnly(e: any): boolean { + if (!e) return false; + if (e.kind === 'Num') return true; + if (e.kind === 'CallExpr') return e.builtin === 'PI' || e.builtin === 'E'; + if (e.kind === 'Unary') return e.op === '-' && comptimeOnly(e.arg); + if (e.kind === 'Binary') return ['+', '-', '*', '/'].includes(e.op) && comptimeOnly(e.left) && comptimeOnly(e.right); + return false; +} +function hasFloatLit(e: any): boolean { + if (!e) return false; + if (e.kind === 'Num') return !!e.float || /[.e]/.test(String(e.value)); + return hasFloatLit(e.arg) || hasFloatLit(e.left) || hasFloatLit(e.right); +} + +function key(name: string): string { return up(name); } + +/* ------------------------------------------------------------------ */ +/* Which routines raise */ +/* ------------------------------------------------------------------ */ + +/** Does a statement list contain something that raises in Mojo? `raising` is the set of subroutines already known to. */ +function bodyRaises(body: any[], raising: Set<string>): boolean { + let hit = false; + const exprHit = (e: any) => { + walkExpr(e, (x: any) => { + if (hit) return false; + if (x.kind === 'Index' && x.target?.type?.k === 'HASHMAP') hit = true; + if (x.kind === 'CallExpr') { + if (!x.builtin && raising.has(key(x.name))) hit = true; + if (x.builtin === 'TO_INTEGER' || x.builtin === 'TO_FLOAT') hit = true; + } + return undefined; + }); + }; + walkStatements(body, (s: any) => { + if (hit) return false; + if (s.kind === 'Input' || s.kind === 'Assert' || s.kind === 'BlankStmt') hit = true; + if (s.kind === 'Call' && raising.has(key(s.name))) hit = true; + for (const e of statementExprs(s)) exprHit(e); + return undefined; + }); + return hit; +} + +function inferRaising(program: any, seed: Iterable<string>): Set<string> { + const raising = new Set<string>(seed); + let changed = true; + while (changed) { + changed = false; + for (const sub of program.subs.values()) { + const k = key(sub.name); + if (!raising.has(k) && bodyRaises(sub.node.body, raising)) { raising.add(k); changed = true; } + } + } + return raising; +} + +/* ------------------------------------------------------------------ */ +/* The translator */ +/* ------------------------------------------------------------------ */ + +class Gen { + out: Line[] = []; + pending: Pending[] = []; + notes = new Map<string, Diagnostic>(); + gaps: Diagnostic[] = []; + gapOwners = new Set<string>(); + imports = new Set<string>(); + names = new Map<string, string>(); + constNames = new Map<string, string>(); + commentsByLine = new Map<number, any>(); + usedComments = new Set<number>(); + hoistOK = true; + /** The OUTPUT value (or a piece of its & chain) being written: text made from it is printed, not used as text. */ + printed: any = null; + temp = 0; + curLine = 1; + srcLines: string[]; + varParams = new Map<string, boolean[]>(); + /** The type an empty list literal takes from where it is going (an assignment's target, a parameter, a return type). */ + hint: any = null; + /** The expression whose value lands directly in a place with a declared type; a list literal there is written in brackets and takes the place's type. */ + landing: any = null; + /** Per body: the variables handed to a BYREF parameter before anything assigns them. */ + unsetByref = new WeakMap<any[], Set<string>>(); + temps = new Set<string>(); + /** DECLAREs inside a block whose names are used after it, keyed by the routine's top-level statement they are written before. */ + liftedBefore = new Map<any, any[]>(); + lifted = new Set<any>(); + /** DECLAREs of a scalar that a line may read before anything certainly gives it a value: they start at the type's zero. */ + zeroInit = new Set<any>(); + /** Set while translating a call's own text (the argument literals), which has no type annotations. */ + constructor(private program: any, private source: string, private raising: Set<string>, private opts: MojoOptions) { + for (const c of program.comments || []) this.commentsByLine.set(c.line, c); + this.srcLines = source.split('\n'); + for (const sub of [...program.subs.values(), ...(program.libs?.values() ?? [])]) { + const changed = this.changedIn(sub.node.body); + const assigned = this.assignedIn(sub.node.body); + // a parameter handed on to a BYREF parameter is changed too: Mojo passes only a variable it may change as mut + const handed = this.handedByref(sub.node.body); + this.varParams.set(key(sub.name), sub.params.map((p: any) => p.mode !== 'BYREF' && ((changed.has(up(p.name)) && (isCollection(p.type) || p.type?.k === 'ARRAY')) || assigned.has(up(p.name)) || handed.has(up(p.name))))); + } + } + + /* ---------- bookkeeping ---------- */ + /** + * Where a body's names live. A DECLARE in the notation names a variable for the rest of the routine; in Mojo a + * `var` inside an if or a loop ends with that block, so a name declared there and used after it is declared at + * the routine's top level, at its zero, just before the statement that holds it. And Mojo rejects reading a + * variable that is not given a value on every path to the read, where the notation stops only when a run takes + * such a path: a scalar declared without a value whose first use is not a plain assignment at the same level + * starts at its zero. + */ + scopes(body: any[]): void { + const chainOf = new Map<any, any[][]>(); + const topOf = new Map<any, any>(); + const visit = (block: any[], chain: any[][], top: any) => { + for (const s of block) { + const t = top ?? s; + chainOf.set(s, chain); topOf.set(s, t); + for (const b of childBlocks(s)) visit(b, [...chain, b], t); + } + }; + visit(body, [body], null); + const mentioned = (s: any, deep: boolean): Set<string> => { + const out = new Set<string>(); + const one = (x: any) => { + for (const e of statementExprs(x)) walkExpr(e, (y: any) => { if (y.kind === 'Name') out.add(up(y.name)); return undefined; }); + if ((x.kind === 'Assign' || x.kind === 'Input') && x.target) { const r = rootName(x.target); if (r) out.add(up(r)); } + if (x.kind === 'Declare' && x !== s) for (const n of x.names) out.add(up(n.name)); + }; + if (deep) walkStatements([s], (x: any) => { one(x); return undefined; }); else one(s); + return out; + }; + const all = [...chainOf.keys()]; + const uses = new Map<any, Set<string>>(all.map((x) => [x, mentioned(x, false)])); + for (const d of all) { + if (d.kind !== 'Declare') continue; + const chain = chainOf.get(d)!; + const block = chain[chain.length - 1]; + const names = d.names.map((n: any) => up(n.name)); + if (chain.length > 1 && all.some((x) => !chainOf.get(x)!.includes(block) && names.some((n: string) => uses.get(x)!.has(n)))) { + this.lifted.add(d); + const t = topOf.get(d); + this.liftedBefore.set(t, [...(this.liftedBefore.get(t) ?? []), d]); + continue; + } + } + // which scalars a read may find without a value, path by path, as Mojo's own check sees it: a loop's body may not + // run, an IF without ELSE may not, and a branch that ends in RETURN, BREAK or CONTINUE does not go on. A read that + // no path gives a value, and that every run reaches once the declaration is passed, stops every run of the + // notation; Mojo rejecting it says the same, so it is left alone. The same read inside an IF or a loop below the + // declaration may never happen (DECLARE N, then IF B THEN OUTPUT N with B false runs to its end), and Mojo + // rejects it all the same: that variable starts at zero too. `depth` counts the blocks a statement is inside. + const watched = new Map<string, any>(); + for (const d of all) if (d.kind === 'Declare' && !this.lifted.has(d) && !d.init && d.resolved && !isCollection(d.resolved) && d.resolved.k !== 'ARRAY') for (const n of d.names) watched.set(up(n.name), d); + interface Have { must: Set<string>; may: Set<string> } + const declaredAt = new Map<string, number>(); + const read = (n: string, h: Have, depth: number) => { if (watched.has(n) && !h.must.has(n) && (h.may.has(n) || depth > (declaredAt.get(n) ?? 0))) this.zeroInit.add(watched.get(n)); }; + const reads = (x: any, h: Have, depth: number) => { + const target = (x.kind === 'Assign' || x.kind === 'Input') && x.target.kind === 'Name' ? x.target : null; + for (const e of statementExprs(x)) walkExpr(e, (y: any) => { if (y.kind === 'Name' && y !== target) read(up(y.name), h, depth); return undefined; }); + if (x.kind === 'Assign' && x.target.kind !== 'Name') { const r = rootName(x.target); if (r) read(up(r), h, depth); } + }; + const copy = (h: Have): Have => ({ must: new Set(h.must), may: new Set(h.may) }); + const meet = (outs: Array<Have | null>): Have | null => { + const live = outs.filter((o): o is Have => !!o); + if (!live.length) return null; + return { must: new Set([...live[0].must].filter((n) => live.every((o) => o.must.has(n)))), may: new Set(live.flatMap((o) => [...o.may])) }; + }; + const flow = (block: any[], h: Have | null, depth = 0): Have | null => { + for (const x of block) { + if (!h) return null; + reads(x, h, depth); + switch (x.kind) { + case 'Declare': for (const n of x.names) { declaredAt.set(up(n.name), depth); if (x.init) { h.must.add(up(n.name)); h.may.add(up(n.name)); } else { h.must.delete(up(n.name)); h.may.delete(up(n.name)); } } break; + case 'Assign': case 'Input': if (x.target.kind === 'Name') { h.must.add(up(x.target.name)); h.may.add(up(x.target.name)); } break; + case 'If': { const hh = h; h = meet([...x.branches.map((b: any) => flow(b.body, copy(hh), depth + 1)), x.elseBody ? flow(x.elseBody, copy(hh), depth + 1) : copy(hh)]); break; } + case 'Case': { const hh = h; h = meet([...x.arms.map((a: any) => flow(a.body, copy(hh), depth + 1)), x.otherwise ? flow(x.otherwise, copy(hh), depth + 1) : copy(hh)]); break; } + case 'For': case 'ForEach': case 'While': case 'Repeat': { + // a later pass may read what an earlier one assigned, and after the loop any of it may hold a value + const later = [...this.assignedIn(x.body)].filter((n) => !(x.kind === 'For' || x.kind === 'ForEach') || n !== up(x.var)); + const inner = copy(h); for (const n of later) inner.may.add(n); + flow(x.body, inner, depth + 1); + for (const n of later) h.may.add(n); + break; + } + case 'Return': case 'Break': case 'Continue': return null; + default: break; + } + } + return h; + }; + flow(body, { must: new Set(), may: new Set() }); + } + + changedIn(body: any[]): Set<string> { + const out = new Set<string>(); + walkStatements(body, (x: any) => { + if (x.kind === 'BuiltinCall' && x.args[0]) { const r = rootName(x.args[0]); if (r) out.add(up(r)); } + if ((x.kind === 'Assign' || x.kind === 'Input') && x.target.kind !== 'Name') { const r = rootName(x.target); if (r) out.add(up(r)); } + if (x.kind === 'Call') x.args.forEach((a: any, i: number) => { if (x.sub?.params?.[i]?.mode === 'BYREF' && a.kind !== 'Name') { const r = rootName(a); if (r) out.add(up(r)); } }); + return undefined; + }); + return out; + } + + /** Names a body hands, whole, to a BYREF parameter (CALL P(X), with X BYREF in P). */ + handedByref(body: any[]): Set<string> { + const out = new Set<string>(); + walkStatements(body, (x: any) => { + if (x.kind === 'Call') x.args.forEach((a: any, i: number) => { if (x.sub?.params?.[i]?.mode === 'BYREF' && a.kind === 'Name') out.add(up(a.name)); }); + return undefined; + }); + return out; + } + + /** Names a body gives a new value to directly (X <- ..., INPUT X, a FOR counter). */ + assignedIn(body: any[]): Set<string> { + const out = new Set<string>(); + walkStatements(body, (x: any) => { + if ((x.kind === 'Assign' || x.kind === 'Input') && x.target.kind === 'Name') out.add(up(x.target.name)); + if (x.kind === 'For' || x.kind === 'ForEach') out.add(up(x.var)); + return undefined; + }); + return out; + } + + withHint<T>(t: any, fn: () => T, landing: any = null): T { + const saved = this.hint; const savedLanding = this.landing; + this.hint = t; this.landing = landing; + try { return fn(); } finally { this.hint = saved; this.landing = savedLanding; } + } + + /** + * Variables a body hands to a BYREF parameter before any line of it gives them a value (an out-parameter in the + * notation). Mojo passes only an initialized variable as `mut`, so their declarations start at the type's zero. + */ + passedUnset(body: any[]): Set<string> { + const have = this.unsetByref.get(body); + if (have) return have; + const out = new Set<string>(); const assigned = new Set<string>(); + const byrefArgs = (args: any[], sub: any) => args.forEach((a: any, i: number) => { + if (sub?.params?.[i]?.mode === 'BYREF' && a.kind === 'Name' && !assigned.has(up(a.name))) out.add(up(a.name)); + }); + walkStatements(body, (x: any) => { + for (const e of statementExprs(x)) walkExpr(e, (y: any) => { if (y.kind === 'CallExpr' && !y.builtin) byrefArgs(y.args, y.sub); return undefined; }); + if (x.kind === 'Call') byrefArgs(x.args, x.sub); + if ((x.kind === 'Assign' || x.kind === 'Input') && x.target.kind === 'Name') assigned.add(up(x.target.name)); + if (x.kind === 'Declare' && x.init) for (const n of x.names) assigned.add(up(n.name)); + if (x.kind === 'For' || x.kind === 'ForEach') assigned.add(up(x.var)); + return undefined; + }); + this.unsetByref.set(body, out); + return out; + } + + note(id: string, text: string): void { + if (!this.notes.has(id)) this.notes.set(id, { lang: 'mojo', severity: 'note', id: `translate-${id}`, line: this.curLine, message: text }); + } + gap(why: string): never { throw new Gap(why); } + need(name: string): void { this.imports.add(name); } + + name(n: string): string { + const k = up(n); + const have = this.names.get(k); + if (have) return have; + let s = snake(n); + if (RESERVED.has(s)) s = `${s}_`; + // two pseudo names can make one Mojo name (RunTotal and RUN_TOTAL are both run_total): the second gets a number + const taken = new Set(this.names.values()); + if (taken.has(s)) { let i = 2; while (taken.has(`${s}_${i}`)) i++; s = `${s}_${i}`; } + this.names.set(k, s); + return s; + } + /** A constant's Mojo name. Two constants can make one name, as two variables can (WordList2 and WORD_LIST2): the second gets a number. */ + constName(n: string): string { + const k = up(n); + const have = this.constNames.get(k); + if (have) return have; + let s = snake(n).toUpperCase(); + const taken = new Set(this.constNames.values()); + if (taken.has(s)) { let i = 2; while (taken.has(`${s}_${i}`)) i++; s = `${s}_${i}`; } + this.constNames.set(k, s); + return s; + } + /** A record's struct name: its own, unless Mojo already gives that name a meaning (TYPE Dict, TYPE Copyable). */ + structName(n: string): string { return MOJO_NAMES.has(n) ? `${n}_` : n; } + fresh(stem: string): string { const t = `_${stem}${++this.temp}`; this.temps.add(t); return t; } + + flush(indent: number, from: number | null): void { + for (const h of this.pending) this.out.push({ text: `${' '.repeat(indent + h.d)}${h.text}`, from }); + this.pending = []; + } + emit(indent: number, text: string, from: number | null): void { + this.flush(indent, from); + this.out.push({ text: text ? `${' '.repeat(indent)}${text}` : '', from }); + } + /** Write lines just before the statement being translated. `always`: the lines compute a constant, so a condition that is checked again may use them too. */ + hoist(lines: Array<[number, string]>, always = false): void { + if (!this.hoistOK && !always) this.gap('a computed value inside a condition that is checked again each time, or on the right of AND or OR, where the notation computes it only when the left side lets it'); + for (const [d, text] of lines) this.pending.push({ d, text }); + } + + leadingComments(line: number, indent: number): void { + for (const [l, c] of this.commentsByLine) { + if (l >= line || this.usedComments.has(l) || !c.standalone) continue; + this.usedComments.add(l); + this.emit(indent, `# ${c.text}`, l); + } + } + trailing(line: number): string { + const c = this.commentsByLine.get(line); + if (!c || c.standalone || this.usedComments.has(line)) return ''; + this.usedComments.add(line); + return ` # ${c.text}`; + } + + /* ---------- types and defaults ---------- */ + type(t: any): string { + if (!t) this.gap('a value whose type is not known'); + switch (t.k) { + case 'INTEGER': case 'FLOAT': return mojoNumber(t)!; + case 'CHAR': case 'STRING': return 'String'; + case 'BOOL': return 'Bool'; + case 'ARRAY': if (t.dims === 2) return `List[List[${this.type(t.of)}]]`; return `List[${this.type(t.of)}]`; + case 'LIST': return `List[${this.type(t.of)}]`; + case 'HASHMAP': return `Dict[${this.type(t.key)}, ${this.type(t.val)}]`; + case 'SET': this.need('Set'); return `Set[${this.type(t.of)}]`; + case 'RECORD': return this.structName(t.name); + default: return this.gap(`a value of kind ${t.k}`); + } + } + + /** What a declared-but-unset value holds: the notation's collections start empty, a record starts with every field at its default. */ + defaultOf(t: any): string { + switch (t?.k) { + case 'INTEGER': return '0'; + case 'FLOAT': return '0.0'; + case 'BOOL': return 'False'; + case 'CHAR': case 'STRING': return 'String("")'; + case 'LIST': case 'HASHMAP': case 'SET': return `${this.type(t)}()`; + case 'RECORD': return `${this.structName(t.name)}(${t.fieldNames.map((f: string) => this.defaultOf(t.fields.get(up(f)))).join(', ')})`; + default: return this.gap('a value with no default'); + } + } + + /** A value that would share storage with its source, copied: M <- L gives M its own list. */ + copied(e: any, text: string): string { + if (!isCollection(e.type) || !(['Name', 'Field', 'Index'].includes(e.kind) || this.temps.has(text))) return text; + this.note('copy', 'In the notation, assigning or returning a collection gives a separate copy. Mojo spells that .copy(), because Mojo tracks which piece of code owns a value.'); + return `${text}.copy()`; + } + + /** + * A value going where another number type belongs, converted as the notation converts it: an INTEGER into a + * FLOAT becomes Float64(...) (Float32(...) for a FLOAT32), an INTEGER into a sized type becomes Int32(...), a + * FLOAT32 into a FLOAT becomes Float64(...). A literal needs nothing: Mojo gives it the type it lands in. + */ + widened(target: any, e: any, text: string): string { + if (!target || !e?.type || e.kind === 'Num') return text; + const want = mojoNumber(target); const have = mojoNumber(e.type); + if (!want || !have || want === have) return text; + // probes/int-overflow-int8 and int-uint-wrap (wrapping), int-convert-narrow (Int8(300) is 44: the low bits) + if (isSized(target) && target.k === 'INTEGER') this.note('sized', 'In the notation, a value that does not fit a sized whole-number type stops the run. In Mojo, Int8 and the other fixed-width types wrap around silently, and a conversion such as Int8(x) keeps only the low bits.'); + return `${want}(${text})`; + } + + /* ---------- program ---------- */ + run(): MojoResult { + const p = this.program; + this.scopes(p.main.body); + for (const sub of p.subs.values()) this.scopes(sub.node.body); + for (const c of p.constants.values()) { + this.curLine = c.line; + this.leadingComments(c.line, 0); + this.names.set(up(c.name), this.constName(c.name)); + // with its type, a constant is a Float64 or a String to Mojo, as in the notation: untyped, it would be a literal, which Mojo + // folds exactly (RATE + 0.2 == 0.3 with RATE = 0.1) and which lacks some of String's methods (replace) + this.guarded(null, c.line, () => this.emit(0, `comptime ${this.constName(c.name)}${c.type && mojoNumber(c.type) || c.type?.k === 'STRING' || c.type?.k === 'CHAR' || c.type?.k === 'BOOL' ? `: ${this.type(c.type)}` : ''} = ${this.withHint(c.type, () => this.expr(c.node.value), c.node.value)}${this.trailing(c.line)}`, c.line)); + } + for (const d of p.defines.values()) { + this.curLine = d.line; + this.leadingComments(d.line, 0); + this.guarded(null, d.line, () => { + const vals = d.node.values.map((v: any) => this.expr(v)).join(', '); + this.emit(0, `def ${this.name(d.name)}() -> ${this.type(d.type)}:${this.trailing(d.line)}`, d.line); + this.emit(1, `return ${this.type(d.type)}(${vals})`, d.line); + }); + // probes/global-var: "global variables are not supported" + this.note('define', 'A DEFINE set becomes a small function that builds the Set, because Mojo has no global variables; the code calls it where the pseudo names the set.'); + this.emit(0, '', null); + this.emit(0, '', null); + } + for (const t of p.types.values()) { + this.curLine = t.node.line; + this.leadingComments(t.node.line, 0); + this.emit(0, '@fieldwise_init', t.node.line); + this.emit(0, `struct ${this.structName(t.name)}(Copyable, Movable):`, t.node.line); + this.note('struct', 'A record becomes a struct with fields. It declares the traits Copyable and Movable, so its values can be copied and moved as the notation copies records.'); + for (const f of t.fieldNames) this.guarded(null, t.node.line, () => this.emit(1, `var ${this.name(f)}: ${this.type(t.fields.get(up(f)))}`, t.node.line)); + this.emit(0, '', null); + this.emit(0, '', null); + } + for (const b of this.opts.libraryBlocks || []) { + this.emit(0, `# From the Programming+ library: ${b.id} (verified in pplus-library; generated from its pseudo)`, null); + for (const l of b.lines) this.emit(0, l, null); + for (const n of b.imports) this.imports.add(n); + this.emit(0, '', null); + this.emit(0, '', null); + } + for (const sub of p.subs.values()) { this.subroutine(sub); this.emit(0, '', null); this.emit(0, '', null); } + if (this.opts.defsOnly) return this.assemble(); + this.mainFunction(); + this.leadingComments(Infinity, 0); + return this.assemble(); + } + + assemble(): MojoResult { + const imports = new Map<string, string[]>(); + for (const n of this.imports) { + const mod = USED_STDLIB[n]; + if (!mod) continue; + if (!imports.has(mod)) imports.set(mod, []); + imports.get(mod)!.push(n); + } + const head: Line[] = [{ text: GENERATED_HEADER, from: null }]; + for (const [mod, names] of [...imports].sort((a, b) => a[0].localeCompare(b[0]))) head.push({ text: `from ${mod} import ${names.sort().join(', ')}`, from: null }); + head.push({ text: '', from: null }); + const lines = [...head, ...this.out]; + while (lines.length && !lines[lines.length - 1].text.trim()) lines.pop(); + return { source: lines.map((l) => l.text).join('\n') + '\n', map: lines.map((l) => l.from), notes: [...this.notes.values(), ...this.gaps] }; + } + + /** Run `fn`; when it finds something with no faithful Mojo, write a visible `raise` in its place. */ + guarded(owner: string | null, line: number, fn: () => void, indent = 0): void { + const saved = this.pending.length; + try { fn(); } catch (err) { + if (!(err instanceof Gap)) throw err; + this.pending.length = saved; + const text = (this.srcLines[line - 1] ?? '').trim(); + this.gaps.push({ lang: 'mojo', severity: 'warning', id: 'translate-gap', line, message: `Programming+ could not translate this line into Mojo (${err.message}). The generated code stops here with an error instead of guessing.` }); + this.emit(indent, `# pseudo line ${line}: ${text}`, line); + this.emit(indent, `raise Error("not translated: pseudo line ${line}")`, line); + this.gapOwners.add(owner ?? 'main'); + } + } + + subroutine(sub: any): void { + const s = sub.node; + this.curLine = s.line; + this.leadingComments(s.line, 0); + const k = key(sub.name); + const flags = this.varParams.get(k) || []; + let header = ''; + try { + const params = sub.params.map((p: any, i: number) => { + const n = this.name(p.name); + if (p.mode === 'BYREF') { this.note('mut', 'BYREF becomes the mut argument convention, which lets the function change the caller\'s variable.'); return `mut ${n}: ${this.type(p.type)}`; } + if (flags[i]) { this.note('var-param', 'A parameter the procedure gives a new value to or changes is declared var, so the function owns a copy it may change; for a collection the caller passes .copy().'); return `var ${n}: ${this.type(p.type)}`; } + return `${n}: ${this.type(p.type)}`; + }).join(', '); + const ret = s.kind === 'Function' ? ` -> ${this.type(sub.returns)}` : ''; + header = `def ${this.name(sub.name)}(${params})${this.raising.has(k) ? ' raises' : ''}${ret}:`; + } catch (err) { + if (!(err instanceof Gap)) throw err; + header = `def ${this.name(sub.name)}() raises:`; + this.gaps.push({ lang: 'mojo', severity: 'warning', id: 'translate-gap', line: s.line, message: `Programming+ could not write this routine's signature in Mojo (${err.message}).` }); + this.gapOwners.add(k); + this.emit(0, `${header}${this.trailing(s.line)}`, s.line); + this.emit(1, `raise Error("not translated: pseudo line ${s.line}")`, s.line); + return; + } + this.emit(0, `${header}${this.trailing(s.line)}`, s.line); + for (const r of sub.requires) this.emit(1, `# REQUIRE ${sourceOf(r.expr)}`, r.line); + for (const e of sub.ensures) this.emit(1, `# ENSURE ${sourceOf(e.expr)}`, e.line); + const before = this.out.length; + this.block(s.body, 1, { sub, owner: k }); + if (this.out.length === before) this.emit(1, 'pass', s.line); + } + + /** `def main()`: the program's own lines, then the call the student is running. */ + mainFunction(): void { + const p = this.program; + const body = p.main.body; + const callText = (this.opts.call || '').trim(); + const mainRaises = this.raising.has('MAIN'); + if (!body.length && !callText) return; + this.emit(0, `def main()${mainRaises ? ' raises' : ''}:`, body[0]?.line ?? null); + const before = this.out.length; + if (body.length) this.block(body, 1, { sub: null, owner: 'main' }); + if (callText) this.callLine(callText, 1); + if (this.out.length === before) this.emit(1, 'pass', null); + } + + /** print(f(args)): the arguments are literals in the call's own text, written for the parameter types. */ + callLine(text: string, d: number): void { + this.guarded('main', 1, () => { + const parsed: any = parseExpression(text); + if (parsed.diags.some((x: any) => x.severity === 'error') || parsed.expr.kind !== 'CallExpr') this.gap('the call is not a call'); + const call = parsed.expr; + const sub = this.program.subs.get(up(call.name)); + if (!sub || sub.params.length !== call.args.length) this.gap('the call does not match a routine'); + const flags = this.varParams.get(key(sub.name)) || []; + const args = call.args.map((a: any, i: number) => this.literal(a, sub.params[i].type, flags[i])); + const expr = `${this.name(sub.name)}(${args.join(', ')})`; + if (sub.kind === 'Function') this.emit(d, `print(${expr})`, null); + else this.emit(d, expr, null); + }, d); + } + + /** A literal argument, in the Mojo that fits its parameter. */ + literal(e: any, t: any, owned = false): string { + switch (e.kind) { + case 'Num': { const s = Object.is(e.value, -0) ? '-0.0' : String(e.value); return t?.k === 'FLOAT' && !/[.e]/.test(s) ? `${s}.0` : s; } + case 'Str': case 'Char': return JSON.stringify(e.value); + case 'Bool': return e.value ? 'True' : 'False'; + case 'Unary': if (e.op === '-') return `-${this.literal(e.arg, t)}`; break; + // an argument lands in a parameter, whose type the brackets take + case 'ListLit': return e.items.length ? `[${e.items.map((x: any) => this.literal(x, t?.of)).join(', ')}]` : `${this.type(t)}()`; + case 'SetLit': this.need('Set'); return `${this.type(t)}(${e.items.map((x: any) => this.literal(x, t?.of)).join(', ')})`; + default: break; + } + void owned; + return this.gap(`the argument ${sourceOf(e)}`); + } + + block(body: any[], d: number, ctx: Ctx): void { + const before = this.out.length; + for (const s of body) this.stmt(s, d, ctx); + if (this.out.length === before) this.emit(d, 'pass', null); + } + + stmt(s: any, d: number, ctx: Ctx): void { + this.curLine = s.line ?? this.curLine; + for (const decl of this.liftedBefore.get(s) ?? []) { + this.guarded(ctx.owner, decl.line, () => { + for (const n of decl.names) this.emit(d, `var ${this.name(n.name)}: ${this.type(decl.resolved)} = ${decl.resolved.k === 'ARRAY' ? `${this.type(decl.resolved)}()` : this.defaultOf(decl.resolved)}`, decl.line); + }, d); + // probes/var-block-scope + this.note('lifted', 'A name declared inside an IF, a CASE or a loop is used after it. In Mojo a var declared in a block ends with the block, so the translation declares it before, at its zero, and the DECLARE becomes an assignment.'); + } + this.leadingComments(s.line, d); + this.guarded(ctx.owner, s.line, () => this.stmtBody(s, d, ctx), d); + } + + /* ---------- statements ---------- */ + stmtBody(s: any, d: number, ctx: Ctx): void { + const tr = () => this.trailing(s.line); + switch (s.kind) { + case 'Declare': { + const t = s.resolved; + if (this.lifted.has(s)) { + // declared before the block (see scopes); here the notation starts the variable afresh: a collection empty, a record at its defaults + if (s.init) return; + for (const n of s.names) { + const v = this.name(n.name); + if (t.k === 'ARRAY') this.arrayDecl(v, t, d, s.line, false); + else if (isCollection(t)) this.emit(d, `${v} = ${this.defaultOf(t)}${tr()}`, s.line); + } + return; + } + // DECLARE x : T <- value (one line in the pseudo) is one line in Mojo too: var x: T = value + if (s.init && s.names.length === 1 && t.k !== 'ARRAY' && t.k !== 'RECORD') { + const a = s.init; + const value = this.withHint(a.target.type, () => this.expr(a.value), a.value); + // a list in brackets needs the declared type to become a List; a constructor call says its own + const typed = ['LIST', 'HASHMAP', 'SET'].includes(t.k) && !value.startsWith('[') && !value.startsWith('{') ? '' : `: ${this.type(t)}`; + this.emit(d, `var ${this.name(s.names[0].name)}${typed} = ${this.widened(a.target.type, a.value, this.copied(a.value, value))}${tr()}`, s.line); + a.mergedIntoDeclare = true; + return; + } + for (const n of s.names) { + const v = this.name(n.name); + if (t.k === 'ARRAY') { this.arrayDecl(v, t, d, s.line); continue; } + if (['LIST', 'HASHMAP', 'SET'].includes(t.k) || t.k === 'RECORD') this.emit(d, `var ${v} = ${this.defaultOf(t)}${tr()}`, s.line); + else if (this.passedUnset(ctx.sub ? ctx.sub.node.body : this.program.main.body).has(up(n.name))) { + this.emit(d, `var ${v}: ${this.type(t)} = ${this.defaultOf(t)}${tr()}`, s.line); + // probes/mut-uninitialized: a variable with no value cannot be passed as mut + this.note('out-argument', 'In the notation a BYREF parameter can hand back a value through a variable that holds nothing yet. Mojo passes only a variable that already has a value as mut, so its declaration starts it at zero (or empty text).'); + } else if (this.zeroInit.has(s)) { + this.emit(d, `var ${v}: ${this.type(t)} = ${this.defaultOf(t)}${tr()}`, s.line); + // probes/var-maybe-unset + this.note('maybe-unset', 'Mojo rejects reading a variable that some path leaves without a value, where the notation stops only if a run takes that path. A variable given its value inside an IF or a loop starts at zero (or empty text) in the translation.'); + } else this.emit(d, `var ${v}: ${this.type(t)}${tr()}`, s.line); + if (t.k === 'RECORD') this.note('record-init', 'The translation creates a struct with a value for every field at once (its @fieldwise_init constructor): every field starts at its default, and the pseudo\'s field assignments change them afterwards.'); + } + return; + } + case 'Assign': { + if (s.mergedIntoDeclare) return; + const value = this.withHint(s.target.type, () => this.expr(s.value), s.value); + let text = this.widened(s.target.type, s.value, this.copied(s.value, value)); + // Mojo builds a String, a list or a record straight into the element or field it goes to; when the value is made from + // the same collection (L[0] <- JOIN(L, "-")), that is two uses of it at once, which the compiler rejects. Named first, it is not. + // A list built from itself (L <- SLICE(L, 0, 2)) is the same. + const tt = s.target.type ?? s.value.type; + const root = s.target.kind !== 'Name' ? rootName(s.target) : isCollection(tt) && s.value.kind !== 'Name' ? s.target.name : null; + if (root && (isCollection(tt) || tt?.k === 'STRING') && /[\w\]]\(/.test(text)) { + let reads = false; + walkExpr(s.value, (y: any) => { if (y.kind === 'Name' && up(y.name) === up(root)) reads = true; return undefined; }); + if (reads) { const t = this.fresh('value'); this.emit(d, `var ${t}: ${this.type(tt)} = ${text}`, s.line); text = `${t}^`; } + } + this.emit(d, `${this.lhs(s.target)} = ${text}${tr()}`, s.line); + return; + } + case 'Input': { + const t = s.target.type; + const read = 'input()'; + const conv = t?.k === 'INTEGER' ? `atol(${read})` : t?.k === 'FLOAT' ? `atof(${read})` : t?.k === 'BOOL' ? `${read} == "TRUE"` : read; + this.emit(d, `${this.lhs(s.target)} = ${conv}${tr()}`, s.line); + return; + } + case 'Output': { + const vals = s.values.map((v: any) => this.outputValue(v)); + this.emit(d, `print(${vals.join(', ')}${vals.length > 1 ? ', sep=""' : ''})${tr()}`, s.line); + return; + } + case 'If': { + const conds = s.branches.map((b: any, i: number) => { this.hoistOK = i === 0; try { return this.expr(b.cond); } finally { this.hoistOK = true; } }); + s.branches.forEach((b: any, i: number) => { + this.emit(d, `${i ? 'elif' : 'if'} ${conds[i]}:${i ? this.trailing(b.line) : tr()}`, b.line); + this.block(b.body, d + 1, ctx); + }); + if (s.elseBody) { this.emit(d, 'else:', s.elseLine); this.block(s.elseBody, d + 1, ctx); } + return; + } + case 'Case': { + let subj = this.expr(s.subject); + // the subject is worked out once, before any arm is looked at: one that calls something is named first + let calls = false; + walkExpr(s.subject, (y: any) => { if (y.kind === 'CallExpr') calls = true; return undefined; }); + if (calls) { + const t = this.fresh('subject'); + this.emit(d, `var ${t}: ${this.type(s.subject.type)} = ${subj}`, s.line); + subj = t; + } + // an arm's labels are looked at only when no arm above matched, so nothing of a label is written ahead of the CASE + this.hoistOK = false; + let arms: string[]; + try { + arms = s.arms.map((a: any) => a.labels.map((l: any) => { + if (l.value) return `${subj} == ${this.expr(l.value)}`; + // the subject was written once, above; among FLOATs a whole number is written as one + const float = [l.low, s.subject, l.high].some((x: any) => x?.type?.k === 'FLOAT'); + const as = (x: any, text: string): string => (!float || x?.type?.k !== 'INTEGER' ? text : this.wholeLiteral(x) ? `${text}.0` : `Float64(${text})`); + return `${as(l.low, this.expr(l.low))} <= ${as(s.subject, subj)} < ${as(l.high, this.expr(l.high))}`; + }).join(' or ')); + } finally { this.hoistOK = true; } + s.arms.forEach((a: any, i: number) => { + this.emit(d, `${i ? 'elif' : 'if'} ${arms[i]}:`, a.line); + this.block(a.body, d + 1, ctx); + }); + if (s.otherwise) { this.emit(d, s.arms.length ? 'else:' : 'if True:', s.otherwiseLine); this.block(s.otherwise, d + 1, ctx); } + // Mojo 1.2.0 has match (probes/match-statement); a chain of if and elif keeps the notation's half-open ranges and lists of values as they read + this.note('case', 'CASE becomes a chain of if and elif: each arm is tested in order, and the first that matches runs.'); + return; + } + case 'For': { + const i = this.name(s.var); + const a = this.expr(s.start); const b = this.expr(s.stop); + const r = s.step ? `range(${a}, ${b}, ${this.expr(s.step)})` : a === '0' ? `range(${b})` : `range(${a}, ${b})`; + for (const inv of s.invariants || []) this.emit(d, `# INVARIANT ${sourceOf(inv.expr)}`, inv.line); + if (s.parallel) this.emit(d, '# PARALLEL FOR: the passes are independent; this translation runs them one after another', s.line); + this.emit(d, `for ${i} in ${r}:${tr()}`, s.line); + this.block(s.body, d + 1, { ...ctx, loop: s }); + return; + } + case 'ForEach': { + const x = this.name(s.var); + const it = s.iterable; + const coll = this.expr(it); + // FOR EACH walks a snapshot, so a body that changes the text or the list, or gives it a new value, does not change the walk + const root = rootName(it); + const changes = !!root && (this.changedIn(s.body).has(up(root)) || this.assignedIn(s.body).has(up(root)) || this.handedByref(s.body).has(up(root))); + if (changes) this.note('each-snapshot', 'FOR EACH walks the collection as it was when the loop began. The body changes it, so the translation walks a copy.'); + if (it.type?.k === 'STRING') { + const k = this.fresh(`${x}_pos`); + let text = coll; + if (changes) { text = this.fresh('walk'); this.emit(d, `var ${text} = String(${coll})`, s.line); } + this.emit(d, `for ${k} in range(${text}.count_codepoints()):${tr()}`, s.line); + this.emit(d + 1, `var ${x} = String(${text}[codepoint=${k}])`, s.line); + } else if (it.type?.k === 'HASHMAP' || it.type?.k === 'SET') { + let walk = coll; + if (changes) { walk = this.fresh('walk'); this.emit(d, `var ${walk} = ${coll}.copy()`, s.line); } + this.emit(d, `for ${x} in ${walk}${it.type.k === 'HASHMAP' ? '.keys()' : ''}:${tr()}`, s.line); + this.note('each-order', 'The lab visits SET members and HASHMAP keys smallest first. Mojo visits them in its own order, which the notation never promised; a program whose answer depends on that order is worth rewriting.'); + } else if (it.kind === 'Range') { + this.emit(d, `for ${x} in range(${this.expr(it.lo)}, ${this.expr(it.hi)}):${tr()}`, s.line); + } else { + this.emit(d, `for ${x} in ${changes ? `${coll}.copy()` : coll}:${tr()}`, s.line); + } + this.block(s.body, d + 1, { ...ctx, loop: s }); + return; + } + case 'While': { + this.hoistOK = false; + let cond: string; + try { cond = this.expr(s.cond); } finally { this.hoistOK = true; } + for (const inv of s.invariants || []) this.emit(d, `# INVARIANT ${sourceOf(inv.expr)}`, inv.line); + if (s.decreases) this.emit(d, `# DECREASES ${sourceOf(s.decreases.expr)}`, s.decreases.line); + this.emit(d, `while ${cond}:${tr()}`, s.line); + this.block(s.body, d + 1, { ...ctx, loop: s }); + return; + } + case 'Repeat': { + this.hoistOK = false; + let cond: string; + try { cond = this.expr(s.cond); } finally { this.hoistOK = true; } + this.emit(d, `while True:${tr()}`, s.line); + this.block(s.body, d + 1, { ...ctx, loop: s }); + this.emit(d + 1, `if ${cond}:`, s.endLine); + this.emit(d + 2, 'break', s.endLine); + this.note('repeat', 'REPEAT ... UNTIL has no direct counterpart: it becomes while True, with a break at the end once the UNTIL condition holds.'); + return; + } + case 'Break': this.emit(d, `break${tr()}`, s.line); return; + case 'Continue': { + // in a REPEAT, CONTINUE goes on to the UNTIL test; Mojo's continue would skip the test written at the end of the body + if (ctx.loop?.kind === 'Repeat') { + this.hoistOK = false; + let cond: string; + try { cond = this.expr(ctx.loop.cond); } finally { this.hoistOK = true; } + this.emit(d, `if ${cond}:`, s.line); + this.emit(d + 1, 'break', s.line); + } + this.emit(d, `continue${tr()}`, s.line); + return; + } + case 'Return': { + if (!s.value) { this.emit(d, `return${tr()}`, s.line); return; } + const v = this.copied(s.value, this.withHint(ctx.sub?.returns, () => this.expr(s.value), s.value)); + this.emit(d, `return ${this.widened(ctx.sub?.returns, s.value, v)}${tr()}`, s.line); + return; + } + case 'Call': { + const flags = this.varParams.get(key(s.name)) || []; + // a variable handed BYREF may not also be read by another argument of the same call (CALL P(L, LENGTH(L)) with L + // BYREF): Mojo rejects the two uses at once. The notation copies a by-value argument when the call begins, so the + // translation does that first, into a name of its own. + const byref = new Set(s.args.filter((a: any, i: number) => s.sub?.params?.[i]?.mode === 'BYREF' && a.kind === 'Name').map((a: any) => up(a.name))); + const args = s.args.map((a: any, i: number) => { + const text = this.callArg(a, s.sub?.params?.[i], flags[i]); + if (s.sub?.params?.[i]?.mode === 'BYREF' || !byref.size) return text; + let reads = false; + walkExpr(a, (y: any) => { if (y.kind === 'Name' && byref.has(up(y.name))) reads = true; return undefined; }); + if (!reads) return text; + // already a value of its own, worked out before the call (a SPLIT or SORTED the translation named): nothing to copy + if (this.temps.has(text)) return text; + const t = this.fresh('arg'); + const pt = s.sub?.params?.[i]?.type ?? a.type; + this.emit(d, `var ${t}: ${this.type(pt)} = ${isCollection(a.type) && !text.endsWith('.copy()') && ['Name', 'Field', 'Index'].includes(a.kind) ? `${text}.copy()` : text}`, s.line); + return isCollection(a.type) || a.type?.k === 'ARRAY' || flags[i] ? `${t}^` : t; + }); + this.emit(d, `${this.name(s.name)}(${args.join(', ')})${tr()}`, s.line); + return; + } + case 'BuiltinCall': this.emit(d, `${this.builtinStmt(s)}${tr()}`, s.line); return; + case 'ExprStmt': this.emit(d, `${this.expr(s.expr)}${tr()}`, s.line); return; + case 'Assert': { + const c = this.expr(s.expr); + this.emit(d, `if not (${c}):`, s.line); + this.emit(d + 1, `raise Error("ASSERT failed on pseudo line ${s.line}: ${sourceOf(s.expr).replace(/["\\]/g, '')}")`, s.line); + return; + } + case 'BlankStmt': this.gap(`a blank to fill in: <${s.text}>`); return; + default: return; + } + } + + /** DECLARE A : ARRAY[n] OF T: a list filled to its length, the way the verified programs do it (an append loop, since List(length=) is not run by the interpreter). */ + arrayDecl(v: string, t: any, d: number, line: number, declare = true): void { + const dims = (t.dimExprs || []).map((x: any) => this.expr(x)); + const fill = this.defaultOf(t.of); + const intro = declare ? 'var ' : ''; + if (dims.length === 2) { + const row = this.fresh('row'); + this.emit(d, `${intro}${v} = ${this.type(t)}()`, line); + this.emit(d, `for _ in range(${dims[0]}):`, line); + this.emit(d + 1, `var ${row} = List[${this.type(t.of)}]()`, line); + this.emit(d + 1, `for _ in range(${dims[1]}):`, line); + this.emit(d + 2, `${row}.append(${fill})`, line); + this.emit(d + 1, `${v}.append(${row}^)`, line); + } else { + this.emit(d, `${intro}${v} = ${this.type(t)}()`, line); + this.emit(d, `for _ in range(${dims[0]}):`, line); + this.emit(d + 1, `${v}.append(${fill})`, line); + } + this.note('array', 'A fixed-length ARRAY becomes a list filled to its full length with an append loop, so every cell holds a value.'); + } + + callArg(a: any, param: any, owned: boolean | undefined): string { + const text = this.widened(param?.type, a, this.withHint(param?.type, () => this.expr(a), a)); + return owned ? this.copied(a, text) : text; + } + + builtinStmt(s: any): string { + const elem = s.args[0]?.type?.of; + // what goes into a collection lands in its element type, and a collection or record put there is a copy of its own + const item = (x: any): string => this.copied(x, this.withHint(elem, () => this.expr(x), x)); + const a = this.expr(s.args[0]); + const [b, c] = s.name === 'APPEND' ? [item(s.args[1]), ''] : s.name === 'INSERT' ? [this.expr(s.args[1]), item(s.args[2])] : s.args.slice(1).map((x: any) => this.expr(x)); + switch (s.name) { + case 'APPEND': return `${a}.append(${b})`; + case 'INSERT': return `${a}.insert(${b}, ${c})`; + case 'REMOVE': return `_ = ${a}.pop(${b})`; + case 'ADD': return `${a}.add(${b})`; + // DELETE of a member or key that is not there changes nothing in the notation; Set.remove and Dict.pop(key) would raise + case 'DELETE': + if (s.args[0]?.type?.k === 'SET') return `${a}.discard(${b})`; + return `_ = ${a}.pop(${b}, ${this.defaultOf(s.args[0]?.type?.val)})`; + case 'SORT': this.need('sort'); return `sort(${a})`; + default: return this.gap(`the built-in ${s.name}`); + } + } + + lhs(t: any): string { return this.expr(t, true); } + + outputValue(v: any): string { + if (v.type?.k === 'BOOL') return `("TRUE" if ${this.expr(v)} else "FALSE")`; + this.printed = v; + try { return this.expr(v); } finally { this.printed = null; } + } + + /** + * Text made from a value Mojo writes otherwise than the notation: a FLOAT (6.02e+23 for 6.02e23, 1e+16 for + * 10000000000000000.0, and where two shortest spellings tie, the other one), or a collection of FLOATs, text, BOOLs, + * a map or a set (quotes, True, the order). Printed, it shows the same value; used as text (its LENGTH, a comparison, + * a variable that holds it) it is other text, so outside an OUTPUT it is not translated. + */ + textOf(x: any, printed: boolean, text: string): string { + const differs = (t: any): boolean => !!t && (t.k === 'FLOAT' || ['HASHMAP', 'SET', 'RECORD'].includes(t.k) + || ((t.k === 'LIST' || t.k === 'ARRAY') && (['STRING', 'CHAR', 'BOOL'].includes(t.of?.k) || differs(t.of)))); + if (differs(x.type) && !printed) this.gap('a FLOAT or a collection made into text that is used as text: Mojo spells some of them otherwise (6.02e+23 for the notation\'s 6.02e23)'); + return text; + } + + /* ---------- expressions ---------- */ + /** An iteration source for a quantifier or set builder: a range, or a collection's items. */ + domain(dom: any): string { + return dom.kind === 'Range' ? `range(${this.expr(dom.lo)}, ${this.expr(dom.hi)})` : this.expr(dom); + } + + expr(e: any, isTarget = false): string { + const P = (x: any) => (x && ['Binary', 'Chain', 'Quant', 'Unary'].includes(x.kind) ? `(${this.expr(x)})` : this.expr(x)); + switch (e.kind) { + case 'Num': { if (Object.is(e.value, -0)) return '-0.0'; const t = String(e.value); return e.float && !/[.e]/.test(t) ? `${t}.0` : t; } + case 'Str': case 'Char': return JSON.stringify(e.value); + case 'Bool': return e.value ? 'True' : 'False'; + case 'Blank': return this.gap(`a blank to fill in: <${e.text}>`); + case 'TypeRef': return e.name === 'INTEGER' ? 'Int' : e.name === 'FLOAT' || e.name === 'FLOAT64' ? 'Float64' : e.name === 'FLOAT32' ? 'Float32' : e.name.startsWith('UINT') ? `UInt${e.name.slice(4)}` : `Int${e.name.slice(3)}`; + case 'Name': + if (e.sym?.kind === 'const') return this.constName(e.name); + if (e.sym?.kind === 'define') return `${this.name(e.name)}()`; + if (up(e.name) === 'RESULT' && e.sym?.kind === 'result') return this.gap('RESULT'); + return this.name(e.name); + case 'Unary': return e.op === 'NOT' ? `not ${P(e.arg)}` : `-${P(e.arg)}`; + case 'Binary': { + const map: Record<string, string> = { AND: 'and', OR: 'or', DIV: '//', MOD: '%', '^': '**', '&': '+', IN: 'in' }; + // probes/float-special: 1.0 / 0.0, -1.0 / 0.0 and 0.0 / 0.0 print inf -inf nan; under USE IEEE the notation gives the same + if (e.op === '/' && !this.program.ieee) this.note('divide', 'In the notation, dividing by zero stops the program. In Mojo a Float64 division by zero gives inf, -inf or nan and the program goes on.'); + // probes/int-div-zero and int-mod-zero: an Int // or % by zero gives 0, and the program goes on + if ((e.op === 'DIV' || e.op === 'MOD') && !this.nonzeroLiteral(e.right)) this.note('divide-int', 'In the notation, DIV or MOD by zero stops the program. In Mojo a whole-number // or % by zero gives 0 and the program goes on.'); + if (e.op === '/' && mojoNumber(e.type) === 'Float32') { + const side = (x: any) => (x.kind === 'Num' || mojoNumber(x.type) === 'Float32' ? this.sideExpr(x, e, x === e.left ? 'l' : 'r') : `Float32(${this.expr(x)})`); + return `${side(e.left)} / ${side(e.right)}`; + } + if (e.op === '/') return `Float64(${this.expr(e.left)}) / Float64(${this.expr(e.right)})`; + if (isSized(e.type) && e.type.k === 'INTEGER' && ['+', '-', '*', '^', 'DIV', 'MOD'].includes(e.op)) this.note('sized', 'In the notation, a value that does not fit a sized whole-number type stops the run. In Mojo, Int8 and the other fixed-width types wrap around silently, and a conversion such as Int8(x) keeps only the low bits.'); + if (e.op === 'IN' && e.right.type?.k === 'ARRAY' && e.right.type.dims === 2) return `${this.sideExpr(e.left, e, 'l')} in ${this.flat(e.right, this.expr(e.right))}`; + if (e.op === 'IN' && e.right.kind === 'Range') { + // a range holds whole numbers, so 0.3 IN [0, 12) is FALSE in the notation; `0.0 <= x < 12.0` would say True + if (e.left.type?.k === 'FLOAT') this.gap('a FLOAT tested against a range, which holds whole numbers only'); + const [lo, x, hi] = this.together([e.right.lo, e.left, e.right.hi]); + return `${lo} <= ${x} < ${hi}`; + } + if (e.op === '^' && (e.type?.k === 'FLOAT')) this.powerNote(e.right); + if (e.op === '&') { + // a chain of & that is printed is printed as a whole: its pieces are printed too + const printed = this.printed === e; + const side = (x: any) => { + this.printed = printed && ((x.kind === 'Binary' && x.op === '&') || x.builtin === 'TO_STRING') ? x : null; + try { + if (x.type?.k === 'CHAR' || x.type?.k === 'STRING') return P(x); + if (x.type?.k === 'BOOL') return `String("TRUE" if ${P(x)} else "FALSE")`; + return this.textOf(x, printed, `String(${this.expr(x)})`); + } finally { this.printed = null; } + }; + return `${side(e.left)} + ${side(e.right)}`; + } + if (e.op === 'AND' || e.op === 'OR') { + // the right side is computed only when the left side lets it (`N > 0 AND MEAN_OF(L) > 2`), so nothing of it may be + // written before the line: a loop hoisted from there would run, and could stop, where the notation skips it + const l = this.sideExpr(e.left, e, 'l'); + const saved = this.hoistOK; + this.hoistOK = false; + try { return `${l} ${map[e.op]} ${this.sideExpr(e.right, e, 'r')}`; } finally { this.hoistOK = saved; } + } + let left = this.sideExpr(e.left, e, 'l'); + // Mojo folds arithmetic on float literals exactly (0.1 + 0.2 == 0.3); the notation computes with Float64, so the left side is made one + if (litOnly(e.left) && litOnly(e.right) && (hasFloatLit(e.left) || hasFloatLit(e.right)) && !left.startsWith('Float64(') && ['+', '-', '*', '==', '!=', '<', '<=', '>', '>='].includes(e.op)) left = `Float64(${left})`; + return `${left} ${map[e.op] || e.op} ${this.sideExpr(e.right, e, 'r')}`; + } + case 'Chain': { + // A <= B < C stops at the first comparison that fails, so C is computed only when A <= B holds + const sides = e.ops.includes('IN') ? e.operands.map(P) : this.together(e.operands, 2); + return e.operands.map((_o: any, i: number) => (i ? `${e.ops[i - 1] === 'IN' ? 'in' : e.ops[i - 1]} ${sides[i]}` : sides[i])).join(' '); + } + case 'Index': { + const t = this.expr(e.target); + if (e.target.type?.k === 'STRING' && !isTarget) return `String(${t}[codepoint=${this.expr(e.indices[0])}])`; + if (e.indices.length === 2) return `${t}[${this.expr(e.indices[0])}][${this.expr(e.indices[1])}]`; + return `${t}[${this.expr(e.indices[0])}]`; + } + case 'Field': return `${this.expr(e.target)}.${this.name(e.field)}`; + case 'CallExpr': return this.call(e); + case 'ListLit': { + const t = e.type?.k === 'LIST' && e.type.of?.k !== 'ANY' ? e.type : this.hint?.k === 'LIST' || this.hint?.k === 'ARRAY' ? this.hint : e.type; + if (!e.items.length) return `${this.type(t)}()`; + const direct = this.landing === e; + // every item lands in the list's element type, so a list inside a list is in brackets too + const lit = `[${e.items.map((x: any) => this.widened(t?.of, x, this.copied(x, this.withHint(t?.of, () => this.expr(x), x)))).join(', ')}]`; + if (direct) return lit; + // nothing here gives the brackets a type (Mojo would make a fixed-size Array of them), so the list is named first, with its type + const name = `_list${++this.temp}`; + this.hoist([[0, `var ${name}: ${this.type(t)} = ${lit}`]], constantOnly(e)); + this.note('list-literal', 'A list written in brackets takes its type from where it lands: a declared variable, a parameter, a returned value. Where nothing gives it one, the translation names the list first, with its type.'); + return name; + } + case 'SetLit': this.need('Set'); return `${this.type(e.type?.of?.k !== 'ANY' ? e.type : this.hint)}(${e.items.map((x: any) => this.expr(x)).join(', ')})`; + case 'MapLit': { + // keys and values land in the map's types: an INTEGER value in a map of FLOAT is converted, a list value is in brackets + const t = e.type?.k === 'HASHMAP' && e.type.val?.k !== 'ANY' ? e.type : this.hint?.k === 'HASHMAP' ? this.hint : e.type; + return `{${e.entries.map((x: any) => `${this.widened(t?.key, x.key, this.withHint(t?.key, () => this.expr(x.key), x.key))}: ${this.widened(t?.val, x.value, this.copied(x.value, this.withHint(t?.val, () => this.expr(x.value), this.landing === e ? x.value : null)))}`).join(', ')}}`; + } + case 'Range': return `range(${this.expr(e.lo)}, ${this.expr(e.hi)})`; + case 'SetBuilder': { + const v = this.name(e.var); + const dom = this.domain(e.domain); + const t = this.fresh('set'); + this.hoistOK = false; + let body: string; + try { body = this.expr(e.body); } finally { this.hoistOK = true; } + this.hoist([[0, `var ${t} = ${this.type(e.type)}()`], [0, `for ${v} in ${dom}:`], [1, `if ${body}:`], [2, `${t}.add(${v})`]]); + // Mojo 1.2.0 has set comprehensions (probes/set-comprehension); the loop keeps the notation's order of work visible + this.note('comprehension', 'A set written by rule becomes a loop, written just before the line that uses it, that fills a Set.'); + return t; + } + case 'Quant': { + const v = this.name(e.var); + const dom = this.domain(e.domain); + const t = this.fresh(e.q === 'FORALL' ? 'all' : e.q === 'EXISTS' ? 'any' : 'sum'); + const saved = this.hoistOK; + this.hoistOK = false; + let body: string; + try { body = this.expr(e.body); } finally { this.hoistOK = saved; } + if (e.q === 'FORALL') this.hoist([[0, `var ${t} = True`], [0, `for ${v} in ${dom}:`], [1, `if not (${body}):`], [2, `${t} = False`], [2, 'break']]); + else if (e.q === 'EXISTS') this.hoist([[0, `var ${t} = False`], [0, `for ${v} in ${dom}:`], [1, `if ${body}:`], [2, `${t} = True`], [2, 'break']]); + else this.hoist([[0, `var ${t} = ${e.type?.k === 'FLOAT' ? '0.0' : '0'}`], [0, `for ${v} in ${dom}:`], [1, `${t} += ${body}`]]); + // probes/all-generator: all(x > 0 for x in xs) does not parse in Mojo 1.2.0 + this.note('quantifier', 'FORALL, EXISTS, and SUM have no one-line Mojo spelling (Mojo has no generator expressions to hand to all() or sum()); they become a loop, written just before the line that uses them, that sets a flag or adds to a running total.'); + return t; + } + default: return this.gap(`a ${e.kind} expression`); + } + } + + /** An operand of a binary operator: parenthesized when precedence needs it, and an Int meeting a Float64 is converted. */ + sideExpr(x: any, parent: any, side: 'l' | 'r'): string { + const BIND: Record<string, number> = { OR: 1, AND: 2, '==': 4, '!=': 4, '<': 4, '<=': 4, '>': 4, '>=': 4, IN: 4, '&': 5, '+': 6, '-': 6, '*': 7, '/': 7, DIV: 7, MOD: 7, '^': 9 }; + const other = side === 'l' ? parent.right : parent.left; + // Mojo has no comparison or power with a whole-number literal on the left of a Float64 (`90 <= score`, `2 ** x`): written as a float, it has + // and none with any literal on the left of a Float32: there the literal is given the type + if (side === 'l' && litOnly(x) && mojoNumber(other?.type) === 'Float32' && ['==', '!=', '<', '<=', '>', '>='].includes(parent.op)) return `Float32(${this.expr(x)})`; + if (side === 'l' && this.wholeLiteral(x) && other?.type?.k === 'FLOAT' && ['==', '!=', '<', '<=', '>', '>=', '^'].includes(parent.op)) { + const lit = this.beside(x, other); + return parent.op === '^' && lit.startsWith('-') ? `(${lit})` : lit; // -100.0 ** 2.0 is -(100.0 ** 2.0) + } + let s = this.expr(x); + if (['+', '-', '*', '==', '!=', '<', '<=', '>', '>=', 'DIV', 'MOD', '^'].includes(parent.op) && x.kind !== 'Num' && other?.type) { + // two sides of different number types meet as the notation has them meet + const mine = mojoNumber(x.type); const theirs = mojoNumber(other.type); + const compare = ['==', '!=', '<', '<=', '>', '>='].includes(parent.op); + if (mine && theirs && mine !== theirs) { + if (x.type.k === 'INTEGER' && other.type.k === 'FLOAT') s = `${theirs}(${s})`; + else if (x.type.k === 'INTEGER' && other.type.k === 'INTEGER' && !x.type.bits) s = `${theirs}(${s})`; + else if (compare && x.type.k === 'FLOAT' && other.type.k === 'FLOAT' && mine === 'Float32') s = `Float64(${s})`; + else if (compare && x.type.k === 'INTEGER' && other.type.k === 'INTEGER') s = `Int(${s})`; + } + } + if (x.kind === 'Chain' || x.kind === 'Quant') return `(${s})`; + if (x.kind === 'Unary' && x.op === 'NOT' && BIND[parent.op] > 3) return `(${s})`; + // ** binds tighter than a minus sign or a ~ in Mojo: -2 ** 8 is -(2 ** 8), and the notation's (-2) ^ 8 is 256 + if (parent.op === '^' && side === 'l' && (s.startsWith('-') || s.startsWith('~'))) return `(${s})`; + if (x.kind !== 'Binary') return s; + const a = BIND[x.op]; const b = BIND[parent.op]; + // ** groups from the right, as ^ does: (X ^ 2) ^ 3 keeps its brackets, X ^ (2 ^ 3) needs none + if (parent.op === '^' && x.op === '^') return side === 'l' ? `(${s})` : s; + if (a < b || (a === b && side === 'r' && !(parent.op === x.op && ['+', '*', 'AND', 'OR', '&'].includes(parent.op))) || (a === 4 && b === 4)) return `(${s})`; + return s; + } + + /** A number literal that is not zero, with or without a minus sign: a divisor that cannot be zero. */ + nonzeroLiteral(x: any): boolean { + const n = x?.kind === 'Unary' && x.op === '-' ? x.arg : x; + return n?.kind === 'Num' && Number(n.value) !== 0; + } + + /** + * The note for a FLOAT raised to a power. Mojo 1.2.0 raises a Float64 to a whole power by squaring, which can differ + * from a correctly rounded result in the last digit or two (probes/float-pow-whole: 1.1 ** 20 prints 6.727499949325609, + * the true value rounds to ...611), and to any other power with a routine good to about 13 digits (probes/float-pow-accuracy); + * a whole exponent of about 1e10 or more stops the built program (probes/float-pow-huge). + */ + powerNote(exponent: any): void { + const n = exponent?.kind === 'Unary' && exponent.op === '-' ? exponent.arg : exponent; + const whole = exponent?.type?.k === 'INTEGER' || (n?.kind === 'Num' && Number.isInteger(Number(n.value))); + const fraction = n?.kind === 'Num' && !Number.isInteger(Number(n.value)); + const huge = 'In Mojo 1.2.0 a whole exponent of about 1e10 or more stops the built program.'; + if (whole) this.note('power-whole', `Mojo raises a Float64 to a whole power by repeated multiplication, which can differ from the notation's correctly rounded result in the last digit or two. ${huge}`); + else if (fraction) this.note('power', 'Mojo raises a Float64 to a fractional power with its own routine: the result is good to about 13 significant digits, where the notation\'s is good to 16.'); + else this.note('power', `Mojo computes ** on a Float64 with its own routines: to a whole power by repeated multiplication, which can differ in the last digit or two, and to a fractional power to about 13 significant digits, where the notation's is good to 16. ${huge}`); + } + + /** Is this a whole-number literal, with or without a minus sign? */ + wholeLiteral(x: any): boolean { + const n = x?.kind === 'Unary' && x.op === '-' ? x.arg : x; + return n?.kind === 'Num' && !n.float && !/[.e]/.test(String(n.value)); + } + + /** A whole-number literal standing beside a FLOAT, written as a float literal (90 beside a Float64 is 90.0); anything else unchanged. */ + beside(x: any, neighbor: any): string { + const s = this.expr(x); + return this.wholeLiteral(x) && neighbor?.type?.k === 'FLOAT' ? `${s}.0` : s; + } + + /** + * The operands of one chained comparison (`lo <= x < hi`), written so Mojo can compare each pair: when a FLOAT is + * among whole numbers, a whole-number literal becomes a float literal and a whole-number value becomes Float64(...). + */ + together(nodes: any[], lazyFrom = Infinity): string[] { + // an operand from `lazyFrom` on is computed only when the comparisons before it hold, so nothing of it is written ahead of the line + const quiet = (x: any): string => { + if (nodes.indexOf(x) < lazyFrom) return this.expr(x); + const saved = this.hoistOK; + this.hoistOK = false; + try { return this.expr(x); } finally { this.hoistOK = saved; } + }; + const P = (x: any) => (x && ['Binary', 'Chain', 'Quant', 'Unary'].includes(x.kind) && !this.wholeLiteral(x) ? `(${quiet(x)})` : quiet(x)); + const float = nodes.find((x) => x?.type?.k === 'FLOAT'); + if (!float || nodes.every((x) => x?.type?.k === 'FLOAT')) return nodes.map(P); + const T = mojoNumber(float.type)!; + return nodes.map((x) => (x?.type?.k !== 'INTEGER' ? P(x) : this.wholeLiteral(x) ? `${this.expr(x)}.0` : `${T}(${quiet(x)})`)); + } + + /** The text of a value a method is called on: in brackets when it is an expression (`(a + b).upper()`), and a String when it is a text literal. */ + receiver(node: any, text: string): string { + // literals joined with & are still one literal to Mojo, a StringLiteral, which lacks some of String's methods (replace) + const literal = (x: any): boolean => x?.kind === 'Str' || x?.kind === 'Char' || (x?.kind === 'Binary' && x.op === '&' && literal(x.left) && literal(x.right)); + if (literal(node)) return `String(${text})`; + return ['Binary', 'Chain', 'Unary', 'Quant'].includes(node?.kind) ? `(${text})` : text; + } + + /** A value as the Mojo number type T: unchanged when it already is one, or a literal; converted otherwise. */ + as(T: string, node: any, text: string): string { + if (mojoNumber(node?.type) === T) return text; + return `${T}(${text})`; + } + + /** IS_CLOSE's four values, as Float64 unless they are already FLOATs of one precision. */ + args4(e: any): string[] { + const f32 = e.args.slice(0, 2).every((x: any) => mojoNumber(x.type) === 'Float32'); + return e.args.map((x: any) => (x.kind === 'Num' || mojoNumber(x.type) === (f32 ? 'Float32' : 'Float64') ? this.expr(x) : `${f32 ? 'Float32' : 'Float64'}(${this.expr(x)})`)); + } + + /** A two-dimensional ARRAY's cells as one list, row by row (named first); anything else as it is. */ + flat(node: any, text: string): string { + if (node?.type?.k !== 'ARRAY' || node.type.dims !== 2) return text; + const row = this.fresh('row'); const x = this.fresh('x'); + return this.loopResult('cells', '', `List[${this.type(node.type.of)}]()`, [[0, `for ${row} in ${text}:`], [1, `for ${x} in ${row}:`], [2, `$T.append(${x})`]]); + } + + /** A built-in that is a short loop: write it before the statement and name its result. */ + loopResult(stem: string, type: string, init: string, lines: Array<[number, string]>): string { + const t = this.fresh(stem); + this.hoist([[0, `var ${t}${type ? `: ${type}` : ''} = ${init}`], ...lines.map(([d, x]): [number, string] => [d, x.split('$T').join(t)])]); + return t; + } + + /** + * An argument that a loop written here mentions on every pass. When working it out calls something, it is named + * first, so the call happens once, as in the notation: SLICE(F(X), 0, 2) wrote F(X) inside the loop and ran it + * once per letter. + */ + once(arg: any, text: string): string { + let calls = false; + walkExpr(arg, (y: any) => { if (y.kind === 'CallExpr') calls = true; return undefined; }); + // a name the translation made itself (a list in brackets it typed, a loop's result) is already worked out once + if (!calls || this.temps.has(text) || /^_[a-z]+\d+$/.test(text)) return text; + const t = this.fresh('value'); + this.hoist([[0, `var ${t} = ${text}`]]); + return t; + } + + call(e: any): string { + const printedHere = this.printed === e; + this.printed = null; + if (e.isOld) return this.gap('OLD(...) outside a contract'); + if (e.convert) { + const to = mojoNumber(e.type)!; + if (e.type.k === 'INTEGER') this.note('sized', 'In the notation, a value that does not fit a sized whole-number type stops the run. In Mojo, Int8 and the other fixed-width types wrap around silently, and a conversion such as Int8(x) keeps only the low bits.'); + const arg = e.args[0]; + // a FLOAT literal becomes a Float64 first, so the conversion is from a runtime value, as in the notation + return `${to}(${arg.kind === 'Num' && arg.float ? `Float64(${this.expr(arg)})` : this.expr(arg)})`; + } + if (!e.builtin) { + const flags = this.varParams.get(key(e.name)) || []; + const args = e.args.map((a: any, i: number) => this.callArg(a, e.sub?.params?.[i], flags[i])); + return `${this.name(e.name)}(${args.join(', ')})`; + } + const args: string[] = e.args.map((a: any) => this.expr(a)); + // the statistics and SORTED of a two-dimensional ARRAY look at every cell; in Mojo it is a list of rows + if (['SUM_OF', 'MEAN_OF', 'MIN', 'MAX', 'SORTED', 'VARIANCE_OF', 'STDEV_OF', 'MEDIAN_OF', 'NORM_OF'].includes(e.builtin) && e.args.length === 1) args[0] = this.flat(e.args[0], args[0]); + const [a, b, c] = args; + const t0 = e.args[0]?.type?.k; + const math = (f: string): string => { this.need(f); return `${f}(Float64(${a}))`; }; + switch (e.builtin) { + case 'LENGTH': return t0 === 'STRING' || t0 === 'CHAR' ? `${this.receiver(e.args[0], a)}.count_codepoints()` : `len(${a})`; + case 'ROWS': return `len(${a})`; + case 'COLUMNS': return `len(${a}[0])`; + // Mojo will not make an Int of a float literal at compile time; of a Float64 value it does, dropping the fraction toward zero + case 'INT': return `Int(${e.args[0].type?.k === 'FLOAT' && comptimeOnly(e.args[0]) ? `Float64(${a})` : a})`; + case 'ROUND': { + if (t0 !== 'FLOAT') return a; // a whole number is its own nearest whole number + // Mojo's round() takes a half to the even neighbor (round(2.5) is 2.0); the notation takes it away from zero. Int(x) drops the + // fraction toward zero and x - Float64(Int(x)) is exact, so the half is decided without a rounding of its own. + const x = this.fresh('x'); + this.hoist([[0, `var ${x} = ${mojoNumber(e.args[0].type) === 'Float64' ? a : `Float64(${a})`}`]]); + this.note('round', 'ROUND takes a half away from zero: ROUND(2.5) is 3 and ROUND(-2.5) is -3. Mojo\'s round() takes a half to the even neighbor (round(2.5) is 2.0), so the translation drops the fraction with Int() and then steps one further when the fraction is a half or more.'); + return this.loopResult('rounded', '', `Int(${x})`, [[0, `if ${x} - Float64($T) >= 0.5:`], [1, '$T += 1'], [0, `elif ${x} - Float64($T) <= -0.5:`], [1, '$T -= 1']]); + } + case 'ABS': return `abs(${a})`; + case 'MIN': case 'MAX': { + if (e.args.length === 1 && (t0 === 'LIST' || t0 === 'ARRAY')) { + // Mojo's min() and max() compare their arguments; given one list they hand it back. The smallest item is a loop. + const x = this.fresh('x'); + this.note('min-of-list', 'MIN and MAX of a list look at its items. Mojo\'s min() and max() compare their arguments, so for a list the translation is a loop that keeps the smallest (or largest) item it has seen.'); + return this.loopResult(e.builtin === 'MIN' ? 'least' : 'most', '', `${a}[0]`, [[0, `for ${x} in ${a}:`], [1, `if ${x} ${e.builtin === 'MIN' ? '<' : '>'} $T:`], [2, `$T = ${x}`]]); + } + const mixed = e.type?.k === 'FLOAT' && e.args.length > 1 && e.args.some((x: any) => x.type?.k === 'INTEGER'); + // among FLOATs a whole-number value becomes Float64(...) and a whole-number literal a float literal: min(100, 1e10) has no meaning in Mojo + return `${e.builtin.toLowerCase()}(${args.map((x, i) => (!mixed ? x : this.wholeLiteral(e.args[i]) ? `${x}.0` : this.widened(e.type, e.args[i], x))).join(', ')})`; + } + case 'SQRT': return math('sqrt'); + case 'EXP': return math('exp'); + case 'LOG': return math('log'); + case 'LOG2': + // probes/float-log2-accuracy + this.note('log2', 'In Mojo 1.2.0, log2() of a Float64 is good to about 9 significant digits (log2 of the square root of 2 prints 0.500000001011283), where the notation\'s LOG2 is good to 16. log() and log10() gave the correctly rounded value on the inputs tried.'); + return math('log2'); + case 'LOG10': return math('log10'); + case 'SIN': return math('sin'); + case 'COS': return math('cos'); + case 'TAN': return math('tan'); + case 'ATAN2': this.need('atan2'); return `atan2(Float64(${a}), Float64(${b}))`; + case 'PI': this.need('pi'); return 'pi'; + case 'CHOOSE': this.need('comb'); return `comb(${a}, ${b})`; + case 'FLOOR': this.need('floor'); return `Int(floor(${a}))`; + case 'CEIL': this.need('ceil'); return `Int(ceil(${a}))`; + case 'RAND': this.need('random_float64'); return `random_float64() * ${a}`; + case 'RANDINT': this.need('random_si64'); return `Int(random_si64(${a}, ${b} - 1))`; + case 'SLICE': + if (t0 === 'STRING') { + // the verified programs build a piece of a String letter by letter; s[byte=a:b] counts bytes and is not run by the interpreter + const k = this.fresh('at'); + this.note('slice', 'A piece of a String is built letter by letter, which counts characters the way the notation does.'); + const text = this.once(e.args[0], a); + return this.loopResult('piece', '', 'String("")', [[0, `for ${k} in range(${b}, ${c}):`], [1, `$T += String(${text}[codepoint=${k}])`]]); + } + // a[b:c] is a Span that borrows the list; the notation's SLICE is a new LIST + return `${this.type(e.type)}(${a}[${b}:${c}])`; + case 'UPPER': return `${this.receiver(e.args[0], a)}.upper()`; + case 'LOWER': return `${this.receiver(e.args[0], a)}.lower()`; + case 'ORD': return `ord(${a})`; + case 'CHR': return `chr(${a})`; + // a BOOL is TRUE or FALSE in the notation's text, True or False in Mojo's + case 'TO_STRING': return t0 === 'BOOL' ? `String("TRUE" if ${a} else "FALSE")` : this.textOf(e.args[0], printedHere, `String(${a})`); + case 'TO_INTEGER': return `atol(${a})`; + case 'TO_FLOAT': return t0 === 'STRING' ? `atof(${a})` : `Float64(${a})`; + // a text literal is a StringLiteral in Mojo, which has no replace(); as a String it has + case 'REPLACE': return `${this.receiver(e.args[0], a)}.replace(${b}, ${c})`; + case 'JOIN': return `${this.receiver(e.args[1], b)}.join(${a})`; + case 'INDEX_OF': + if (t0 === 'STRING') return `${this.receiver(e.args[0], a)}.find(${b})`; + { + const i = this.fresh('i'); + const list = this.once(e.args[0], a); const item = this.once(e.args[1], b); + return this.loopResult('at', '', '-1', [[0, `for ${i} in range(len(${list})):`], [1, `if ${list}[${i}] == ${item}:`], [2, `$T = ${i}`], [2, 'break']]); + } + case 'SPLIT': { + const p = this.fresh('piece'); + const parts = this.fresh('parts'); + this.hoist([[0, `var ${parts} = List[String]()`], [0, `for ${p} in ${this.receiver(e.args[0], a)}.split(${b}):`], [1, `${parts}.append(String(${p}))`]]); + return parts; + } + case 'FILL': { + const ty = this.type(e.type?.of ?? e.args[1]?.type); + // the value is worked out once, as in the notation, and each item is a copy of it: FILL(3, F(X)) calls F once + let v = b; + if (!litOnly(e.args[1]) && !['Str', 'Char', 'Bool'].includes(e.args[1]?.kind) && !/^[A-Za-z_]\w*$/.test(b)) { v = this.fresh('item'); this.hoist([[0, `var ${v}: ${ty} = ${b}`]]); } + const item = isCollection(e.args[1]?.type) ? `${v}.copy()` : v; + return this.loopResult('fill', '', `List[${ty}]()`, [[0, 'for _ in range(' + a + '):'], [1, `$T.append(${item})`]]); + } + case 'SUM_OF': { + const x = this.fresh('x'); + if (isSized(e.type)) return this.loopResult('total', this.type(e.type), '0', [[0, `for ${x} in ${a}:`], [1, `$T += ${x}`]]); + const zero = e.type?.k === 'FLOAT' ? '0.0' : '0'; + return this.loopResult('total', '', zero, [[0, `for ${x} in ${a}:`], [1, `$T += ${x}`]]); + } + case 'MEAN_OF': { + const x = this.fresh('x'); + const sum = this.loopResult('total', '', '0.0', [[0, `for ${x} in ${a}:`], [1, `$T += Float64(${x})`]]); + return `(${sum} / Float64(len(${a})))`; + } + case 'DOT': { + const i = this.fresh('i'); + const u = this.once(e.args[0], a); const v = this.once(e.args[1], b); + if (isSized(e.type)) return this.loopResult('dot', this.type(e.type), '0', [[0, `for ${i} in range(len(${u})):`], [1, `$T += ${u}[${i}] * ${v}[${i}]`]]); + return this.loopResult('dot', '', e.type?.k === 'FLOAT' ? '0.0' : '0', [[0, `for ${i} in range(len(${u})):`], [1, `$T += ${u}[${i}] * ${v}[${i}]`]]); + } + case 'VARIANCE_OF': case 'STDEV_OF': { + const x = this.fresh('x'); + const m = this.loopResult('mean', '', '0.0', [[0, `for ${x} in ${a}:`], [1, `$T += Float64(${x})`]]); + this.hoist([[0, `${m} = ${m} / Float64(len(${a}))`]]); + const v = this.loopResult('var', '', '0.0', [[0, `for ${x} in ${a}:`], [1, `$T += (Float64(${x}) - ${m}) * (Float64(${x}) - ${m})`]]); + this.hoist([[0, `${v} = ${v} / Float64(len(${a}) - 1)`]]); + this.note('statistics', 'The standard library has no variance or standard deviation, so they are written as loops: the sample variance divides by n - 1, as the notation does.'); + if (e.builtin === 'VARIANCE_OF') return v; + this.need('sqrt'); + return `sqrt(${v})`; + } + case 'NORM_OF': { + const x = this.fresh('x'); + this.need('sqrt'); + const q = this.loopResult('squares', '', '0.0', [[0, `for ${x} in ${a}:`], [1, `$T += Float64(${x}) * Float64(${x})`]]); + return `sqrt(${q})`; + } + case 'MEDIAN_OF': { + const x = this.fresh('x'); + this.need('sort'); + const of = e.args[0].type?.of; + const sorted = this.loopResult('sorted', '', `${this.type({ k: 'LIST', of })}()`, [[0, `for ${x} in ${a}:`], [1, `$T.append(${x})`]]); + this.hoist([[0, `sort(${sorted})`]]); + const mid = this.loopResult('median', '', '0.0', [ + [0, `if len(${sorted}) % 2 == 1:`], [1, `$T = Float64(${sorted}[len(${sorted}) // 2])`], + [0, 'else:'], [1, `$T = (Float64(${sorted}[len(${sorted}) // 2 - 1]) + Float64(${sorted}[len(${sorted}) // 2])) / 2.0`], + ]); + return mid; + } + case 'KEYS': { + const k = this.fresh('k'); + this.need('sort'); + const keys = this.loopResult('keys', '', `${this.type({ k: 'LIST', of: e.args[0].type.key })}()`, [[0, `for ${k} in ${a}.keys():`], [1, `$T.append(${k})`]]); + this.hoist([[0, `sort(${keys})`]]); + return keys; + } + case 'SORTED': { + const k = this.fresh('x'); + this.need('sort'); + const of = e.args[0].type?.k === 'SET' ? e.args[0].type.of : e.args[0].type?.of; + const items = this.loopResult('sorted', '', `${this.type({ k: 'LIST', of })}()`, [[0, `for ${k} in ${a}:`], [1, `$T.append(${k})`]]); + this.hoist([[0, `sort(${items})`]]); + return items; + } + // ---- numerical work + case 'ASIN': return math('asin'); + case 'ACOS': return math('acos'); + case 'ATAN': return math('atan'); + case 'SINH': return math('sinh'); + case 'COSH': return math('cosh'); + case 'TANH': return math('tanh'); + case 'CBRT': return math('cbrt'); + case 'EXPM1': return math('expm1'); + case 'LOG1P': return math('log1p'); + case 'HYPOT': this.need('hypot'); return `hypot(Float64(${a}), Float64(${b}))`; + case 'POW': + this.powerNote(e.args[1]); + return `Float64(${a}) ** Float64(${b})`; + case 'COPYSIGN': this.need('copysign'); return `copysign(Float64(${a}), Float64(${b}))`; + case 'E': return '2.718281828459045'; + case 'TRUNC': this.need('trunc'); return `trunc(${e.args[0].type?.k === 'FLOAT' ? a : `Float64(${a})`})`; + case 'SIGN': { const T = this.type(e.type); return `(${T}(1) if ${a} > 0 else (${T}(-1) if ${a} < 0 else ${T}(0)))`; } + case 'CLAMP': return `min(max(${a}, ${b}), ${c})`; + case 'FMA': { const T = mojoNumber(e.type)!; this.need('fma'); return `fma(${this.as(T, e.args[0], a)}, ${this.as(T, e.args[1], b)}, ${this.as(T, e.args[2], c)})`; } + case 'IS_NAN': this.need('isnan'); return `isnan(${e.args[0].type?.k === 'FLOAT' ? a : `Float64(${a})`})`; + case 'IS_INF': this.need('isinf'); return `isinf(${e.args[0].type?.k === 'FLOAT' ? a : `Float64(${a})`})`; + case 'IS_FINITE': this.need('isfinite'); return `isfinite(${e.args[0].type?.k === 'FLOAT' ? a : `Float64(${a})`})`; + case 'INF': this.need('inf'); return 'inf[DType.float64]()'; + case 'NAN': this.need('nan'); return 'nan[DType.float64]()'; + case 'ULP': { const T = mojoNumber(e.type)!; this.need('ulp'); return `ulp(${this.as(T, e.args[0], a)})`; } + case 'NEXT_AFTER': { const T = mojoNumber(e.type)!; this.need('nextafter'); return `nextafter(${this.as(T, e.args[0], a)}, ${this.as(T, e.args[1], b)})`; } + case 'IS_CLOSE': { + const d = this.args4(e); + return `(${d[0]} == ${d[1]} or abs(${d[0]} - ${d[1]}) <= max(${d[2]} * max(abs(${d[0]}), abs(${d[1]})), ${d[3]}))`; + } + case 'MAX_VALUE': case 'MIN_VALUE': case 'EPSILON': { + const T = mojoNumber(e.type)!; + const ty = e.type; + if (e.builtin === 'EPSILON') return `${T}(${ty.bits === 32 ? '1.1920928955078125e-07' : '2.220446049250313e-16'})`; + if (ty.k === 'INTEGER') { + const hi = ty.signed ? 2 ** (ty.bits - 1) - 1 : 2 ** ty.bits - 1; const lo = ty.signed ? -(2 ** (ty.bits - 1)) : 0; + return `${T}(${e.builtin === 'MAX_VALUE' ? hi : lo})`; + } + const m = ty.bits === 32 ? '3.4028234663852886e38' : '1.7976931348623157e308'; + return `${T}(${e.builtin === 'MAX_VALUE' ? m : `-${m}`})`; + } + case 'GCD': this.need('gcd'); return `gcd(${a}, ${b})`; + case 'LCM': this.need('lcm'); return `lcm(${a}, ${b})`; + case 'BIT_AND': return `(${a} & ${b})`; + case 'BIT_OR': return `(${a} | ${b})`; + case 'BIT_XOR': return `(${a} ^ ${b})`; + case 'BIT_NOT': return `~${['Name', 'Num', 'CallExpr', 'Index', 'Field'].includes(e.args[0].kind) ? a : `(${a})`}`; + case 'SHIFT_LEFT': case 'SHIFT_RIGHT': { + const T = this.type(e.type); + const n = isSized(e.type) ? `${T}(${b})` : b; + if (e.builtin === 'SHIFT_LEFT' && isSized(e.type)) this.note('sized', 'In the notation, a value that does not fit a sized whole-number type stops the run. In Mojo, Int8 and the other fixed-width types wrap around silently, and a conversion such as Int8(x) keeps only the low bits.'); + return `(${a} ${e.builtin === 'SHIFT_LEFT' ? '<<' : '>>'} ${n})`; + } + case 'WRAP': return `${this.type(e.type)}(${a})`; + default: + return this.gap(`the built-in ${e.builtin}, which Mojo's standard library has no function for`); + } + } +} + +/* ------------------------------------------------------------------ */ +/* Entry point */ +/* ------------------------------------------------------------------ */ + +/** + * Translate pseudo to Mojo. Returns `unsupported` only when the pseudo does + * not parse or check; everything else yields Mojo, with a diagnostic for each + * line that could not be translated. + */ +export function pseudoToMojo(pseudo: string, opts: MojoOptions = {}): MojoResult | { unsupported: Diagnostic[] } { + const st: any = study(pseudo, { coach: false }); + if (!st.ok || !st.program) { + const errors = (st.diags as any[]).filter((d) => d.severity === 'error').map((d) => toDiagnostic(d)); + return { unsupported: errors.length ? errors : [{ lang: 'pseudo', severity: 'error', id: 'translate-no-program', line: 1, message: 'The pseudo does not read yet, so there is nothing to translate.' }] }; + } + const program = st.program; + // the library routines the program uses, each once, the ones they use first + const libs = libraryOrder(program); + const libRaising = new Set<string>(); + const libraryBlocks: Array<{ id: string; lines: string[]; imports: string[] }> = []; + for (const lib of libs) { + const raising = inferRaising(lib.program, libRaising); + const gen = new Gen(lib.program, lib.source, raising, { defsOnly: true }); + gen.run(); + for (const sub of lib.program.subs.values()) if (raising.has(up(sub.name))) libRaising.add(up(sub.name)); + const lines = gen.out.map((l) => l.text); + while (lines.length && !lines[lines.length - 1].trim()) lines.pop(); + libraryBlocks.push({ id: lib.id, lines, imports: [...gen.imports] }); + } + opts = { ...opts, libraryBlocks }; + const callText = (opts.call || '').trim(); + const callName = callText ? /^\s*([A-Za-z_][A-Za-z0-9_]*)/.exec(callText)?.[1] : null; + // Inference, then translation; a line that could not be translated becomes a raise, so its routine raises too, and the pass is run again. + const extra = new Set<string>(libRaising); + for (let pass = 0; ; pass++) { + const raising = inferRaising(program, extra); + if (bodyRaises(program.main.body, raising)) raising.add('MAIN'); + if (callName && raising.has(up(callName))) raising.add('MAIN'); + for (const k of extra) if (k === 'MAIN') raising.add('MAIN'); + const gen = new Gen(program, pseudo, raising, opts); + const result = gen.run(); + const fresh = [...gen.gapOwners].map(up).filter((k) => !raising.has(k === 'MAIN' ? 'MAIN' : k)); + if (!fresh.length || pass >= 3) return result; + for (const k of fresh) extra.add(k); + } +} + +/** The names of every routine a program calls, anywhere in it. */ +function calledNames(program: any): Set<string> { + const out = new Set<string>(); + const seen = new Set<any>(); + const visit = (x: any): void => { + if (!x || typeof x !== 'object' || seen.has(x)) return; + seen.add(x); + if (Array.isArray(x)) { for (const y of x) visit(y); return; } + if ((x.kind === 'CallExpr' || x.kind === 'Call') && x.name) out.add(up(x.name)); + for (const [k, v] of Object.entries(x)) if (k !== 'sub' && k !== 'sym' && k !== 'type' && k !== 'resolved' && v && typeof v === 'object') visit(v); + }; + visit(program.main?.body); + for (const s of program.subs.values()) { visit(s.node.body); visit(s.requires); visit(s.ensures); } + return out; +} + +/** Every library routine a program calls, directly or through another routine, dependencies first, each once. */ +function libraryOrder(program: any, out: Map<string, any> = new Map()): any[] { + const called = calledNames(program); + for (const sub of program.libs?.values() ?? []) { + if (!called.has(up(sub.name))) continue; + if (out.has(sub.lib.id)) continue; + libraryOrder(sub.lib.program, out); + out.set(sub.lib.id, sub.lib); + } + return [...out.values()]; +} + +/** The set of names a generated source may use from the standard library, for the grounding test. */ +export function stdlibNamesUsed(): string[] { return Object.keys(USED_STDLIB); } +export const STD_INDEX = MOJO_STD_EXPORTS; diff --git a/src/sync/types.ts b/src/sync/types.ts new file mode 100644 index 0000000..3366ea4 --- /dev/null +++ b/src/sync/types.ts @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +/** + * The shapes every translator in src/sync/ shares (restructure plan, section 2.3). + * + * Pseudo is the hub: lingo, matho, and Mojo each translate to or from pseudo, + * never to each other. A translator is a pure function of text. + */ +import type { Diagnostic } from '../common/types.js'; + +/** + * Where each line of a translator's output came from. Entry `i` is the + * 1-based line of the input that output line `i + 1` was made from, or null + * for a line with no single source (a header comment, an import, a blank). + * Maps compose through the hub: lingo line -> pseudo line -> Mojo line. + */ +export type LineMap = Array<number | null>; + +/** + * Spellings that the pseudo cannot hold. Matho lowers `\bar{x}` to X_BAR and + * `\sigma` to SIGMA; the map records the student's own spelling, keyed by the + * pseudo name (upper case), so a round trip restores it. + */ +export type NameMap = Record<string, string>; + +/** Source in some language to pseudo. */ +export interface ToPseudo { + (source: string): { pseudo: string; map: LineMap; diagnostics: Diagnostic[]; names?: NameMap } | { error: Diagnostic[] }; +} + +/** Pseudo to some language. `unsupported` means the language cannot say this program. */ +export interface FromPseudo { + (pseudo: string, hints?: { names?: NameMap }): { source: string; map: LineMap; notes: Diagnostic[] } | { unsupported: Diagnostic[] }; +} + +export function isUnsupported(r: ReturnType<FromPseudo>): r is { unsupported: Diagnostic[] } { + return 'unsupported' in r; +} diff --git a/tests/asm-coverage.test.mjs b/tests/asm-coverage.test.mjs new file mode 100644 index 0000000..f88d2cc --- /dev/null +++ b/tests/asm-coverage.test.mjs @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The assembly census (scripts/asm-census.mjs) as a ratchet: every function the +// real compiler produced for the user's own code assembles on the model machine, +// the library functions that assemble never get fewer, and the instructions +// outside the model's set are only the 256-bit AVX2 ones the model says it +// does not have. A new corpus item that brings a new instruction fails here, +// which is the point. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { census } from '../scripts/asm-census.mjs'; +import { parseAsm, runAsmProgram } from '../dist/services/asm.js'; +import { intelLine } from '../dist/mojo/att.js'; + +const LIBRARY_AT_LEAST = 438; // raise as the machine learns more +const OUTSIDE = new Set(['vextracti128', 'vinserti128', 'vpblendvb', 'vperm2i128', 'vpalignr', 'vptest']); // ymm: 256-bit AVX2 + +test('every user function in the corpus listings assembles on the model machine', () => { + const c = census(); + assert.deepEqual(c.userFailures, []); + assert.equal(c.userOk, c.user); + assert.ok(c.user >= 7, `only ${c.user} user functions in the corpus`); +}); + +test('the library functions that assemble do not get fewer, and what is outside the set is only AVX2', () => { + const c = census(); + assert.ok(c.libOk >= LIBRARY_AT_LEAST, `${c.libOk} library functions assemble, fewer than ${LIBRARY_AT_LEAST}`); + const unexpected = c.missing.map(([op]) => op).filter((op) => !OUTSIDE.has(op)); + assert.deepEqual(unexpected, [], 'instructions the corpus uses that the model neither has nor lists as outside it'); +}); + +test('the AT&T translator strips the size suffix from the BMI and atomic mnemonics and keeps lock', () => { + const map = new Map(); + assert.equal(intelLine('andnq %rax, %r9, %rax', map), 'andn rax, r9, rax'); + assert.equal(intelLine('mulxq %rcx, %rdx, %rdx', map), 'mulx rdx, rdx, rcx'); + assert.equal(intelLine('lock xaddq %rax, (%rcx)', map), 'lock xadd qword ptr [rcx], rax'); + assert.equal(intelLine('shlxq %rcx, %rbx, %r11', map), 'shlx r11, rbx, rcx'); + assert.equal(intelLine('vpcmpeqb %xmm1, %xmm0, %xmm3', map), 'vpcmpeqb xmm3, xmm0, xmm1'); +}); + +test('the instructions compiled Mojo uses compute what the manual says', () => { + const run = (src, setup = '') => { const p = parseAsm(src); assert.ok(p.ok, p.diagnostics.map((d) => d.message).join('; ')); return runAsmProgram(p, { setup, machine: 'teaching', maxSteps: 200 }); }; + const st = run([ + 'mov rdx, 6', 'mov rcx, 7', 'mulx r8, r9, rcx', + 'mov rax, 0xff00', 'mov rbx, 0x0ff0', 'andn r10, rax, rbx', + 'mov rcx, 3', 'shlx r11, rbx, rcx', 'sarx r12, rax, rcx', 'bzhi r13, rbx, rcx', + 'mov rsi, 12', 'blsr rdi, rsi', 'blsi r14, rsi', 'blsmsk r15, rsi', 'rorx rbx, rsi, 1', + 'mov rax, 1', 'mov qword ptr [0x1000], 10', 'lock xadd qword ptr [0x1000], rax', + 'mov rax, 11', 'mov rcx, 99', 'lock cmpxchg qword ptr [0x1000], rcx', + 'bts rdx, 4', 'mov eax, 0x12345678', 'bswap eax', + ].join('\n'), 'memory 0x1000 "........"'); + const s = st.states[st.count]; + assert.equal(st.error, null); + assert.equal(s.regs.r9, 42n); assert.equal(s.regs.r8, 0n); + assert.equal(s.regs.r10, 0xf0n); assert.equal(s.regs.r11, 0x7f80n); assert.equal(s.regs.r12, 0x1fe0n); assert.equal(s.regs.r13, 0n); + assert.equal(s.regs.rdi, 8n); assert.equal(s.regs.r14, 4n); assert.equal(s.regs.r15, 7n); assert.equal(s.regs.rbx, 6n); + assert.equal(s.regs.rdx, 22n); assert.equal(s.regs.rax, 0x78563412n); assert.equal(s.flags.ZF, true); + assert.equal(s.mem.find((m) => m.addr === 0x1000).bytes[0], 99, 'cmpxchg stored the new value'); + // vectors: a broadcast compare finds every A in GATTACAGATTACA + const v = run(['vmovdqu xmm1, xmmword ptr [0x1000]', 'mov eax, 0x41', 'vmovd xmm4, eax', 'vpbroadcastb xmm4, xmm4', 'vpcmpeqb xmm3, xmm1, xmm4', 'vpmovmskb r8d, xmm3', 'vpextrb eax, xmm1, 1', 'vpshufd xmm5, xmm1, 0x1b'].join('\n'), 'memory 0x1000 "GATTACAGATTACA.."'); + const vs = v.states[v.count]; + assert.equal(vs.regs.r8, 0b10100101010010n); assert.equal(vs.regs.rax, 0x41n); + // ud2 stops the run and says why + const u = run('mov rax, 1\nud2'); + assert.match(u.error.message, /invalid-opcode/); + // a stub for the standard library stops the run and names the function + const stub = run('call std_io_io_print\nstd_io_io_print:\n ud2 ; std_io_io_print: standard library, not on the model machine'); + assert.match(stub.error.message, /std_io_io_print: standard library, not on the model machine/); +}); diff --git a/tests/asm-float.test.mjs b/tests/asm-float.test.mjs new file mode 100644 index 0000000..8c46bae --- /dev/null +++ b/tests/asm-float.test.mjs @@ -0,0 +1,256 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Scalar double arithmetic, the parity flag, shld/shrd, and the compiler's constant pool. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { parseAsm, runAsmProgram } from '../dist/services/asm.js'; +import { rflagsValue } from '../dist/engines/asm86/interp.js'; +import { intelLine, forSimulator, simulatorSetup } from '../dist/mojo/att.js'; +import { listingFunctions } from '../dist/services/godbolt.js'; + +const dv = new DataView(new ArrayBuffer(8)); +const M64 = 0xffffffffffffffffn; +const bits = (x) => { dv.setFloat64(0, x); return dv.getBigUint64(0); }; +const dbl = (b) => { dv.setBigUint64(0, BigInt(b) & M64); return dv.getFloat64(0); }; +const le = (v) => Array.from({ length: 8 }, (_, i) => Number((BigInt(v) >> BigInt(8 * i)) & 0xffn)); +const lo = (v) => BigInt(v) & M64; + +function run(src, setup = '') { + const p = parseAsm(src); + assert.ok(p.ok, p.diagnostics?.map((d) => d.message).join('; ')); + const r = runAsmProgram(p, { setup, machine: 'teaching', maxSteps: 300 }); + return { s: r.states[r.count], error: r.error?.message }; +} +const load = (reg, x) => [`mov rax, 0x${bits(x).toString(16)}`, `movq ${reg}, rax`]; +const two = (op, a, b) => run([...load('xmm0', a), ...load('xmm1', b), `${op} xmm0, xmm1`].join('\n')).s.regs.xmm0; +const three = (op, a, b) => run([...load('xmm1', a), ...load('xmm2', b), `${op} xmm0, xmm1, xmm2`].join('\n')).s.regs.xmm0; + +test('arithmetic is IEEE binary64, rounded to nearest even', () => { + assert.equal(lo(two('addsd', 0.1, 0.2)), 0x3FD3333333333334n); + assert.equal(lo(two('addsd', 0.1, 0.2)), bits(0.1 + 0.2)); + assert.equal(dbl(lo(two('divsd', 1, 3))), 1 / 3); + assert.equal(dbl(lo(two('mulsd', 1e200, 1e200))), Infinity, 'overflow gives infinity'); + assert.equal(dbl(lo(three('vsubsd', 4, 3))), 1); + assert.equal(dbl(lo(three('vdivsd', 4, 3))), 4 / 3); + const r = run([...load('xmm1', 2), 'vsqrtsd xmm0, xmm1, xmm1'].join('\n')).s.regs.xmm0; + assert.equal(dbl(lo(r)), Math.SQRT2); +}); + +test('special values: division by zero, invalid operations, NaN propagation', () => { + assert.equal(dbl(lo(two('divsd', 1, 0))), Infinity); + assert.equal(dbl(lo(two('divsd', -1, 0))), -Infinity); + assert.equal(lo(two('divsd', 0, 0)), 0xFFF8000000000000n, '0/0 is the default NaN'); + const sq = (x) => lo(run([...load('xmm1', x), 'vsqrtsd xmm0, xmm1, xmm1'].join('\n')).s.regs.xmm0); + assert.equal(sq(-1), 0xFFF8000000000000n); + assert.equal(sq(-0), bits(-0), 'sqrt(-0) is -0'); + // a quiet NaN with a payload is kept; a signalling one is quieted; with two NaNs the first operand wins + const nan = (payload) => `mov rax, 0x${payload.toString(16)}\nmovq xmm0, rax`; + const q = run(`${nan(0x7FF8000000000123n)}\n${load('xmm1', 1).join('\n')}\naddsd xmm0, xmm1`).s.regs.xmm0; + assert.equal(lo(q), 0x7FF8000000000123n); + const sig = run(`${nan(0x7FF0000000000001n)}\n${load('xmm1', 1).join('\n')}\naddsd xmm0, xmm1`).s.regs.xmm0; + assert.equal(lo(sig), 0x7FF8000000000001n); + const both = run(`${nan(0x7FF8000000000001n)}\nmov rax, 0x7FF8000000000002\nmovq xmm1, rax\naddsd xmm0, xmm1`).s.regs.xmm0; + assert.equal(lo(both), 0x7FF8000000000001n); +}); + +test('cvtsi2sd rounds to nearest even and reads the integer width asked for', () => { + const cvt = (setup, line) => lo(run(`${setup}\n${line}`).s.regs.xmm0); + assert.equal(cvt('mov rax, 9007199254740993', 'cvtsi2sd xmm0, rax'), bits(9007199254740992), '2^53+1 ties to even'); + assert.equal(cvt('mov rax, 0x8000000000000000', 'cvtsi2sd xmm0, rax'), bits(-9223372036854775808)); + assert.equal(cvt('mov eax, 0xffffffff', 'cvtsi2sd xmm0, eax'), bits(-1), '32-bit source is signed'); +}); + +test('upper halves: the VEX forms copy from the first source, the SSE forms keep the destination', () => { + // 16 bytes at 0x1000: low half 5.0's bits, high half 0x1111; at 0x1010: low 7.0's bits, high 0x2222 + const mem = `memory 0x1000 bytes ${[...le(bits(5)), ...le(0x1111n), ...le(bits(7)), ...le(0x2222n)].map((b) => '0x' + b.toString(16)).join(' ')}`; + const prog = (body) => run(['movdqu xmm0, xmmword ptr [0x1000]', 'movdqu xmm1, xmmword ptr [0x1010]', body].join('\n'), mem).s.regs.xmm0; + const hi = (v) => v >> 64n; + assert.equal(hi(prog('vaddsd xmm0, xmm1, xmm1')), 0x2222n, 'vaddsd takes the upper half of its first source'); + assert.equal(dbl(lo(prog('vaddsd xmm0, xmm1, xmm1'))), 14); + assert.equal(hi(prog('addsd xmm0, xmm1')), 0x1111n, 'addsd keeps the destination\u2019s'); + assert.equal(hi(prog('movsd xmm0, xmm1')), 0x1111n, 'register movsd merges the low half'); + assert.equal(dbl(lo(prog('movsd xmm0, xmm1'))), 7); + assert.equal(hi(prog('movsd xmm0, qword ptr [0x1010]')), 0n, 'a movsd load clears the upper half'); + assert.equal(hi(prog('vmovsd xmm0, xmm0, xmm1')), 0x1111n, 'three-operand vmovsd keeps the first source\u2019s upper half'); + assert.equal(dbl(lo(prog('vmovsd xmm0, xmm0, xmm1'))), 7); +}); + +const flagsAfter = (a, b, op = 'ucomisd') => run([...load('xmm0', a), ...load('xmm1', b), `${op} xmm0, xmm1`].join('\n')).s.flags; +const pick = (f) => ['ZF', 'PF', 'CF'].map((k) => (f[k] ? 1 : 0)).join(''); + +test('ucomisd and comisd: less sets CF, equal sets ZF, unordered sets all three', () => { + for (const op of ['ucomisd', 'comisd', 'vucomisd', 'vcomisd']) { + assert.equal(pick(flagsAfter(1, 2, op)), '001', `${op} less`); + assert.equal(pick(flagsAfter(2, 1, op)), '000', `${op} greater`); + assert.equal(pick(flagsAfter(2, 2, op)), '100', `${op} equal`); + assert.equal(pick(flagsAfter(NaN, 1, op)), '111', `${op} unordered`); + } + assert.equal(pick(flagsAfter(0, -0)), '100', '+0 equals -0'); + const f = flagsAfter(NaN, 1); + assert.equal(f.OF, false); + assert.equal(f.SF, false); +}); + +test('conditions after a compare: ja, jb, je, jp and setp', () => { + const took = (a, b, j) => run([...load('xmm0', a), ...load('xmm1', b), 'ucomisd xmm0, xmm1', 'mov rcx, 0', `${j} yes`, 'jmp done', 'yes: mov rcx, 1', 'done: nop'].join('\n')).s.regs.rcx; + assert.equal(took(2, 1, 'ja'), 1n); + assert.equal(took(1, 2, 'ja'), 0n); + assert.equal(took(1, 2, 'jb'), 1n); + assert.equal(took(3, 3, 'je'), 1n); + assert.equal(took(NaN, 3, 'jp'), 1n); + assert.equal(took(3, 3, 'jp'), 0n); + assert.equal(took(NaN, 3, 'jnp'), 0n); + const set = run([...load('xmm0', NaN), ...load('xmm1', 1), 'ucomisd xmm0, xmm1', 'setp al', 'setnp bl'].join('\n')).s.regs; + assert.equal(set.rax & 0xffn, 1n); + assert.equal(set.rbx & 0xffn, 0n); +}); + +test('the parity flag: set by logic, arithmetic and shifts; left alone by rotates; cleared by popcnt', () => { + const pf = (src) => run(src).s.flags.PF; + assert.equal(pf('mov rax, 3\ntest rax, rax'), true, '0b11 has two set bits'); + assert.equal(pf('mov rax, 1\ntest rax, rax'), false); + assert.equal(pf('mov rax, 5\nxor rax, rax'), true, 'zero has even parity'); + assert.equal(pf('mov rax, 1\nshl rax, 1'), false); + assert.equal(pf('mov rax, 2\ninc rax'), true); + assert.equal(pf('mov rax, 0\ntest rax, rax\nrol rax, 1'), true, 'rol does not touch PF'); + assert.equal(pf('mov rax, 0\ntest rax, rax\npopcnt rbx, rax'), false); + assert.equal((rflagsValue({ ZF: false, SF: false, CF: false, OF: false, PF: true }) >> 2n) & 1n, 1n, 'PF is bit 2 of RFLAGS'); + assert.equal((rflagsValue({ ZF: false, SF: false, CF: false, OF: false, PF: false }) >> 2n) & 1n, 0n); +}); + +test('shld and shrd shift in bits from the second register', () => { + const r = (src) => run(src, 'memory 0x1000 zero 16').s; + assert.equal(r('mov rax, 0xff\nmov rsi, 0xf000000000000000\nshld rax, rsi, 4').regs.rax, 0xfffn); + assert.equal(r('mov rbx, 0xff\nmov rdi, 0xf\nshrd rbx, rdi, 4').regs.rbx, 0xf00000000000000fn); + assert.equal(r('mov rax, 1\nmov rcx, 4\nmov rsi, 0xf000000000000000\nshld rax, rsi, cl').regs.rax, 0x1fn, 'count in cl'); + // count zero changes nothing, flags included + const z = r('mov rax, 3\ntest rax, rax\nmov rsi, 1\nshld rax, rsi, 0'); + assert.equal(z.regs.rax, 3n); + assert.equal(z.flags.PF, true); + // 32-bit counts are masked to five bits: 33 acts as 1 + assert.equal(r('mov eax, 1\nmov esi, 0x80000000\nshld eax, esi, 33').regs.rax, 3n); + // the same register on both sides is a rotate + assert.equal(r('mov rax, 0x8000000000000001\nshld rax, rax, 1').regs.rax, 3n); + // carry is the last bit shifted out of the destination + assert.equal(r('mov rax, 0x8000000000000000\nmov rsi, 0\nshld rax, rsi, 1').flags.CF, true); + // a memory destination + assert.equal(r('mov qword ptr [0x1000], 0xff\nmov rsi, 0xf000000000000000\nshld qword ptr [0x1000], rsi, 4\nmov rax, qword ptr [0x1000]').regs.rax, 0xfffn); +}); + +test('shld refuses a 16-bit operand and nonsense forms', () => { + for (const bad of ['shld ax, bx, 1', 'vaddsd xmm0, rax', 'cvtsi2sd rax, xmm0', 'ucomisd rax, rbx', 'sqrtsd xmm0']) { + const p = parseAsm(bad); + const failed = !p.ok || (() => { const r = runAsmProgram(p, { setup: '', machine: 'teaching', maxSteps: 10 }); return !!r.error; })(); + assert.ok(failed, bad); + } +}); + +test('AT&T spellings of the new instructions become Intel', () => { + const t = (att) => intelLine(att, new Map()); + assert.equal(t('vcvtsi2sd %rax, %xmm0, %xmm0'), 'vcvtsi2sd xmm0, xmm0, rax'); + assert.equal(t('vcvtsi2sdl (%rax), %xmm0, %xmm0'), 'vcvtsi2sd xmm0, xmm0, dword ptr [rax]'); + assert.equal(t('vcvtsi2sdq 8(%rsp), %xmm1, %xmm1'), 'vcvtsi2sd xmm1, xmm1, qword ptr [rsp + 8]'); + assert.equal(t('vmovsd 8(%rsp), %xmm0'), 'vmovsd xmm0, [rsp + 8]'); + assert.equal(t('shldq $3, %rsi, %rax'), 'shld rax, rsi, 3'); + assert.equal(t('shldq %cl, %rsi, %rax'), 'shld rax, rsi, cl'); + assert.equal(t('setp %al'), 'setp al'); + assert.equal(t('vdivsd .LCPI0_0+8(%rip), %xmm0, %xmm0'), 'vdivsd xmm0, xmm0, [rip + .LCPI0_0 + 8]'); +}); + +// LLVM's output for `fn scale(n: Int) -> Float64: return Float64(n) / 1_000_000.0`, trimmed to what matters. +const LISTING = [ + '"example::scale(::Int)":', + ' vcvtsi2sd %rdi, %xmm0, %xmm0', + ' vdivsd .LCPI0_0(%rip), %xmm0, %xmm0', + ' retq', + '.LCPI0_0:', + ' .quad 0x412e848000000000 # double 1.0E+6', + '.LCPI0_1:', + ' .quad 0x4000000000000000', + ' .quad 0x4008000000000000', +].map((text, i) => ({ text, line: i === 1 ? 3 : null })); + +test('a constant pool in the listing is placed in memory and the code points at it', () => { + const fns = listingFunctions(LISTING); + const fn = fns.find((f) => f.kind === 'user'); + assert.ok(fn, 'the user function is found'); + const sim = forSimulator(fns, fn); + assert.doesNotMatch(sim.source, /LCPI/, 'no constant-pool label is left in the code'); + assert.equal(sim.data.length, 1, 'only the blocks the code uses are placed'); + assert.match(sim.data[0], /^memory 0x6000 bytes 0x00 0x00 0x00 0x00 0x80 0x84 0x2e 0x41 /); + const setup = simulatorSetup(fn, sim.data).replace('rdi = 0', 'rdi = 5000000'); + const p = parseAsm(sim.source); + assert.ok(p.ok, p.diagnostics?.[0]?.message); + const r = runAsmProgram(p, { setup, machine: 'teaching', maxSteps: 50 }); + assert.ok(!r.error, String(r.error?.message)); + assert.equal(dbl(lo(r.states[r.count].regs.xmm0)), 5); +}); + +test('a constant block with a symbol in it is not placed, so the instruction using it is refused as before', () => { + const fns = listingFunctions([ + { text: '"example::f(::Int)":' }, { text: ' vmovsd .LJT(%rip), %xmm0' }, { text: ' retq' }, + { text: '.LJT:' }, { text: ' .quad .LBB0_1' }, + ]); + const sim = forSimulator(fns, fns[0]); + assert.equal(sim.data.length, 0); +}); + +test('a call to the debugger raise hook (every -O0 loop over a range ends with one) returns instead of stopping the run', () => { + const fns = listingFunctions([ + { text: '"example::count(::Int)":' }, + { text: ' movq %rdi, %rax' }, + { text: ' callq "std::builtin::error::__mojo_debugger_raise_hook()"' }, + { text: ' callq "std::builtin::string::String::__init__()"' }, + { text: ' retq' }, + ]); + const sim = forSimulator(fns, fns[0]); + assert.deepEqual(sim.refusedCalls, ['std_builtin_string_String___init__'], 'only real library code is refused'); + assert.match(sim.source, /std_builtin_error___mojo_debugger_raise_hook:\n\s+ret\b/); + // with the String call removed, the function runs to its ret + const fns2 = listingFunctions([{ text: '"example::count(::Int)":' }, { text: ' movq %rdi, %rax' }, { text: ' callq "std::builtin::error::__mojo_debugger_raise_hook()"' }, { text: ' retq' }]); + const p = parseAsm(forSimulator(fns2, fns2[0]).source); + assert.ok(p.ok, p.diagnostics?.[0]?.message); + const r = runAsmProgram(p, { setup: 'rdi = 285', machine: 'teaching', maxSteps: 50 }); + assert.ok(!r.error, String(r.error?.message)); + assert.equal(r.states[r.count].regs.rax, 285n); +}); + +test('read-only strings in every form the listing writes them are placed: .asciz, an empty string as .zero 1, pieces, and an alias', () => { + const lines = [ + '"example::f()":', + ' leaq static_string_e(%rip), %rax', + ' leaq static_string_s(%rip), %rbx', + ' leaq static_string_p(%rip), %rcx', + ' leaq static_string_a(%rip), %rdx', + ' retq', + 'static_string_e:', ' .zero 1', + 'static_string_s:', ' .asciz " "', + 'static_string_p:', ' .ascii "ab"', ' .byte 99, 0', + 'static_string_x:', ' .asciz "hello"', + ' .set static_string_a, static_string_x+2', + ].map((text) => ({ text })); + const fns = listingFunctions(lines); + const sim = forSimulator(fns, fns.find((f) => f.kind === 'user')); + assert.doesNotMatch(sim.source, /rip \+/, 'every address became a number'); + assert.doesNotMatch(sim.source, /does not define/); + const p = parseAsm(sim.source); + assert.ok(p.ok, p.diagnostics?.[0]?.message); + const r = runAsmProgram(p, { setup: sim.data.join('\n'), machine: 'teaching', maxSteps: 20 }); + assert.ok(!r.error, String(r.error?.message)); + const regs = r.states[r.count].regs; + const mem = (addr, n) => sim.data.find((d) => d.startsWith(`memory 0x${addr.toString(16)} `)); + assert.match(mem(regs.rax), /bytes 0x00/, 'the empty string is one zero byte'); + assert.match(mem(regs.rbx), /" "/); + assert.match(mem(regs.rcx), /bytes 0x61 0x62 0x63 0x00/); + assert.equal(regs.rdx, regs.rdx, 'alias resolves'); + const hello = sim.data.find((d) => /"hello"/.test(d)); + assert.ok(hello, 'the aliased string is placed'); + assert.equal(regs.rdx, BigInt(parseInt(/memory (0x[0-9a-f]+)/.exec(hello)[1], 16) + 2), 'the alias points two bytes in'); +}); + +test('vpcmpgtb compares signed bytes', () => { + const r = run(['mov rax, 0x00000000000080ff', 'movq xmm1, rax', 'mov rax, 0x0000000000000102', 'movq xmm2, rax', 'vpcmpgtb xmm0, xmm2, xmm1'].join('\n')).s.regs.xmm0; + // lane 0: 2 > -1 yes; lane 1: 1 > -128 yes; lanes 2..15: 0 > 0 no + assert.equal(r & 0xffffn, 0xffffn); + assert.equal(r >> 16n, 0n); +}); diff --git a/tests/asm-setup.test.mjs b/tests/asm-setup.test.mjs new file mode 100644 index 0000000..6741eac --- /dev/null +++ b/tests/asm-setup.test.mjs @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The setup beside an x86-64 pane, where the listing's read-only data is written. A function sent to the pane from +// a compiled program brings the data it addresses; one table of source locations is half a megabyte, and written +// as `bytes 0x70 0x6b ...` it made a setup larger than the site will open from a link. A long block is quoted text +// with byte escapes now, and this holds the reader and the writer to each other. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { parseAsm, parseSetup, runAsmProgram } from '../dist/services/asm.js'; +import { forSimulator, simulatorSetup, LONG_BLOCK } from '../dist/mojo/att.js'; +import { listingFunctions } from '../dist/services/godbolt.js'; + +const ROOT = new URL('..', import.meta.url).pathname; +const bytesOf = (m) => (m.text != null ? [...m.text].map((c) => c.charCodeAt(0) & 0xff) : m.bytes); + +test('quoted text in a setup takes byte escapes, and a comment sign inside it is text', () => { + const s = parseSetup('memory 0x1000 "a\\x00b\\0c\\xff\\"q\\\\ # not a comment ; nor this // nor this" # a comment\nstdin "x\\n\\ty" ; another\nrdi = 0x1000 // a third'); + assert.deepEqual(bytesOf(s.memory[0]), [...'a\0b\0cÿ"q\\ # not a comment ; nor this // nor this'].map((c) => c.charCodeAt(0))); + assert.equal(s.stdin, 'x\n\ty'); + assert.equal(s.registers.rdi, 0x1000n); + // text that never closes, or closes early, is an error on its line and not a guess + for (const bad of ['memory 0x1000 "abc', 'memory 0x1000 "a"b"', 'memory 0x1000 "abc\\"']) assert.throws(() => parseSetup(bad), /setup line 1: expected text in double quotes/, bad); + // a block of every byte value survives being written long and read back (the longest a region may be is 1 MiB) + const all = Array.from({ length: 4096 }, (_, i) => (i * 7 + (i >> 8)) & 0xff); + const quoted = `"${all.map((b) => (b === 0x22 ? '\\"' : b === 0x5c ? '\\\\' : b >= 32 && b < 127 ? String.fromCharCode(b) : `\\x${b.toString(16).padStart(2, '0')}`)).join('')}"`; + assert.deepEqual(bytesOf(parseSetup(`memory 0x2000 ${quoted}`).memory[0]), all); +}); + +test('a function sent to the pane from demo\'s compiled listing brings its data in a setup a link can hold', () => { + const status = JSON.parse(fs.readFileSync(path.join(ROOT, 'library/mojo/golden/status.json'), 'utf8')); + const item = status.items.find((x) => x.id === 'local/demo'); + const golden = JSON.parse(fs.readFileSync(path.join(ROOT, 'library/mojo/golden', `${item.key}.json`), 'utf8')); + const fns = listingFunctions(golden.result.asm); + let largest = 0; + for (const name of ['cell', 'Grid::__init__', 'align', 'main']) { + const fn = fns.find((f) => f.kind === 'user' && f.name.startsWith(name)); + assert.ok(fn, `demo has no function ${name}`); + const { source, data } = forSimulator(fns, fn); + const setup = simulatorSetup(fn, data); + const parsed = parseSetup(setup); + const held = parsed.memory.reduce((n, m) => n + bytesOf(m).length, 0); + largest = Math.max(largest, held); + // about a character a byte for the long blocks: the whole setup stays under twice the bytes it places, plus its short lines + assert.ok(setup.length < 2 * held + 20000, `${name}: a setup of ${setup.length} characters for ${held} bytes of data`); + assert.ok(setup.length < 1024 * 1024, `${name}: a setup of ${setup.length} characters is more than half of what a link may hold`); + // every block lands where the code addresses it, with no two overlapping + const spans = parsed.memory.map((m) => [m.addr, m.addr + bytesOf(m).length]).sort((a, b) => a[0] - b[0]); + for (let i = 1; i < spans.length; i++) assert.ok(spans[i][0] >= spans[i - 1][1], `${name}: two blocks overlap at 0x${spans[i][0].toString(16)}`); + // and the pane still assembles and runs with it (a large function stops on the model's small stack, which is an answer) + const program = parseAsm(source); + assert.ok(program.ok, `${name}: ${JSON.stringify(program.diagnostics?.slice(0, 2))}`); + const tr = runAsmProgram(program, { setup }); + assert.ok(tr.count >= 1, `${name}: no step`); + } + assert.ok(largest > 400000, `the large table is still part of the test (${largest} bytes)`); + assert.ok(LONG_BLOCK >= 16 && LONG_BLOCK <= 4096); +}); diff --git a/tests/boundary.test.mjs b/tests/boundary.test.mjs new file mode 100644 index 0000000..5c3afa2 --- /dev/null +++ b/tests/boundary.test.mjs @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// pplus-core has no DOM: what a program means is decided without a browser. Its library modules never touch the +// document, the window or browser storage, and never import the course's page widgets, which live beside the +// engines only so the course and pplus-web can take them from one place (docs/engines.md). +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +const ROOT = new URL('..', import.meta.url).pathname; +const SRC = path.join(ROOT, 'src'); + +/** The course's page widgets: they draw into a page, and only pages (the course's, pplus-web's) import them. */ +export const WIDGETS = new Set([ + 'engines/pseudo/dom.js', 'engines/pseudo/views.js', 'engines/pseudo/views-analysis.js', 'engines/pseudo/lab.js', + 'engines/pseudo/editor.js', 'engines/pseudo/exercise.js', 'engines/pseudo/sharpen.js', 'engines/pseudo/plan.js', + 'engines/pseudo/planner.js', 'engines/pseudo/highlight.js', + 'engines/asm86/views.js', 'engines/asm86/hwviews.js', 'engines/asm86/dom.js', 'engines/asm86/sandbox.js', 'engines/asm86/specs-hw.js', + 'engines/shared/dom.js', 'engines/shared/share.js', 'engines/shared/viz.js', +]); + +function walk(dir, out = []) { + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, e.name); + if (e.isDirectory()) walk(p, out); else if (/\.(ts|js)$/.test(e.name)) out.push(path.relative(SRC, p)); + } + return out; +} +const asJs = (p) => p.replace(/\.ts$/, '.js'); +const files = walk(SRC); +const library = files.filter((f) => !WIDGETS.has(asJs(f))); + +test('every widget named here exists, so the list cannot rot', () => { + for (const w of WIDGETS) assert.ok(files.some((f) => asJs(f) === w), `${w} is not in src/`); +}); + +test('no library module touches the document, the window or browser storage', () => { + const bad = []; + for (const f of library) { + const code = fs.readFileSync(path.join(SRC, f), 'utf8').replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/.*$/gm, ''); + for (const w of ['document.', 'window.', 'localStorage', 'sessionStorage', 'navigator.', 'HTMLElement']) if (code.includes(w)) bad.push(`${f}: ${w}`); + } + assert.deepEqual(bad, []); +}); + +test('no library module imports a widget, and nothing imports from outside core', () => { + const bad = []; + for (const f of library) { + const code = fs.readFileSync(path.join(SRC, f), 'utf8'); + for (const m of code.matchAll(/(?:from|import)\s*\(?\s*['"](\.{1,2}\/[^'"]+)['"]/g)) { + const target = path.normalize(path.join(path.dirname(f), m[1])); + if (target.startsWith('..')) bad.push(`${f} imports ${m[1]}, outside src/`); + else if (WIDGETS.has(asJs(target))) bad.push(`${f} imports the widget ${m[1]}`); + } + if (/['"]#core\//.test(code)) bad.push(`${f} uses #core/, which is how pplus-web reaches core`); + } + assert.deepEqual(bad, []); +}); diff --git a/tests/build-args.test.mjs b/tests/build-args.test.mjs new file mode 100644 index 0000000..ec4ed22 --- /dev/null +++ b/tests/build-args.test.mjs @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Mojo is built one stated way everywhere Programming+ builds it or asks godbolt to: library/mojo/pin.json's +// build.args, then a program's own. These tests fail when a builder writes build flags of its own, and when the +// TypeScript and Python halves stop agreeing on how the two combine. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { MOJO_BUILD_ARGS } from '../dist/mojo/generated/pin.js'; +import { mojoBuildArgs, BUILD_ARGS, BUILD_LEVELS } from '../dist/services/godbolt.js'; + +const ROOT = new URL('..', import.meta.url).pathname; +const pin = JSON.parse(fs.readFileSync(path.join(ROOT, 'library/mojo/pin.json'), 'utf8')); +const python = spawnSync('python3', ['--version']).status === 0; + +test('the generated pin carries the build arguments of library/mojo/pin.json, each with its reason', () => { + assert.equal(MOJO_BUILD_ARGS, pin.build.args); + // every option in build.args says why it is there, under the option as build.args spells it + const options = pin.build.args.match(/-\S+(?:\s+(?!-)\S+)?/g) ?? []; + assert.deepEqual(Object.keys(pin.build.why).sort(), options.sort()); + for (const why of Object.values(pin.build.why)) assert.ok(why.length > 40, 'a reason, not a label'); +}); + +const CASES = [ + ['', true], ['-O0 --debug-level=line-tables', true], ['--debug-level=line-tables', true], + ['--fp-mode contract=fast', true], ['--fp-mode=contract=fast -O0', true], ['--target-cpu x86-64-v4', true], + ['', false], ['-O0', false], ['--target-cpu apple-m1', false], +]; + +test('a program\'s own arguments come after the pin\'s, and an option it names replaces the pin\'s', () => { + const base = '--target-cpu x86-64-v3 --fp-mode contract=off'; + assert.equal(mojoBuildArgs('', { base }), base); + assert.equal(mojoBuildArgs('-O0 --debug-level=line-tables', { base }), `${base} -O0 --debug-level=line-tables`); + assert.equal(mojoBuildArgs('--fp-mode contract=fast', { base }), '--target-cpu x86-64-v3 --fp-mode contract=fast'); + assert.equal(mojoBuildArgs('--fp-mode=contract=fast', { base }), '--target-cpu x86-64-v3 --fp-mode=contract=fast'); + assert.equal(mojoBuildArgs('--target-cpu x86-64-v4 -O0', { base }), '--fp-mode contract=off --target-cpu x86-64-v4 -O0'); + // a machine that is not x86-64 cannot build for an x86-64 level: that one option is left out + assert.equal(mojoBuildArgs('-O0', { base, x86_64: false }), '--fp-mode contract=off -O0'); + assert.equal(mojoBuildArgs(), MOJO_BUILD_ARGS); +}); + +test('the Compile button sends the pin\'s arguments, then its level\'s', () => { + for (const level of Object.keys(BUILD_LEVELS)) assert.equal(BUILD_ARGS[level], mojoBuildArgs(BUILD_LEVELS[level])); + assert.ok(BUILD_ARGS.O0.startsWith(MOJO_BUILD_ARGS)); +}); + +test('the Python pipeline combines the arguments exactly as the TypeScript does', { skip: !python && !process.env.CI ? 'python3 is not installed' : false }, () => { + const code = `import sys, json; sys.path.insert(0, 'scripts/mojo')\nfrom mojocorpus.common import build_args, load_pin\ncases = json.loads(sys.argv[1])\nprint(json.dumps({"pin": load_pin().build, "out": [build_args(load_pin().build, own, x86_64=x) for own, x in cases]}))`; + const p = spawnSync('python3', ['-c', code, JSON.stringify(CASES)], { encoding: 'utf8', cwd: ROOT }); + assert.equal(p.status, 0, p.stderr); + const got = JSON.parse(p.stdout); + assert.equal(got.pin, MOJO_BUILD_ARGS); + assert.deepEqual(got.out, CASES.map(([own, x]) => mojoBuildArgs(own, { x86_64: x }))); +}); + +/* Every builder, read: no flag that decides the code is written anywhere but the pin. */ + +const BUILDERS = ['scripts', 'src/services']; // their tests (scripts/**/tests) are not builders +// a code-generation option given a value (naming the option, as the code that combines the arguments does, is not) +const CODEGEN = /x86-64-v\d|contract=(?:off|fast|on)\b|--(?:target-cpu|target-features|target-triple|mcpu|march|mtune|fp-mode)(?:=|["']?\s*,?\s*["']?)(?:\S*\d|\S*=|native|generic|haswell|skylake|icelake|alderlake|sapphirerapids|[+-]\w)/; +function files(dir) { + return fs.readdirSync(path.join(ROOT, dir), { withFileTypes: true }).flatMap((e) => e.isDirectory() ? (['__pycache__', 'tests'].includes(e.name) ? [] : files(path.join(dir, e.name))) + : /\.(?:mjs|js|ts|py)$/.test(e.name) ? [path.join(dir, e.name)] : []); +} +const comment = (line) => /^\s*(?:\/\/|\*|\/\*\*?|#)/.test(line); + +test('no builder writes a code-generation flag of its own: they come from library/mojo/pin.json', () => { + const found = []; + for (const f of [...BUILDERS.flatMap(files), 'pixi.toml', 'package.json']) { + fs.readFileSync(path.join(ROOT, f), 'utf8').split('\n').forEach((line, i) => { + if (!comment(line) && CODEGEN.test(line)) found.push(`${f}:${i + 1}: ${line.trim().slice(0, 120)}`); + }); + } + assert.deepEqual(found, [], 'put the flag in library/mojo/pin.json build.args (or a program\'s own # args:) and take it from mojoBuildArgs or build_args'); +}); + +test('every place that runs the compiler on a program gives it the pin\'s arguments', () => { + const sites = []; + for (const f of BUILDERS.flatMap(files)) { + fs.readFileSync(path.join(ROOT, f), 'utf8').split('\n').forEach((line, i) => { + if (comment(line)) return; + if (/['"]mojo['"],\s*\[\s*['"](?:build|run)['"]/.test(line) || /\[\s*["']mojo["'],\s*["'](?:build|run)["']/.test(line)) sites.push({ f, i, line }); + }); + } + // the ones there are today; a new one fails here until it is read and added + assert.deepEqual(sites.map((s) => s.f).sort(), ['scripts/library.mjs', 'scripts/mojo/corpus.py', 'scripts/mojo/fuzz.mjs', 'scripts/mojo/local.mjs']); + for (const s of sites) assert.match(s.line, /\.\.\.(?:BUILD_ARGS|extra|args\.split)|build_args\(/, `${s.f}:${s.i + 1} builds without the pin's arguments`); + // and every request to godbolt carries arguments that went through them + const asks = []; + for (const f of BUILDERS.flatMap(files)) { + fs.readFileSync(path.join(ROOT, f), 'utf8').split('\n').forEach((line, i) => { + if (!comment(line) && /\b(?:request_body|mojoRequestBody)\(/.test(line) && !/^\s*(?:def|export function)\b/.test(line)) asks.push(`${f}:${i + 1}: ${line.trim()}`); + }); + } + assert.equal(asks.length, 4, asks.join('\n')); + for (const a of asks) assert.match(a, /command_args|args=args|BUILD_ARGS\[/, `${a}: a request with arguments that are not the pin's`); + // the tasks a person runs go through corpus.py, which reads the pin + const tasks = fs.readFileSync(path.join(ROOT, 'pixi.toml'), 'utf8'); + assert.doesNotMatch(tasks, /\bmojo (?:build|run)\b/, 'a pixi task runs the compiler directly; use python scripts/mojo/corpus.py mojo-run'); +}); + +/* And run: a stand-in `mojo` on the PATH writes down every argument each builder gives it. */ + +test('the builders hand the compiler exactly the pin\'s arguments and the program\'s own', { skip: process.platform === 'win32' || (!python && !process.env.CI) ? 'needs a POSIX shell and python3' : false }, () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-fake-mojo-')); + try { + const log = path.join(dir, 'calls'); + fs.writeFileSync(path.join(dir, 'mojo'), `#!/bin/sh\nif [ "$1" = "--version" ]; then echo "Mojo ${pin.mojo.version} (stand-in)"; exit 0; fi\nprintf '%s\\n' "$*" >> "${log}"\nexit 1\n`, { mode: 0o755 }); + const env = { ...process.env, PATH: `${dir}:${process.env.PATH}` }; + const host = { x86_64: process.arch === 'x64' }; + const probe = 'library/mojo/probes/godbolt-compile-button.mojo'; + const own = /^# args: (.*)$/m.exec(fs.readFileSync(path.join(ROOT, probe), 'utf8'))[1].trim(); + const calls = () => { + if (!fs.existsSync(log)) return []; + const lines = fs.readFileSync(log, 'utf8').trim().split('\n'); + fs.rmSync(log); + return lines; + }; + const strip = (line) => line.split(' ').filter((w, i, a) => !['build', 'run', '-o', '--disable-warnings'].includes(w) && a[i - 1] !== '-o' && !w.endsWith('.mojo')).join(' '); + + spawnSync(process.execPath, ['scripts/mojo/local.mjs', '--only', 'own', '--item', 'godbolt-compile-button', '--no-cache', '--jobs', '1'], { cwd: ROOT, env, encoding: 'utf8' }); + assert.deepEqual(calls().map(strip), [mojoBuildArgs(own, host)], 'scripts/mojo/local.mjs'); + + spawnSync('python3', ['scripts/mojo/corpus.py', 'mojo-run', probe], { cwd: ROOT, env, encoding: 'utf8' }); + assert.deepEqual(calls().map(strip), [mojoBuildArgs(own, host)], 'corpus.py mojo-run (pixi run mojo-run, mojo-probes-local, play)'); + + spawnSync(process.execPath, ['scripts/mojo/fuzz.mjs', '--seed', '1', '--programs', '1', '--lines', '2', '--jobs', '1'], { cwd: ROOT, env, encoding: 'utf8' }); + const fuzz = calls().map(strip); + assert.ok(fuzz.length >= 1, 'the fuzzer built something'); + for (const c of fuzz) assert.equal(c, mojoBuildArgs('', host), 'scripts/mojo/fuzz.mjs'); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('mojo-local holds a program given stdin to godbolt\'s stored answer too', { skip: process.platform === 'win32' ? 'needs a POSIX shell' : false }, (t) => { + // godbolt gives a program its stdin as a pipe, in run mode as in the Compile button's (probes/input-pipe-first-line + // printed the first line it was given), so its answer is comparable with a local run on the same stdin + const status = JSON.parse(fs.readFileSync(path.join(ROOT, 'library/mojo/golden/status.json'), 'utf8')); + const row = status.items.find((r) => r.id === 'probes/input-pipe-first-line'); + if (!row?.key) return t.skip('no stored answer for probes/input-pipe-first-line'); + const g = JSON.parse(fs.readFileSync(path.join(ROOT, 'library/mojo/golden', `${row.key}.json`), 'utf8')); + assert.ok(g.stdin && g.result.stdout.trim() === '[a b]', 'godbolt printed the first line of the stdin it was given'); + // a stand-in `mojo` whose every build prints what godbolt printed + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-fake-mojo-')); + try { + fs.writeFileSync(path.join(dir, 'mojo'), `#!/bin/sh\nif [ "$1" = "--version" ]; then echo "Mojo ${pin.mojo.version} (stand-in)"; exit 0; fi\nprintf '#!/bin/sh\\necho "[a b]"\\n' > example; chmod +x example\n`, { mode: 0o755 }); + const p = spawnSync(process.execPath, ['scripts/mojo/local.mjs', '--only', 'own', '--item', 'input-pipe-first-line', '--no-cache', '--jobs', '1', '--json'], + { cwd: ROOT, env: { ...process.env, PATH: `${dir}:${process.env.PATH}` }, encoding: 'utf8' }); + assert.equal(JSON.parse(p.stdout).counts.godboltCompared, process.arch === 'x64' ? 1 : 0, p.stderr); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('no listing godbolt answered holds AVX-512, which x86-64-v3 does not have and the model machine does not know', () => { + const dir = path.join(ROOT, 'library/mojo/golden'); + const found = []; + let listings = 0; + for (const f of fs.readdirSync(dir).filter((x) => /^[0-9a-f]{64}\.json$/.test(x))) { + const g = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')); + if (!g.result?.asm?.length) continue; + listings++; + const avx512 = g.result.asm.find((a) => /%zmm\d|%k[0-7]\b|\{%k[0-7]\}/.test(a.text ?? '')); + if (avx512) found.push(`${g.firstItem} (${g.args}): ${avx512.text.trim()}`); + } + assert.ok(listings >= 8, `only ${listings} listings in the corpus`); + assert.deepEqual(found, []); +}); diff --git a/tests/chain.test.mjs b/tests/chain.test.mjs new file mode 100644 index 0000000..f715fef --- /dev/null +++ b/tests/chain.test.mjs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Core pins pplus-library (library/) by commit. A pointer that names a commit the library checkout does not have, or +// a checkout left behind its pointer, tests core against other programs and stored answers than the commit says: +// core's main once pinned a library commit from before the corpus moved, and a fresh clone had no programs at all +// (docs/ci.md). This is the cheap half of pplus-web's `release-check` (its scripts/release.mjs walks all three +// repositories); here only core's own pointer, with no network. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const ROOT = new URL('..', import.meta.url).pathname; +const LIB = path.join(ROOT, 'library'); +const git = (dir, ...args) => { const r = spawnSync('git', ['-C', dir, ...args], { encoding: 'utf8' }); return r.status === 0 ? r.stdout.trim() : null; }; + +test('the library commit core pins is in library/, and library/ is not behind it', (t) => { + const readable = spawnSync('git', ['--version']).status === 0 && fs.existsSync(path.join(ROOT, '.git')); + if (!readable) { + // a package made by package.py has no .git: there is no pointer to read; under CI there always is one + assert.ok(!process.env.CI, 'CI must run this in a git checkout, with git on the PATH'); + t.diagnostic('not a git checkout: no pointer to read'); + return; + } + const pin = /^160000 ([0-9a-f]{40}) /.exec(git(ROOT, 'ls-files', '--stage', '--', 'library') ?? '')?.[1]; + assert.ok(pin, 'core records no commit for library/ (.gitmodules and the index disagree)'); + assert.ok(fs.existsSync(path.join(LIB, '.git')), 'library/ is not checked out. Fix: git submodule update --init'); + assert.notEqual(git(LIB, 'cat-file', '-e', `${pin}^{commit}`), null, + `core pins library at ${pin.slice(0, 10)}, and library/ does not have that commit. Fix: git submodule update --init (it fetches the pinned commit); if no remote has it, it was never pushed: push pplus-library from the machine that made it`); + const head = git(LIB, 'rev-parse', 'HEAD'); + if (head === pin) return; + assert.equal(git(LIB, 'merge-base', '--is-ancestor', head, pin), null, + `library/ is at ${head.slice(0, 10)}, behind the pinned ${pin.slice(0, 10)}, so every test here reads an older library. Fix: git submodule update`); + if (git(LIB, 'merge-base', '--is-ancestor', pin, head) !== null) t.diagnostic(`library/ is ahead of the pinned ${pin.slice(0, 10)}: when the change is done, commit the new pointer (git add library) with what core regenerates (pixi run library-emit, mojo-emit)`); + else assert.fail(`library/ is at ${head.slice(0, 10)}, which is neither the pinned ${pin.slice(0, 10)} nor after it. Fix: git submodule update, or, if ${head.slice(0, 10)} is the one wanted, git add library`); +}); diff --git a/tests/corpus.test.mjs b/tests/corpus.test.mjs new file mode 100644 index 0000000..69508ec --- /dev/null +++ b/tests/corpus.test.mjs @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The pseudo stress corpus: 37 programs across domains (numbers, text, finance, graphs, simulation, +// statistics, cryptography, records, sets, recursion), plus runs that must stop and mistakes the checker +// must refuse. Each file states what it expects in header comments. See docs/pseudo-stress-test.md. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { studyOf, runPseudo } from '../dist/services/pseudo.js'; + +const dir = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'library', 'pseudo', 'programs'); +const files = fs.readdirSync(dir).filter((f) => f.endsWith('.pseudo')).sort(); + +for (const f of files) { + test(`corpus: ${f}`, () => { + const src = fs.readFileSync(path.join(dir, f), 'utf8'); + const expect = [...src.matchAll(/^\/\/ EXPECT: (.*)$/gm)].map((m) => m[1]); + const input = [...src.matchAll(/^\/\/ INPUT: (.*)$/gm)].map((m) => m[1]); + const stops = /^\/\/ STOPS: (\w+)$/m.exec(src)?.[1]; + const errorsWanted = /^\/\/ ERRORS: (\d+)$/m.exec(src)?.[1]; + const st = studyOf(src); + const errs = st.diagnostics.filter((d) => d.severity === 'error'); + if (errorsWanted) { assert.equal(errs.length, Number(errorsWanted), errs.map((d) => `L${d.line} ${d.id}`).join(', ')); return; } + assert.ok(st.ok, errs.map((d) => `L${d.line} ${d.id}: ${d.message}`).join(' | ')); + const tr = runPseudo(st, { input, maxSteps: 200000 }); + if (stops) { assert.equal(tr.error?.kind, stops, tr.error?.message || 'no error'); return; } + assert.equal(tr.error, null, tr.error?.message); + assert.deepEqual(tr.output.filter((l) => !/^\w+ returned /.test(l)), expect); + }); +} + +test('the corpus covers every statement kind and every built-in group', () => { + const all = files.map((f) => fs.readFileSync(path.join(dir, f), 'utf8')).join('\n'); + for (const kw of ['DECLARE', 'CONSTANT', 'TYPE', 'DEFINE', 'IF', 'ELSEIF', 'ELSE', 'CASE', 'OTHERWISE', 'FOR', 'STEP', 'FOR EACH', 'WHILE', 'REPEAT', 'UNTIL', 'BREAK', 'CONTINUE', 'FUNCTION', 'PROCEDURE', 'BYREF', 'CALL', 'RETURN', 'INPUT', 'OUTPUT', 'REQUIRE', 'INVARIANT', 'DECREASES', 'FORALL', 'EXISTS', 'SUM', 'DIV', 'MOD', 'IN', 'ARRAY', 'LIST', 'HASHMAP', 'SET', 'PARALLEL FOR']) { + assert.ok(new RegExp(`\\b${kw}\\b`).test(all), `corpus never uses ${kw}`); + } + for (const b of ['LENGTH', 'ROWS', 'INT', 'ROUND', 'ABS', 'MIN', 'MAX', 'SLICE', 'UPPER', 'LOWER', 'ORD', 'CHR', 'TO_STRING', 'TO_INTEGER', 'KEYS', 'SORTED', 'APPEND', 'REMOVE', 'ADD', 'SORT', 'RANDINT', 'SPLIT', 'JOIN', 'INDEX_OF', 'REPLACE', 'FLOOR', 'CEIL', 'LOG10', 'SIN', 'COS', 'ATAN2', 'PI', 'CHOOSE', 'SUM_OF', 'MEAN_OF', 'VARIANCE_OF', 'STDEV_OF', 'MEDIAN_OF', 'DOT', 'NORM_OF', 'FILL']) assert.ok(all.includes(b + '('), `corpus never calls ${b}`); +}); diff --git a/tests/curriculum.test.mjs b/tests/curriculum.test.mjs new file mode 100644 index 0000000..fd05dee --- /dev/null +++ b/tests/curriculum.test.mjs @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The lesson corpus verifies the interpreter: the dependency map must be strict and every program must run. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { LESSONS, CONCEPTS, CONCEPT_BY_ID, FLAVORS, teacherOf, prerequisitesOf, depthOf, allPrerequisitesOf, statusOf, solutionOf, starterOf } from '../dist/curriculum/index.js'; +import { studyOf, runPseudo } from '../dist/services/pseudo.js'; +import { problemById } from '../dist/common/problems.js'; + +const conceptIds = new Set(CONCEPTS.map((c) => c.id)); +const flavorIds = new Set(FLAVORS.map((f) => f.id)); + +test('lesson ids are unique and every concept id is real', () => { + const ids = new Set(); + for (const L of LESSONS) { + assert.ok(!ids.has(L.id), `duplicate lesson id ${L.id}`); ids.add(L.id); + for (const c of [...L.teaches, ...L.requires]) assert.ok(conceptIds.has(c), `${L.id} names unknown concept ${c}`); + assert.ok(L.teaches.length >= 1, `${L.id} teaches nothing`); + assert.ok(L.purpose.length > 20 && L.explanation.length >= 3 && L.checks.length >= 2 && L.flavors.length >= 2, `${L.id} is thin`); + for (const f of L.flavors) assert.ok(flavorIds.has(f.id), `${L.id} has unknown flavor ${f.id}`); + for (const c of L.checks) assert.ok(c.answer >= 0 && c.answer < c.options.length, `${L.id} check answer out of range`); + for (const p of L.problems || []) assert.ok(problemById(p), `${L.id} points at missing problem ${p}`); + if (L.showAsmOf) assert.ok(problemById(L.showAsmOf)?.sources.asm, `${L.id} shows asm of a problem without any`); + } +}); + +test('every concept is taught by exactly one lesson, and every concept is used', () => { + for (const c of CONCEPTS) { + const teachers = LESSONS.filter((l) => l.teaches.includes(c.id)); + assert.equal(teachers.length, 1, `${c.id} is taught by ${teachers.map((l) => l.id).join(', ') || 'nobody'}`); + } + // every foundations and core concept must feed a later lesson; only advanced concepts may be frontier + const required = new Set(LESSONS.flatMap((l) => l.requires)); + for (const L of LESSONS.filter((l) => l.stage !== 'advanced')) for (const c of L.teaches) assert.ok(required.has(c), `${c} (from ${L.id}) is never required by a later lesson`); +}); + +test('the dependency map is strict: every requirement is taught by an earlier lesson, and there are no cycles', () => { + const seen = new Set(); + for (const L of LESSONS) { + for (const c of L.requires) { + const t = teacherOf(c); + assert.ok(t, `${L.id} requires ${c}, which nothing teaches`); + assert.ok(seen.has(t.id), `${L.id} requires ${c}, taught by ${t.id}, which comes later in the order`); + assert.ok(!L.teaches.includes(c), `${L.id} both teaches and requires ${c}`); + } + seen.add(L.id); + } + for (const L of LESSONS) assert.ok(depthOf(L) >= 0, `${L.id} is in a cycle`); + const roots = LESSONS.filter((l) => l.requires.length === 0); + assert.equal(roots.length, 1, `exactly one lesson should assume nothing: ${roots.map((l) => l.id).join(', ')}`); + for (const L of LESSONS.slice(1)) assert.ok(allPrerequisitesOf(L).some((p) => p.id === roots[0].id), `${L.id} does not trace back to ${roots[0].id}`); +}); + +test('a lesson is available exactly when every prerequisite lesson is done', () => { + const none = { done: [], checks: {} }; + const first = LESSONS[0]; + assert.equal(statusOf(first, none), 'available'); + assert.equal(statusOf(LESSONS[LESSONS.length - 1], none), 'locked'); + const last = LESSONS[LESSONS.length - 1]; + const all = { done: allPrerequisitesOf(last).map((l) => l.id), checks: {} }; + assert.equal(statusOf(last, all), 'available'); + const direct = prerequisitesOf(last)[0].id; + const butOne = { done: all.done.filter((id) => id !== direct), checks: {} }; + assert.notEqual(statusOf(last, butOne), 'available'); + const order = ['foundations', 'core', 'advanced']; + for (const L of LESSONS) for (const p of prerequisitesOf(L)) assert.ok(order.indexOf(p.stage) <= order.indexOf(L.stage), `${L.id} depends on a later stage`); +}); + +function runProgram(label, prog) { + const st = studyOf(solutionOf(prog)); + const errs = st.diagnostics.filter((d) => d.severity === 'error'); + assert.equal(errs.length, 0, `${label}: ${errs.map((d) => `L${d.line} ${d.id}: ${d.message}`).join(' | ')}`); + const tr = runPseudo(st, { input: prog.input || [], maxSteps: 200000 }); + const out = tr.output.filter((l) => !/^\w+ returned /.test(l)); + if (prog.stops) { assert.equal(tr.error?.kind, prog.stops, `${label}: expected a ${prog.stops} stop, got ${tr.error?.kind || 'none'}`); } + else assert.equal(tr.error, null, `${label}: ${tr.error?.message}`); + assert.deepEqual(out, prog.expect, label); + return st; +} + +for (const L of LESSONS) { + test(`lesson ${L.id}: the example and every flavor run and print what they say`, () => { + runProgram(`${L.id}/example`, L.example); + for (const f of L.flavors) { + runProgram(`${L.id}/${f.id}`, f.solution); + const starter = f.starter ?? starterOf(f.solution); + assert.ok(starter.length > 0); + if (!f.starter) assert.ok(/\/\/ TODO:/.test(starter) || !/\/\/ ~/.test(f.solution.pseudo), `${L.id}/${f.id}: starter has no blanks`); + assert.ok(!/\/\/ ~/.test(starter), `${L.id}/${f.id}: marker leaked into the starter`); + assert.ok(!/\/\/ ~/.test(solutionOf(f.solution)), `${L.id}/${f.id}: marker leaked into the solution`); + } + }); +} + +test('a starter blanks exactly the marked lines', () => { + const p = { pseudo: 'A <- 1\nB <- 2 // ~ set B\nC <- 3', expect: [] }; + assert.equal(starterOf(p), 'A <- 1\n// TODO: set B\nC <- 3'); + assert.equal(solutionOf(p), 'A <- 1\nB <- 2\nC <- 3'); +}); + +test('every lesson uses only keywords its prerequisites (or it) have introduced', () => { + // a rough but real check: the statement keyword that introduces a concept must not appear + // in a lesson that neither teaches nor (transitively) requires that concept + const introduces = { + 'IF': 'selection', 'FOR': 'for-loop', 'WHILE': 'while', 'REPEAT': 'repeat', 'FUNCTION': 'function', 'PROCEDURE': 'procedure-sub', 'BYREF': 'byref', 'CASE': 'case', + 'HASHMAP': 'hashmap', 'SET': 'set', 'TYPE': 'record', 'ARRAY': 'array', 'INVARIANT': 'invariant', 'ENSURE': 'specification', 'BREAK': 'early-exit', 'CONTINUE': 'early-exit', 'INPUT': 'io', + }; + const exempt = { 'sets': ['TYPE'], 'what-stops-a-run': ['FUNCTION'] }; + const violations = []; // TYPE … = SET OF is the set lesson's own form; the stops lesson reviews a function + for (const L of LESSONS) { + const held = new Set([...L.teaches, ...allPrerequisitesOf(L).flatMap((p) => p.teaches), ...L.requires]); + const programs = [L.example.pseudo, ...L.flavors.flatMap((f) => [f.solution.pseudo, f.starter || ''])].join('\n'); + for (const [kw, concept] of Object.entries(introduces)) { + if ((exempt[L.id] || []).includes(kw)) continue; + if (new RegExp(`(^|[^\\w])${kw}\\b`, 'm').test(programs.replace(/"[^"]*"/g, '""').replace(/\/\/.*$/gm, '')) && !held.has(concept)) violations.push(`${L.id} uses ${kw} but neither teaches nor requires ${concept}`); + } + } + assert.deepEqual(violations, []); +}); diff --git a/tests/dafny.test.mjs b/tests/dafny.test.mjs new file mode 100644 index 0000000..ddf5d83 --- /dev/null +++ b/tests/dafny.test.mjs @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The pinned Dafny (scripts/dafny.py): its offline tests, and, when it has been fetched, that it is the pinned +// version and verifies a program. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +// Under CI (the CI variable set, as Gitea and GitHub Actions set it) a missing tool fails these instead of skipping +// them: a skip in CI is how a gap hides. pixi brings python; CI fetches the pinned Dafny (pixi run dafny-fetch). +const py = spawnSync('python3', ['--version']); +const ROOT = new URL('..', import.meta.url).pathname; + +test('the Dafny pin\'s own tests pass (python3)', { skip: py.status !== 0 && !process.env.CI ? 'python3 is not installed' : false }, () => { + assert.equal(py.status, 0, 'python3 is not installed, and CI must run this'); + const r = spawnSync('python3', ['-m', 'unittest', 'discover', '-s', 'scripts/tests'], { encoding: 'utf8', cwd: ROOT }); + assert.equal(r.status, 0, r.stderr); +}); + +const exe = ROOT + '.tools/dafny-4.11.0/dafny/dafny'; +test('the fetched Dafny is the pinned one, and verifies a program', { skip: fs.existsSync(exe) || process.env.CI ? false : 'Dafny has not been fetched (pixi run dafny-fetch)' }, () => { + assert.ok(fs.existsSync(exe), 'Dafny has not been fetched, and CI must run this: pixi run dafny-fetch'); + const v = spawnSync(exe, ['--version'], { encoding: 'utf8' }); + assert.match(v.stdout, /^4\.11\.0\+/); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-dafny-')); + fs.writeFileSync(dir + '/max.dfy', 'method Max(a: int, b: int) returns (m: int) ensures m >= a && m >= b { if a > b { m := a; } else { m := b; } }\n'); + const r = spawnSync(exe, ['verify', dir + '/max.dfy'], { encoding: 'utf8' }); + fs.rmSync(dir, { recursive: true, force: true }); + assert.equal(r.status, 0, r.stdout + r.stderr); + assert.match(r.stdout, /1 verified, 0 errors/); +}); diff --git a/tests/documents.test.mjs b/tests/documents.test.mjs new file mode 100644 index 0000000..5f67c31 --- /dev/null +++ b/tests/documents.test.mjs @@ -0,0 +1,240 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// What the documents say, held to what the files and the tools say. The README files, AGENTS.md files and the +// hand-written documents in docs/ make claims a stranger reads first: the one-sentence description, what each +// notation is, the library's modules and its example program, the library's own rules for its routines, and the +// assembly census. Each test here fails when one of those claims stops being true, so a document cannot drift +// from what it describes. (docs/sync-coverage.md is written by its own test and is not read here.) +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { studyOf, runPseudo } from '../dist/services/pseudo.js'; +import { census } from '../scripts/asm-census.mjs'; + +const CORE = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const LIB = path.join(CORE, 'library'); +const read = (p) => fs.readFileSync(path.join(CORE, p), 'utf8'); + +/** The documents that describe the project in prose, in both repositories. */ +const DOCUMENTS = ['README.md', 'AGENTS.md', 'docs/trust.md', 'docs/library.md', 'docs/exercises.md', 'docs/asm-coverage.md', 'library/README.md', 'library/AGENTS.md']; +const SENTENCE = 'Programming+ takes a procedure from an idea to Mojo to the machine.'; +/** What each name the project made up means, in the words every document uses the first time it says the name. */ +const GLOSS = { + lingo: 'steps in plain English', + matho: 'equations written in LaTeX', + pseudo: 'pseudocode precise enough to run, prove and translate', + speco: 'a specification to prove against', +}; +const MOJO_GLOSS = "Modular's programming language, pinned to version 1.2.0"; + +/** A document's prose: no code blocks, no code spans, no link targets, no bold or italic marks. */ +function prose(text) { + return text.replace(/```[\s\S]*?```/g, ' ').replace(/`[^`\n]*`/g, ' ').replace(/\]\([^)]*\)/g, ']').replace(/\*+/g, ''); +} + +test('every document that describes the project starts from the same sentence', () => { + for (const doc of DOCUMENTS.filter((d) => !d.endsWith('asm-coverage.md'))) { + assert.ok(read(doc).includes(SENTENCE), `${doc} does not say "${SENTENCE}"`); + } +}); + +test('each name the project made up is glossed, in the same words, where a document first uses it', () => { + for (const doc of DOCUMENTS) { + const text = prose(read(doc)); + for (const [name, gloss] of Object.entries(GLOSS)) { + const m = new RegExp(`(?<![\\w./-])${name}(?![\\w/-])`, 'i').exec(text); + if (!m) continue; + const after = text.slice(m.index + name.length, m.index + name.length + gloss.length + 8).replace(/^[\s:(]+/, ''); + assert.ok(after.startsWith(gloss), `${doc}: the first "${name}" is not followed by its gloss "${gloss}" but by "${after}"`); + } + } +}); + +test('the two README files list the five notations in the same words', () => { + const notations = (doc) => read(doc).split('\n').filter((l) => /^- \*\*(lingo|matho|pseudo|speco|Mojo)\*\*: /.test(l)); + const core = notations('README.md'); + assert.equal(core.length, 5, 'README.md does not list the five notations'); + assert.deepEqual(notations('library/README.md'), core); + for (const [name, gloss] of Object.entries(GLOSS)) assert.ok(core.some((l) => l.startsWith(`- **${name}**: ${gloss}`)), `README.md: ${name}`); + assert.ok(core.some((l) => l.startsWith(`- **Mojo**: ${MOJO_GLOSS}`)), 'README.md: Mojo'); + for (const doc of ['AGENTS.md', 'docs/trust.md', 'docs/exercises.md']) assert.ok(read(doc).includes(MOJO_GLOSS), `${doc} does not say what Mojo is in the same words`); +}); + +test("the library README's module table is the library's modules and routines", () => { + const rows = [...read('library/README.md').matchAll(/^\| `(\w+)` \| (.*) \|$/gm)]; + const table = Object.fromEntries(rows.map(([, mod, cell]) => [mod, cell.replace(/\([^)]*\)/g, '').split(',').map((x) => x.trim().split(' ')[0]).sort()])); + const disk = {}; + for (const mod of fs.readdirSync(path.join(LIB, 'modules'))) { + const dir = path.join(LIB, 'modules', mod); + if (!fs.existsSync(path.join(dir, 'module.json'))) continue; + disk[mod] = fs.readdirSync(dir).filter((r) => fs.existsSync(path.join(dir, r, 'routine.json'))) + .map((r) => JSON.parse(fs.readFileSync(path.join(dir, r, 'routine.json'), 'utf8')).name).sort(); + } + assert.deepEqual(table, disk); +}); + +test("the library README's USE example prints what the README says it prints", () => { + const readme = read('library/README.md'); + const m = /A program uses a routine by name\.[^`]*```\n([\s\S]*?)```\n\nprints `([^`]*)`/.exec(readme); + assert.ok(m, 'the example and its printed line are not where this test looks'); + const tr = runPseudo(studyOf(m[1]), {}); + assert.equal(tr.error, null, tr.error?.message); + assert.equal(tr.output.join('\n'), m[2]); +}); + +test("the core README's prove example ends with the line the README quotes", () => { + const m = /pixi run cli prove (\S+)\s+# .*the last line is "([^"]+)"/.exec(read('README.md')); + assert.ok(m, 'the prove example is not where this test looks'); + const r = spawnSync(process.execPath, ['scripts/cli.mjs', 'prove', m[1]], { cwd: CORE, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout.trim().split('\n').pop(), m[2]); +}); + +/** Every routine of the library, with its directory, its routine.json and its routine.pseudo. */ +function routines() { + const out = []; + for (const mod of fs.readdirSync(path.join(LIB, 'modules'))) { + const dir = path.join(LIB, 'modules', mod); + if (!fs.statSync(dir).isDirectory()) continue; + for (const r of fs.readdirSync(dir)) { + const file = path.join(dir, r, 'routine.json'); + if (!fs.existsSync(file)) continue; + const json = JSON.parse(fs.readFileSync(file, 'utf8')); + out.push({ id: `${mod}.${json.name}`, dir: path.join(dir, r), json, pseudo: fs.readFileSync(path.join(dir, r, 'routine.pseudo'), 'utf8') }); + } + } + assert.ok(out.length > 0, 'no routines: is library/ checked out?'); + return out; +} + +test('library rule 6: every routine cites a paper by DOI, a book by edition and section, or a page by URL', () => { + for (const r of routines()) { + assert.ok(Array.isArray(r.json.sources) && r.json.sources.length, `${r.id} has no sources`); + for (const s of r.json.sources) { + const title = s.title || ''; + assert.doesNotMatch(title, /^Any\b/, `${r.id}: "${title}" names no source`); + const linked = typeof s.url === 'string' && /^https:\/\//.test(s.url); + const book = /\b(sections?|formula|chapter)\b/i.test(title) && /\b(edition|ed\.)|\b(1[89]|20)\d\d\b/i.test(title); + assert.ok(linked || book, `${r.id}: "${title}" has neither a URL nor an edition and section`); + } + } +}); + +test('library rule 5: every tolerance has a note that says why', () => { + for (const r of routines()) { + for (const ref of r.json.references || []) { + if (ref.rel === undefined && ref.abs === undefined) continue; + assert.ok((ref.note || '').length > 40, `${r.id} ${ref.lang}/${ref.level} has a tolerance and no note that explains it`); + } + } +}); + +test("library rule 4: a routine the prover leaves open says so in its comment", () => { + const status = JSON.parse(fs.readFileSync(path.join(LIB, 'status.json'), 'utf8')).routines; + const open = routines().filter((r) => status[r.id]?.checks?.prover?.status === 'open'); + for (const r of open) { + const comment = r.pseudo.split('\n').filter((l) => l.startsWith('//')).join(' '); + assert.match(comment, /\bobligation\b[^.]*\bopen\b|\bopen\b[^.]*\bobligation\b/, `${r.id} is recorded open, and its comment does not say which obligation or why`); + } +}); + +/** British spellings, and words US English spells or uses differently. ("analyses" is left out: it is also the plural of analysis.) */ +const BRITISH = /\b(behaviour|colour|favour|honour|labell(ed|ing)|modell(ed|ing)|travell(ed|ing)|cancell(ed|ing)|neighbour\w*|optimis\w*|memois\w*|normalis\w*|initialis\w*|serialis\w*|organis\w*|recognis\w*|minimis\w*|maximis\w*|summaris\w*|utilis\w*|realise[ds]?|analysed?|analysing|licence|centre|catalogue|grey|whilst|amongst|towards)\b/i; + +test("library rule 8 and core's: the documents and the routines' text are in US English", () => { + const texts = DOCUMENTS.map((d) => [d, read(d)]); + for (const r of routines()) { + const j = r.json; + texts.push([`${r.id} routine.json`, [j.summary, ...(j.tags || []), ...(j.references || []).map((x) => x.note), j.proof?.note].join('\n')]); + texts.push([`${r.id} routine.pseudo`, r.pseudo.split('\n').filter((l) => l.trim().startsWith('//')).join('\n')]); + } + for (const mod of fs.readdirSync(path.join(LIB, 'modules'))) { + const f = path.join(LIB, 'modules', mod, 'module.json'); + if (fs.existsSync(f)) texts.push([`${mod}/module.json`, fs.readFileSync(f, 'utf8')]); + } + for (const [where, text] of texts) { + const m = BRITISH.exec(text); + assert.equal(m, null, `${where}: "${m?.[0]}"`); + } +}); + +test('the sources, the scripts, the tests and the regression programs are in US English too', () => { + // what is written here, not what is generated, harvested or stored: a comment, a message a student reads and a + // name in the code are all the project's own words + const LEFT_OUT = [/^src\/mojo\/generated\//, /^src\/library\/generated\.ts$/, /^tests\/documents\.test\.mjs$/, /^tests\/snapshots\//, /^scripts\/mojo\/tests\/fixtures\//]; + // words a student may write, which a reader has to know: not our prose + const ALLOWED = { 'src/engines/pseudo/english.js': /initialise/g }; + const files = []; + const walk = (dir) => { + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + const f = path.join(dir, e.name); + if (e.isDirectory()) { if (!['node_modules', '__pycache__'].includes(e.name)) walk(f); } + else if (/\.(ts|js|mjs|py|sh|pseudo|lingo|speco|tex|md|toml|yml|mojo)$/.test(e.name)) files.push(f); + } + }; + for (const d of ['src', 'scripts', 'tests', '.gitea', path.join('library', 'pseudo'), path.join('library', 'modules'), ...['local', 'probes', 'stress'].map((x) => path.join('library', 'mojo', x))]) if (fs.existsSync(d)) walk(d); + assert.ok(files.length > 300, `${files.length} files read`); + const found = []; + for (const f of files) { + const rel = f.split(path.sep).join('/'); + if (LEFT_OUT.some((r) => r.test(rel))) continue; + let text = fs.readFileSync(f, 'utf8'); + if (ALLOWED[rel]) text = text.replace(ALLOWED[rel], ''); + const m = BRITISH.exec(text); + if (m) found.push(`${rel}: "${m[0]}"`); + } + assert.deepEqual(found, []); +}); + +test('the Mojo corpus has a stored answer for every program but the compiler\'s opt-in diagnostic tests and those that do not run', () => { + // what README.md and library/README.md say about godbolt's stored answers + const items = JSON.parse(fs.readFileSync(path.join(LIB, 'mojo', 'golden', 'status.json'), 'utf8')).items; + for (const it of items) { + if (it.corpus === 'diagnostics') continue; + assert.notEqual(it.verdict, 'unverified', `${it.id} has no stored answer`); + if (it.verdict === 'skipped') assert.match(it.why, /no main/, `${it.id} is skipped: ${it.why}`); + } +}); + +test("docs/asm-coverage.md's table is today's assembly census", () => { + const doc = read('docs/asm-coverage.md'); + const now = (label) => { + const m = new RegExp(`^\\| ${label} \\| [^|]* \\| ([^|]*) \\|$`, 'm').exec(doc); + assert.ok(m, `no "${label}" row`); + return m[1].trim(); + }; + const c = census(); + assert.equal(now('distinct instructions in the listings'), String(c.ops.size)); + assert.match(now("of which outside the model's set"), new RegExp(`^${c.missing.length} `)); + assert.equal(now("the user's own functions that assemble"), `${c.userOk} of ${c.user}`); + assert.equal(now('standard-library and runtime functions that assemble'), `${c.libOk} of ${c.lib}`); +}); + +test('TypeScript is the only npm package, as README.md and AGENTS.md say', () => { + const pkg = JSON.parse(read('package.json')); + assert.equal(pkg.dependencies, undefined, 'package.json has runtime dependencies'); + assert.deepEqual(Object.keys(pkg.devDependencies || {}), ['typescript']); +}); + +test('no document sends a reader to a file git does not track (the owner\'s notes in work/)', () => { + const tracked = (dir) => spawnSync('git', ['-C', dir, 'ls-files', '*.md'], { encoding: 'utf8' }).stdout.split('\n').filter(Boolean); + const docs = [...tracked(CORE).filter((f) => !f.startsWith('library/')), ...tracked(LIB).map((f) => `library/${f}`)]; + assert.ok(docs.length >= 20, `${docs.length} tracked documents`); + for (const doc of docs) { + for (const m of read(doc).matchAll(/`(work\/[^`\s]*\.md)`/g)) assert.fail(`${doc} names ${m[1]}, which is git-ignored and absent from a clone`); + } +}); + +test("docs/ci.md names every task `pixi run ci` runs, and mojo-corpus.md does not say CI leaves out the compiler's tasks", () => { + const ci = /^ci = \{ depends-on = \[([^\]]*)\]/m.exec(read('pixi.toml')); + assert.ok(ci, 'pixi.toml has no ci task with depends-on'); + const tasks = [...ci[1].matchAll(/"([\w-]+)"/g)].map((m) => m[1]); + assert.ok(tasks.includes('mojo-local'), 'the ci task no longer runs mojo-local: update docs/ci.md and docs/mojo-corpus.md'); + const line = read('docs/ci.md').split('\n').find((l) => l.startsWith('`pixi run ci` is ')); + assert.ok(line, 'docs/ci.md has no line that says what `pixi run ci` is'); + for (const t of tasks) assert.ok(line.includes(`\`${t}\``), `docs/ci.md does not say that pixi run ci runs ${t}`); + assert.doesNotMatch(read('docs/mojo-corpus.md'), /(?:not|never|neither) runs? in CI/, 'docs/mojo-corpus.md says a task of ci does not run in CI'); +}); diff --git a/tests/engine-solver.test.mjs b/tests/engine-solver.test.mjs new file mode 100644 index 0000000..4b91689 --- /dev/null +++ b/tests/engine-solver.test.mjs @@ -0,0 +1,219 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The engines' own tests: the decision procedure (src/engines/pseudo/solver.js) and the proof +// obligations (vcgen.js). This repository owns the engines (docs/engines.md); these tests are +// what a change to either file answers to, next to tests/prover-soundness.test.mjs. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { prove } from '../dist/engines/pseudo/solver.js'; +import { + sym, num, lit, TRUE, FALSE, and, or, not, cmp, ite, add, sub, mul, arith, at, length, app, +} from '../dist/engines/pseudo/logic.js'; +import { verify } from '../dist/engines/pseudo/vcgen.js'; +import { studyOf } from '../dist/services/pseudo.js'; + +const x = sym('X', 'int'); const y = sym('Y', 'int'); const z = sym('Z', 'int'); +const i = sym('I', 'int'); const j = sym('J', 'int'); +const t = sym('T', 'string'); +const f = sym('F', 'float'); const g = sym('G', 'float'); +const n = (v) => num(v, 'int'); + +/* ---------- a table: what it must prove, and what it must leave unknown ---------- */ + +const MUST_PROVE = [ + ['order is transitive', [cmp('<', x, y), cmp('<', y, z)], cmp('<', x, z)], + ['whole numbers: X > 0 means X >= 1', [cmp('>', x, n(0))], cmp('>=', x, n(1))], + ['a case split closes', [or(cmp('>=', x, n(0)), cmp('<', x, n(0)))], cmp('>=', ite(cmp('>=', x, n(0)), x, sub(n(0), x)), n(0))], + ['equal positions hold equal letters', [cmp('==', i, j)], cmp('==', at(t, i, 'char'), at(t, j, 'char'))], + ['DIV by a positive constant never grows a non-negative number', [cmp('>=', x, n(0))], cmp('<=', arith('DIV', x, n(2), 'int'), x)], + ['MOD by 3 lies in [0, 3)', [], and(cmp('<=', n(0), arith('MOD', x, n(3), 'int')), cmp('<', arith('MOD', x, n(3), 'int'), n(3)))], + ['a length is never negative', [], cmp('>=', length(t), n(0))], + ['contradictory facts prove anything', [cmp('<', x, n(0)), cmp('>', x, n(0))], cmp('==', x, n(7))], + ['a position inside the bounds stays inside after a step', [cmp('<=', n(0), i), cmp('<', add(i, n(1)), length(t))], cmp('<', i, length(t))], +]; + +const MUST_NOT_PROVE = [ + ['X < Y does not give X < Y - 1', [cmp('<', x, y)], cmp('<', x, sub(y, n(1)))], + ['a FLOAT above 0 need not be at least 1', [cmp('>', f, num(0, 'float'))], cmp('>=', f, num(1, 'float'))], + ['nothing proves FALSE', [], FALSE], + // A known gap, not a fault: equations are not tightened for whole numbers, so 2X == 1 is not + // seen to be impossible. When the solver learns it, move this line to MUST_PROVE. + ['known gap: 2X == 1 is not seen to have no whole-number solution', [cmp('==', mul(n(2), x), n(1))], FALSE], + ['X MOD 3 need not be 0', [], cmp('==', arith('MOD', x, n(3), 'int'), n(0))], + ['a product of two unknowns is opaque, not non-negative', [], cmp('>=', mul(x, y), n(0))], + ['rounded FLOAT addition is opaque', [], cmp('==', app('FLOAT+', [f, g], 'float'), app('FLOAT+', [g, f], 'float'))], + ['different positions may hold different letters', [cmp('!=', i, j)], cmp('==', at(t, i, 'char'), at(t, j, 'char'))], + ['a letter read from text is not a given letter', [], cmp('==', at(t, i, 'char'), lit('A', 'char'))], +]; + +test('the solver proves what follows', () => { + for (const [name, facts, goal] of MUST_PROVE) assert.equal(prove(facts, goal).status, 'proved', name); +}); + +test('the solver leaves unknown what does not follow, or what it cannot see into', () => { + for (const [name, facts, goal] of MUST_NOT_PROVE) assert.notEqual(prove(facts, goal).status, 'proved', name); + assert.equal(prove([], TRUE).status, 'proved'); +}); + +/* ---------- random formulas, checked against every small assignment ---------- */ + +/** Evaluate a term over whole-number values for the symbols; undefined when the term leaves that fragment. */ +function evaluate(e, env) { + switch (e.k) { + case 'num': return e.v; + case 'lit': return e.v; + case 'sym': return env.get(e.id); + case 'op': { + const a = e.args.map((u) => evaluate(u, env)); + if (a.some((v) => v === undefined)) return undefined; + switch (e.op) { + case '+': return a[0] + a[1]; + case '-': return a[0] - a[1]; + case '*': return a[0] * a[1]; + case 'neg': return -a[0]; + case 'DIV': return a[1] === 0 ? undefined : Math.floor(a[0] / a[1]); + case 'MOD': return a[1] === 0 ? undefined : a[0] - a[1] * Math.floor(a[0] / a[1]); + default: return undefined; + } + } + case 'cmp': { + const a = evaluate(e.a, env); const b = evaluate(e.b, env); + if (a === undefined || b === undefined) return undefined; + return { '==': a === b, '!=': a !== b, '<': a < b, '<=': a <= b, '>': a > b, '>=': a >= b }[e.op]; + } + case 'and': { let r = true; for (const u of e.args) { const v = evaluate(u, env); if (v === undefined) return undefined; r = r && v; } return r; } + case 'or': { let r = false; for (const u of e.args) { const v = evaluate(u, env); if (v === undefined) return undefined; r = r || v; } return r; } + case 'not': { const v = evaluate(e.a, env); return v === undefined ? undefined : !v; } + case 'ite': { const c = evaluate(e.c, env); if (c === undefined) return undefined; return evaluate(c ? e.a : e.b, env); } + default: return undefined; + } +} + +function rng(seed) { + let s = seed >>> 0; + return () => { s = (s + 0x6D2B79F5) >>> 0; let r = Math.imul(s ^ (s >>> 15), 1 | s); r ^= r + Math.imul(r ^ (r >>> 7), 61 | r); return ((r ^ (r >>> 14)) >>> 0) / 4294967296; }; +} + +test('the solver never proves a formula that a small assignment breaks (random linear integer formulas)', () => { + const r = rng(2026); + const pick = (xs) => xs[Math.floor(r() * xs.length)]; + const vars = [x, y, z]; + const term = (d) => { + const roll = r(); + if (d <= 0 || roll < 0.3) return r() < 0.6 ? pick(vars) : n(Math.floor(r() * 7) - 3); + if (roll < 0.5) return add(term(d - 1), term(d - 1)); + if (roll < 0.65) return sub(term(d - 1), term(d - 1)); + if (roll < 0.75) return mul(n(Math.floor(r() * 5) - 2), term(d - 1)); + if (roll < 0.85) return arith(pick(['DIV', 'MOD']), term(d - 1), n(pick([2, 3])), 'int'); + return ite(atom(d - 1), term(d - 1), term(d - 1)); + }; + const atom = (d) => cmp(pick(['==', '!=', '<', '<=', '>', '>=']), term(d), term(d)); + const formula = (d) => { + const roll = r(); + if (d <= 0 || roll < 0.5) return atom(1); + if (roll < 0.7) return and(formula(d - 1), formula(d - 1)); + if (roll < 0.9) return or(formula(d - 1), formula(d - 1)); + return not(formula(d - 1)); + }; + const values = [-4, -3, -2, -1, 0, 1, 2, 3, 4]; + let proved = 0; + for (let k = 0; k < 400; k++) { + const facts = Array.from({ length: Math.floor(r() * 3) }, () => formula(2)); + const goal = formula(2); + let res; + try { res = prove(facts, goal); } catch { continue; } + if (res.status !== 'proved') continue; + proved++; + for (const a of values) for (const b of values) for (const c of values) { + const env = new Map([[x.id, a], [y.id, b], [z.id, c]]); + const hold = facts.map((u) => evaluate(u, env)); + if (hold.some((v) => v === undefined) || hold.some((v) => !v)) continue; + const v = evaluate(goal, env); + if (v === undefined) continue; + assert.ok(v, `proved, but X=${a}, Y=${b}, Z=${c} satisfies the facts and breaks the goal (formula ${k})`); + } + } + assert.ok(proved >= 40, `only ${proved} of the random formulas were proved; the test is not exercising the solver`); +}); + +/* ---------- obligations from programs ---------- */ + +const statuses = (src) => { + const st = studyOf(src); + assert.ok(st.ok, 'the program parses'); + return verify(st.program.program, { budgetMs: 3000 }).units.flatMap((u) => u.obligations.map((o) => `${o.kind}@${o.line}:${o.status}`)); +}; + +test('vcgen: a counting loop with its invariant proves its ENSURE and every index', () => { + const s = statuses([ + 'FUNCTION COUNT_BASE(B : CHAR, T : STRING) RETURNS INTEGER', + ' ENSURE RESULT == LENGTH({ K IN [0, LENGTH(T)) : T[K] == B })', + ' DECLARE Total : INTEGER', + ' Total <- 0', + ' FOR I <- 0 TO LENGTH(T)', + ' INVARIANT Total == LENGTH({ K IN [0, I) : T[K] == B })', + ' IF T[I] == B THEN', + ' Total <- Total + 1', + ' ENDIF', + ' NEXT I', + ' RETURN Total', + 'ENDFUNCTION', + ].join('\n')); + assert.ok(s.includes('ensure@11:proved'), s.join(' ')); + assert.ok(s.includes('index@7:proved'), s.join(' ')); + assert.ok(s.includes('invariant-keep@6:proved') || s.some((x) => x.startsWith('invariant-keep') && x.endsWith(':proved')), s.join(' ')); +}); + +test('vcgen: an index one past the end is not proved', () => { + const s = statuses([ + 'FUNCTION LAST(T : STRING) RETURNS CHAR', + ' RETURN T[LENGTH(T)]', + 'ENDFUNCTION', + ].join('\n')); + assert.deepEqual(s.filter((x) => x.startsWith('index@2')), ['index@2:unknown']); +}); + +test('vcgen: a division by a parameter needs a REQUIRE', () => { + const without = statuses(['FUNCTION HALF(N : INTEGER, D : INTEGER) RETURNS INTEGER', ' RETURN N DIV D', 'ENDFUNCTION'].join('\n')); + assert.deepEqual(without.filter((x) => x.startsWith('divide')), ['divide@2:unknown']); + const with_ = statuses(['FUNCTION HALF(N : INTEGER, D : INTEGER) RETURNS INTEGER', ' REQUIRE D > 0', ' RETURN N DIV D', 'ENDFUNCTION'].join('\n')); + assert.deepEqual(with_.filter((x) => x.startsWith('divide')), ['divide@3:proved']); +}); + +test('vcgen: FILL and RANDINT ask whether their arguments are allowed', () => { + const s = statuses([ + 'FUNCTION F(N : INTEGER) RETURNS INTEGER', + ' REQUIRE N >= 2', + ' DECLARE Xs : LIST OF INTEGER', + ' Xs <- FILL(N, 0)', + ' RETURN RANDINT(0, LENGTH(Xs))', + 'ENDFUNCTION', + ].join('\n')); + assert.ok(s.includes('argument@4:proved'), s.join(' ')); + assert.ok(s.includes('argument@5:proved'), s.join(' ')); +}); + +test('vcgen: integer arithmetic carries the note on what "proved" assumes about whole numbers', () => { + const st = studyOf(['FUNCTION INC(N : INTEGER) RETURNS INTEGER', ' ENSURE RESULT > N', ' RETURN N + 1', 'ENDFUNCTION'].join('\n')); + const u = verify(st.program.program, {}).units.find((x) => x.name === 'INC'); + assert.ok(u.obligations.every((o) => o.status === 'proved')); + assert.ok(u.notes.some((nt) => nt.kind === 'int-model'), 'the int-model note is missing'); +}); + +test('constants and coefficients are added exactly, never in doubles', () => { + // As doubles, 9007199254740991 + 2 is 9007199254740992, so A*X + (2*X - A*X) came to 1*X and X + A + 2 - A to X + 1 + // (library/pseudo/prover/linear-doubles.pseudo, constant-doubles.pseudo). + const A = 9007199254740991; + const twice = add(mul(n(A), x), sub(mul(n(2), x), mul(n(A), x))); + assert.equal(prove([], cmp('==', twice, x)).status, 'unknown'); + assert.equal(prove([], cmp('==', twice, mul(n(2), x))).status, 'proved'); + const plusTwo = sub(add(add(x, n(A)), n(2)), n(A)); + assert.equal(prove([], cmp('==', plusTwo, add(x, n(1)))).status, 'unknown'); + assert.equal(prove([], cmp('==', plusTwo, add(x, n(2)))).status, 'proved'); + // two sums that differ by one and fold to the same double stay different + assert.equal(prove([not(cmp('==', add(n(1), add(n(A - 1), n(3))), add(n(3), add(n(1), n(A - 2)))))], FALSE).status, 'unknown'); + // 0.1 * (3 * F) is 0.3 * F: as doubles the left coefficient was 0.30000000000000004 + const third = (c, v) => mul(num(c, 'float'), v); + assert.equal(prove([cmp('>', f, num(0, 'float'))], cmp('>', third(0.1, mul(n(3), f)), third(0.3, f))).status, 'unknown'); + assert.equal(prove([], cmp('==', third(0.1, mul(n(3), f)), third(0.3, f))).status, 'proved'); +}); diff --git a/tests/exercises.test.mjs b/tests/exercises.test.mjs new file mode 100644 index 0000000..792b7e9 --- /dev/null +++ b/tests/exercises.test.mjs @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Every exercise file is checked here, so adding one is safe. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { EXERCISES } from '../dist/exercises/index.js'; +import template from '../dist/exercises/_template.js'; +import { Workspace } from '../dist/common/workspace.js'; +import { PROBLEMS, problemById, workForProblem } from '../dist/common/problems.js'; + + +test('exercise ids are unique, URL-safe, and the template is not registered', () => { + const ids = EXERCISES.map((e) => e.id); + assert.equal(new Set(ids).size, ids.length); + for (const id of ids) assert.match(id, /^[a-z0-9]+(-[a-z0-9]+)*$/, id); + assert.ok(!ids.includes(template.id)); +}); + +test('every exercise with an expected answer produces it', () => { + for (const p of PROBLEMS) { + if (!p.expect) continue; + const st = workForProblem(p); + if (st.sources.pseudo) st.lang = 'pseudo'; + const s = new Workspace(st); + const tr = s.trace(); + assert.ok(tr && !tr.error, `${p.id}: ${s.runError || tr?.error?.message}`); + const last = tr.output.at(-1); + assert.ok(last.endsWith(` returned ${p.expect}`), `${p.id}: expected …returned ${p.expect}, got ${last}`); + } +}); + +test('the template itself is a valid exercise', () => { + const s = new Workspace(workForProblem({ ...template, lang: 'pseudo' })); + const tr = s.trace(); + assert.ok(tr && !tr.error, s.runError || tr?.error?.message); + assert.ok(tr.output.at(-1).endsWith(`returned ${template.expect}`)); + assert.equal(s.proof().report.counts.open + s.proof().report.counts.broken, 0); +}); + diff --git a/tests/fuzz.test.mjs b/tests/fuzz.test.mjs new file mode 100644 index 0000000..bff551c --- /dev/null +++ b/tests/fuzz.test.mjs @@ -0,0 +1,91 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The fuzzer behind the Tests view (core/fuzz.ts): it says what it tried and what it checked, and what it reports +// is what the engines do. Run on every exercise, in pseudo and in Mojo. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { STRATEGY } from '../dist/common/fuzz.js'; +import { Workspace } from '../dist/common/workspace.js'; +import { PROBLEMS, problemById, workForProblem } from '../dist/common/problems.js'; + + +const open = (p, lang, call) => { const st = workForProblem(p); st.lang = lang; if (call) st.call = call; return new Workspace(st); }; +const ALL = []; +for (const p of PROBLEMS) for (const lang of ['pseudo', 'mojo']) if ((p.sources?.[lang] ?? '').trim()) ALL.push({ p, lang, s: open(p, lang) }); + +test('every report says what it tried, what it checked, and what happened, and adds up', () => { + let routines = 0; + for (const { p, lang, s } of ALL) { + for (const r of s.fuzz({ budget: 120 })) { + routines++; + const id = `${p.id} ${lang} ${r.name}`; + if (!r.cases.length) { assert.ok(r.problem, `${id}: an untested routine says why`); continue; } + assert.ok(r.checks.length >= 3, `${id}: the checks are listed`); + assert.equal(r.params.length, (r.signature.match(/\(([^)]*)\)/)?.[1].trim() ? r.signature.match(/\(([^)]*)\)/)[1].split(',').length : 0), `${id}: one generator per parameter`); + for (const prm of r.params) assert.ok(prm.how.length > 10, `${id}: ${prm.name}'s generator is described`); + assert.equal(r.strategies.reduce((a, x) => a + x.count, 0), r.cases.length, `${id}: the strategies' counts add up to the cases`); + for (const x of r.strategies) assert.ok(STRATEGY[x.id], `${id}: ${x.id} is a known strategy`); + assert.equal(Object.values(r.counts).reduce((a, b) => a + b, 0), r.cases.length, `${id}: the outcomes add up`); + const tried = r.cases.filter((c) => c.strategy !== 'shrink' && c.strategy !== 'repeat'); + assert.ok(tried.length <= r.budget, `${id}: the budget holds`); + assert.equal(new Set(r.cases.filter((c) => c.strategy !== 'repeat').map((c) => c.call)).size, r.cases.filter((c) => c.strategy !== 'repeat').length, `${id}: no input is tried twice (except to repeat it)`); + r.cases.forEach((c, i) => assert.equal(c.n, i + 1)); + for (const c of r.coverage) assert.ok(c.inputs <= r.cases.length && (c.inputs === 0) === (c.hits === 0), `${id}: line ${c.line}'s coverage`); + if (r.cases.some((c) => c.outcome === 'pass')) assert.ok(r.coverage.some((c) => c.inputs > 0), `${id}: passing inputs reach lines`); + for (const c of r.cases) if (c.outcome === 'fail') assert.ok(c.message, `${id}: a failed case says what went wrong`); + for (const f of r.failures) { + assert.equal(f.shrink[0], f.found, `${id}: shrinking starts at the input that found it`); + assert.equal(f.shrink[f.shrink.length - 1], f.call, `${id}: and ends at the one reported`); + } + } + } + assert.ok(routines >= 30, `the exercises have routines to fuzz (${routines})`); +}); + +test('the same seed makes the same inputs; another seed makes others', () => { + const s = open(problemById('count-base'), 'pseudo'); + const a = s.fuzz({ seed: 7, budget: 100 })[0].cases.map((c) => c.call); + const b = new Workspace(s.state).fuzz({ seed: 7, budget: 100 })[0].cases.map((c) => c.call); + const c = s.fuzz({ seed: 8, budget: 100 })[0].cases.map((c) => c.call); + assert.deepEqual(a, b); + assert.notDeepEqual(a, c); + // the edge cases and every short string do not depend on the seed; only the random and mutated ones do + const first = (xs) => xs.slice(0, 40); + assert.deepEqual(first(a), first(c)); +}); + +test('the off-by-one bug is found, shrunk to its smallest input, and stepping that input fails the same way', () => { + const s = open(problemById('match-off-by-one'), 'pseudo'); + const [r] = s.fuzz(); + assert.ok(r.failures.length >= 1, 'the bug is found'); + const f = r.failures[0]; + assert.match(f.call, /^MATCH\(/); + assert.ok(f.shrink.length >= 1); + // shrinking never makes an input bigger + const size = (call) => call.replace(/[^ACGTN]/g, '').length; + for (let i = 1; i < f.shrink.length; i++) assert.ok(size(f.shrink[i]) <= size(f.shrink[i - 1]), 'each step is no bigger'); + // the stepper on that call stops with the same kind of error, on the same line + const run = open(problemById('match-off-by-one'), 'pseudo', f.call).trace(); + assert.ok(run.error, 'the run stops'); + assert.equal(run.error.line, f.line); + // and the brief report the home page reads agrees + assert.ok(s.tests()[0].failures.some((x) => x.call === f.call)); +}); + +test('coverage-guided mutation: the inputs that reached new lines are the ones mutated', () => { + const [r] = open(problemById('tally'), 'mojo').fuzz(); + const fresh = r.cases.filter((c) => c.newLines?.length); + assert.ok(fresh.length >= 1, 'some inputs reach lines nothing earlier reached'); + assert.ok(r.cases.some((c) => c.strategy === 'mutate'), 'and mutations of them are tried'); + const reached = new Set(fresh.flatMap((c) => c.newLines)); + for (const c of r.coverage) if (c.inputs) assert.ok(reached.has(c.line), `line ${c.line} was first reached by a case marked new`); +}); + +test('a REQUIRE skips an input instead of failing it, and says so', () => { + const withRequire = ALL.filter(({ lang, s }) => lang === 'pseudo' && s.fuzz({ budget: 120 }).some((r) => r.checks.some((c) => /break a REQUIRE \(/.test(c)))); + for (const { s } of withRequire) for (const r of s.fuzz({ budget: 120 })) for (const c of r.cases) if (c.outcome === 'skip') assert.ok(c.kind === 'require' || c.kind === 'declined'); +}); + +test('every report repeats some inputs to check the answer does not change', () => { + for (const { s } of ALL.slice(0, 12)) for (const r of s.fuzz({ budget: 120 })) if (r.counts.pass > 8) assert.ok(r.cases.some((c) => c.strategy === 'repeat'), r.name); +}); diff --git a/tests/hardware.test.mjs b/tests/hardware.test.mjs new file mode 100644 index 0000000..07ec955 --- /dev/null +++ b/tests/hardware.test.mjs @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The x86-64 model machine: for every problem that has assembly, on every machine, the hardware model runs, the +// machine presets reach it, and the pipeline, micro-op, out-of-order and encoding data hold together. The app's +// board, which decides what the course views are given, is tested in pplus-web. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { stageOccupancy, STAGES } from '../dist/engines/asm86/pipeline.js'; +import { coreAt } from '../dist/engines/asm86/ooo.js'; +import { decompose } from '../dist/engines/asm86/uops.js'; +import fs from 'node:fs'; +import { stripHeader, buildListing } from '../scripts/mojo/listings.mjs'; +import { forSimulator, simulatorSetup } from '../dist/mojo/att.js'; +import { listingFunctions } from '../dist/services/godbolt.js'; +import { hardwareFor, resolveMachine } from '../dist/engines/asm86/machine.js'; +import { viewNames } from '../dist/engines/asm86/views.js'; +import { SERVICES } from '../dist/services/index.js'; +import { MACHINE_PRESETS, parseMachine, machineName, machineSpec, machineLabel } from '../dist/common/machine.js'; +import { Workspace } from '../dist/common/workspace.js'; +import { PROBLEMS, problemById, workForProblem } from '../dist/common/problems.js'; + + +/** Every catalog problem with an x86-64 pane, run as the Hardware area runs it. */ +const WITH_ASM = PROBLEMS.filter((p) => (p.sources?.asm ?? '').trim()); +const traceOf = (p, machine) => { + const st = workForProblem(p); + if (machine) st.machine = machine; + st.lang = 'asm'; + const s = new Workspace(st); + const tr = s.trace(); + assert.ok(tr && tr.lang === 'asm', `${p.id}: the x86-64 pane runs`); + return { session: s, tr, raw: tr.raw.trace }; +}; + +test('the hardware model runs on every problem with assembly, on both machines, with and without virtual memory', () => { + for (const p of WITH_ASM) for (const machine of ['teaching', 'realistic', 'teaching+paging', 'realistic+paging']) { + const { raw, tr } = traceOf(p, machine); + const hw = hardwareFor(raw); + assert.doesNotThrow(() => { hw.memory(); hw.pipeline(); hw.core(); }, `${p.id} on ${machine}`); + const perf = tr.raw.hardware().perf(); + assert.equal(perf.instructions, raw.stats.instructions, `${p.id} on ${machine}: the counters read the same run`); + for (const t of [0, 1, Math.floor(raw.count / 2), raw.count]) assert.ok(hw.memory().stateAt(t).h, `${p.id}: caches at step ${t}`); + } +}); + +test('a machine is a preset with virtual memory on or off, and the run carries it to the hardware model', () => { + assert.deepEqual(MACHINE_PRESETS.slice(0, 2), ['teaching', 'realistic']); + for (const preset of MACHINE_PRESETS) for (const paging of [false, true]) { + const name = machineName({ preset, paging }); + assert.deepEqual(parseMachine(name), { preset, paging }, name); + assert.equal(resolveMachine(machineSpec(name)).paging.enabled, paging, `${name}: paging`); + assert.match(machineLabel(name), new RegExp(`^${preset}`)); + } + assert.deepEqual(parseMachine(undefined), { preset: 'teaching', paging: false }); + assert.deepEqual(parseMachine('warp'), { preset: 'teaching', paging: false }, 'an unknown machine is the teaching one'); + const p = PROBLEMS.find((x) => x.id === 'asm-two-pointers'); + const { session, tr, raw } = traceOf(p, 'realistic+paging'); + assert.equal(tr.raw.machine, 'realistic+paging'); + const hw = hardwareFor(raw); + assert.equal(hw.config.caches.l1d.line, 64, 'the views see the realistic machine'); + assert.ok(hw.config.paging.enabled, 'and its virtual memory'); + assert.ok(hw.memory().steps.some((r) => r.translations.length), 'addresses are translated'); + session.state.machine = 'teaching'; + assert.notEqual(session.trace(), tr, 'a new machine is a new run'); + assert.equal(hardwareFor(session.trace().raw.trace).config.paging.enabled, false); +}); + +test('time: the pipeline, micro-op, out-of-order and encoding data the Pipeline and Whole chip presets draw', () => { + for (const p of WITH_ASM) for (const machine of ['teaching', 'realistic']) { + const { raw } = traceOf(p, machine); + const hw = hardwareFor(raw); + const pipe = hw.pipeline(); + const core = hw.core(); + assert.equal(pipe.rows.length >= raw.count, true, `${p.id}: a pipeline row per instruction run (and any wrong-path fetches)`); + for (const r of pipe.rows.slice(0, 200)) { + for (const st of STAGES) assert.equal(stageOccupancy(pipe, r.E[st])[st] != null, true, `${p.id}: ${st} is occupied when a row enters it`); + } + for (let c = 0; c <= core.cycles; c += Math.max(1, Math.floor(core.cycles / 50))) { + const at = coreAt(core, c); + assert.ok(at.rob.length <= hw.config.core.rob, `${p.id} cycle ${c}: the reorder buffer never holds more than it has room for`); + } + for (const ins of raw.program.instrs) { const u = decompose(ins); assert.ok(Array.isArray(u) && u.length >= 1, `${p.id}: ${ins.text} splits into micro-ops`); } + // the encoding view and the listing's bytes need the program laid out in memory + if (raw.layout?.ok) for (const ins of raw.program.instrs) assert.ok(raw.layout.len[ins.index] >= 1 && raw.layout.len[ins.index] <= 15, `${p.id}: ${ins.text} is 1 to 15 bytes`); + } + assert.ok(WITH_ASM.some((p) => traceOf(p).raw.layout?.ok), 'the problems encode'); +}); diff --git a/tests/helpers/sync-programs.mjs b/tests/helpers/sync-programs.mjs new file mode 100644 index 0000000..cf1fc21 --- /dev/null +++ b/tests/helpers/sync-programs.mjs @@ -0,0 +1,196 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The programs the sync translators are held to: every exercise with pseudo, every program in library/pseudo/programs/, +// and small fixtures that each exercise one construct. Shared by the translator tests and the coverage report. +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { EXERCISES } from '../../dist/exercises/index.js'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +export const corpusDir = path.join(here, '..', '..', 'library', 'pseudo', 'programs'); + +/** One construct each, with the call that runs it. */ +export const FIXTURES = { + 'fixture:loop-step': [`FUNCTION EVERY_THIRD(L : LIST OF INTEGER) RETURNS INTEGER + DECLARE Total : INTEGER + Total <- 0 + FOR I <- 0 TO LENGTH(L) STEP 3 + Total <- Total + L[I] + NEXT I + RETURN Total +ENDFUNCTION`, 'EVERY_THIRD([1, 2, 3, 4, 5, 6, 7])'], + 'fixture:loop-down': [`FUNCTION COUNTDOWN(N : INTEGER) RETURNS INTEGER + DECLARE Total : INTEGER + Total <- 0 + FOR I <- N TO 0 STEP -1 + Total <- Total * 2 + I + NEXT I + RETURN Total +ENDFUNCTION`, 'COUNTDOWN(4)'], + 'fixture:break-continue': [`FUNCTION FIRST_BIG(L : LIST OF INTEGER, M : INTEGER) RETURNS INTEGER + DECLARE Found : INTEGER + Found <- -1 + FOR I <- 0 TO LENGTH(L) + IF L[I] < 0 THEN + CONTINUE + ENDIF + IF L[I] > M THEN + Found <- L[I] + BREAK + ENDIF + NEXT I + RETURN Found +ENDFUNCTION`, 'FIRST_BIG([1, -5, 7, 20], 6)'], + 'fixture:early-return': [`FUNCTION HAS_NEG(L : LIST OF INTEGER) RETURNS BOOL + FOR EACH X IN L + IF X < 0 THEN + RETURN TRUE + ENDIF + NEXT X + RETURN FALSE +ENDFUNCTION`, 'HAS_NEG([1, 2, -3])'], + 'fixture:min-max': [`FUNCTION SPAN(L : LIST OF INTEGER) RETURNS INTEGER + DECLARE Lo : INTEGER + DECLARE Hi : INTEGER + Lo <- L[0] + Hi <- L[0] + FOR EACH X IN L + Lo <- MIN(Lo, X) + Hi <- MAX(Hi, X + 0) + NEXT X + RETURN Hi - Lo +ENDFUNCTION`, 'SPAN([4, 9, 2, 7])'], + 'fixture:string-build': [`FUNCTION DOUBLED(T : STRING) RETURNS STRING + DECLARE Out : STRING + Out <- "" + FOR EACH C IN T + Out <- Out & C & C + NEXT C + RETURN Out +ENDFUNCTION`, 'DOUBLED("GATTACA")'], + 'fixture:slice': [`FUNCTION HEAD3(T : STRING) RETURNS STRING + RETURN SLICE(T, 0, 3) +ENDFUNCTION`, 'HEAD3("GATTACA")'], + 'fixture:set': [`FUNCTION DISTINCT(T : STRING) RETURNS INTEGER + DECLARE Seen : SET OF CHAR + FOR EACH C IN T + ADD(Seen, C) + NEXT C + RETURN LENGTH(Seen) +ENDFUNCTION`, 'DISTINCT("GATTACA")'], + 'fixture:routine-calls': [`FUNCTION DOUBLE(N : INTEGER) RETURNS INTEGER + RETURN N * 2 +ENDFUNCTION +FUNCTION QUAD(N : INTEGER) RETURNS INTEGER + RETURN DOUBLE(DOUBLE(N)) +ENDFUNCTION`, 'QUAD(3)'], + 'fixture:procedure': [`PROCEDURE NOTE_SQUARE(N : INTEGER) + OUTPUT N, " ", N * N +ENDPROCEDURE +PROCEDURE RUN(N : INTEGER) + CALL NOTE_SQUARE(N) + CALL NOTE_SQUARE(N + 1) +ENDPROCEDURE`, 'RUN(3)'], + 'fixture:case': [`FUNCTION GRADE(S : INTEGER) RETURNS STRING + DECLARE G : STRING + CASE OF S + 90 TO 101 : G <- "A" + 80 TO 90 : G <- "B" + OTHERWISE : G <- "C" + ENDCASE + RETURN G +ENDFUNCTION`, 'GRADE(85)'], + 'fixture:float-literal': [`FUNCTION HALF(N : INTEGER) RETURNS FLOAT + DECLARE X : FLOAT + X <- 0.5 + RETURN X * N +ENDFUNCTION`, 'HALF(3)'], + 'fixture:div-mod': [`FUNCTION DIGITS(N : INTEGER) RETURNS INTEGER + DECLARE M : INTEGER + DECLARE C : INTEGER + M <- N + C <- 0 + WHILE M > 0 + M <- M DIV 10 + C <- C + 1 + ENDWHILE + RETURN C + N MOD 7 +ENDFUNCTION`, 'DIGITS(12345)'], + 'fixture:forall': [`FUNCTION SORTED_UP(L : LIST OF INTEGER) RETURNS BOOL + RETURN FORALL K IN [0, LENGTH(L) - 1) : L[K] <= L[K + 1] +ENDFUNCTION`, 'SORTED_UP([1, 2, 3])'], + 'fixture:comments': [`// compute it +FUNCTION TWICE(N : INTEGER) RETURNS INTEGER + // set up + DECLARE R : INTEGER + R <- N * 2 // doubled + RETURN R +ENDFUNCTION`, 'TWICE(4)'], + 'fixture:list-item': [`FUNCTION BUMP(L : LIST OF INTEGER) RETURNS LIST OF INTEGER + DECLARE Out : LIST OF INTEGER + FOR EACH X IN L + APPEND(Out, X + 1) + NEXT X + Out[0] <- 99 + RETURN Out +ENDFUNCTION`, 'BUMP([1, 2, 3])'], + 'fixture:value-set-in-branch': [`FUNCTION SIGN_WORD(N : INTEGER) RETURNS STRING + DECLARE W : STRING + IF N < 0 THEN + W <- "minus" + ELSEIF N == 0 THEN + W <- "zero" + ELSE + W <- "plus" + ENDIF + RETURN W +ENDFUNCTION`, 'SIGN_WORD(-4)'], +}; + +/** Every program: { id, name, pseudo, call, input }. */ +export function programs() { + const out = []; + for (const e of EXERCISES) if (e.sources.pseudo) out.push({ id: `ex:${e.id}`, name: e.id, pseudo: e.sources.pseudo, call: e.call ?? null, input: [] }); + for (const f of fs.readdirSync(corpusDir).filter((x) => x.endsWith('.pseudo')).sort()) { + const src = fs.readFileSync(path.join(corpusDir, f), 'utf8'); + out.push({ id: f, name: f, pseudo: src, call: null, input: [...src.matchAll(/^\/\/ INPUT: (.*)$/gm)].map((m) => m[1]) }); + } + for (const [id, [pseudo, call]] of Object.entries(FIXTURES)) out.push({ id, name: id, pseudo, call, input: [] }); + return out; +} + +/** Remove // comments and blank lines: the lowering strips comments, so a round trip never brings them back. */ +export const stripComments = (t) => t.split('\n').map((l) => l.replace(/\s*\/\/.*$/, '')).filter((l) => l.trim()).join('\n'); + +/** [matho source, a call that runs its last function]. The same sets tests/matho.test.mjs runs. */ +export const EQUATIONS = { + 'mean-and-variance': ['\\bar{x} = \\frac{1}{n} \\sum_{i=0}^{n-1} x_i\n\\sigma^2 = \\frac{1}{n - 1} \\sum_{i=0}^{n-1} (x_i - \\bar{x})^2', 'SIGMA_SQUARED(4, [2.0, 4.0, 4.0, 6.0])'], + 'mean': ['\\bar{x} = \\frac{1}{n} \\sum_{i=0}^{n-1} x_i', 'X_BAR(4, [2.0, 4.0, 4.0, 6.0])'], + 'sum-from-one': ['\\bar{x} = \\frac{1}{n} \\sum_{i=1}^{n} x_i', 'X_BAR(4, [1.0, 2.0, 3.0, 4.0])'], + 'sum-no-bounds': ['S = \\sum_i x_i^2', 'S([1.0, 2.0, 3.0])'], + 'binomial': ['P(k) = \\binom{n}{k} p^k (1 - p)^{n - k}', 'P(2, 4, 0.5)'], + 'height': ['h(t) = v_0 t - \\frac{1}{2} g t^2', 'H(2.0, 10.0, 9.81)'], + 'kinetic': ['\\mathrm{KE} = \\frac{1}{2} m v^2', 'KE(3.0, 4.0)'], + 'gravity': ['F = \\frac{G m_1 m_2}{r^2}', 'F(2.0, 3.0, 4.0, 2.0)'], + 'omega': ['\\omega = 2 \\pi f', 'OMEGA(0.5)'], + 'normal-density': ['f(x) = \\frac{1}{\\sqrt{2\\pi\\sigma^2}} e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}', 'F(0.0, 1.0, 0.0)'], + 'ph': ['\\text{pH} = -\\log_{10} [H]', 'PH(0.001)'], + 'log2': ['b = \\log_2 n', 'B(8.0)'], + 'ln-and-log': ['y = \\ln x + \\log x', 'Y(10.0)'], + 'factorial': ['n! = \\prod_{k=1}^{n} k', 'FACTORIAL(5)'], + 'choose': ['c = \\frac{n!}{k! (n-k)!}', 'C(5, 2)'], + 'cosine': ['\\cos\\theta = \\frac{\\vec{a} \\cdot \\vec{b}}{\\|\\vec{a}\\| \\, \\|\\vec{b}\\|}', 'COS_THETA([1.0, 0.0], [1.0, 0.0])'], + 'distance': ['d = \\sqrt{\\sum_{i=0}^{n-1} (p_i - q_i)^2}', 'D(2, [0.0, 0.0], [3.0, 4.0])'], + 'matrix-row': ['y_i = \\sum_{j=0}^{n-1} A_{ij} x_j', 'Y(1, 2, [[1.0, 2.0], [3.0, 4.0]], [1.0, 1.0])'], + 'nested-sums': ['s = \\sum_{i=0}^{n-1} \\sum_{j=0}^{m-1} a_{ij}', 'S(2, 2, [[1.0, 2.0], [3.0, 4.0]])'], + 'relu': ['\\mathrm{relu}(x) = \\begin{cases} 0 & \\text{if } x < 0 \\\\ x & \\text{otherwise} \\end{cases}', 'RELU(-2.0)'], + 'piecewise': ['f(x) = \\begin{cases} 0 & x < 0 \\\\ x & 0 \\le x < 1 \\\\ 2 - x & 1 \\le x \\le 2 \\\\ 0 & \\text{otherwise} \\end{cases}', 'F(1.5)'], + 'calls-an-earlier-equation': ['g(x) = x^2 + 1\nh(x) = 2 g(x) + c', 'H(3.0, 1.0)'], + 'sum-over-a-set': ['m = \\sum_{x \\in S} x', 'M([1.0, 2.0, 3.5])'], + 'floor': ['h = \\lfloor n / 2 \\rfloor', 'H(7)'], + 'ceiling': ['h = \\lceil n / 2 \\rceil', 'H(7)'], + 'absolute': ['d = |a - b|', 'D(2.0, 5.0)'], + 'max-min': ['m = \\max(a, b) - \\min(a, b)', 'M(2.0, 5.0)'], + 'mod': ['r = a \\bmod b', 'R(7, 3)'], +}; diff --git a/tests/library-harness.test.mjs b/tests/library-harness.test.mjs new file mode 100644 index 0000000..0d9df92 --- /dev/null +++ b/tests/library-harness.test.mjs @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The program a compiled reference of the library is run as (scripts/library.mjs harness): it holds no case. Its +// cases arrive at run time on one line of stdin, so no runner's answer is the compiler working out a call on +// literal arguments while it builds (Mojo folds such a call with arithmetic of its own; built with contraction on, +// its fold does not fuse a multiply-add as the built code does: library/mojo/probes/float-contract-runtime). The +// line carries every value exactly. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { checkMojo } from '../dist/mojo/check.js'; +import { runMojo } from '../dist/mojo/interp.js'; +import { harness, harnessInput, floatWords, referencePrograms } from '../scripts/library.mjs'; + +/** Another value of the same type: other numbers, other text, a list one item longer. */ +function other(v, t) { + switch (t?.k) { + case 'FLOAT': return Number(v) * 3 + 1.5; + case 'INTEGER': return typeof v === 'bigint' ? v + 7n : Number(v) + 7; + case 'BOOL': return !v; + case 'CHAR': return v === 'Q' ? 'R' : 'Q'; + case 'STRING': return `${v}x y`; + case 'LIST': case 'ARRAY': { const xs = Array.isArray(v) ? v : v?.data ?? []; return [...xs.map((x) => other(x, t.of)), other(xs[0] ?? (t.of.k === 'STRING' ? '' : 0), t.of)]; } + default: throw new Error(`no other value for ${t?.k}`); + } +} + +test('a compiled reference\'s program is the same whatever its cases are: they arrive on stdin', async () => { + const programs = (await referencePrograms()).filter((p) => p.ref.lang === 'mojo' || p.ref.lang === 'c'); + assert.ok(programs.filter((p) => p.ref.lang === 'mojo').length >= 17, 'every Mojo reference'); + assert.ok(programs.filter((p) => p.ref.lang === 'c').length >= 4, 'every C reference'); + for (const p of programs) { + const id = `${p.id} ${p.ref.lang}/${p.ref.level}`; + const src = p.make(); + const changed = p.calls.map((c) => ({ args: c.args.map((v, i) => other(v, p.sub.params[i].type)) })); + assert.equal(harness(p.ref.lang, p.ref, p.fname, p.sub, changed), src, `${id}: the program changes with the cases' arguments, so they are written into it`); + assert.equal(harness(p.ref.lang, p.ref, p.fname, p.sub, p.calls.slice(0, 1)), src, `${id}: the program changes with the number of cases`); + assert.notEqual(harnessInput(p.ref.lang, p.sub, changed), p.input(), `${id}: the input line does not carry the arguments`); + assert.match(p.input(), new RegExp(`^${p.calls.length} .* end\\n$`), id); + assert.ok(src.includes(p.ref.source.replace(/\s+$/, '')), `${id}: the reference's own text is in its program unchanged`); + } +}); + +test('a FLOAT is sent as m and e with x = m * 2^e exactly, m odd and below 2^53', () => { + const dv = new DataView(new ArrayBuffer(8)); + const xs = [0.1, 1e-16, 1e300, 5e-324, 2.2250738585072014e-308, -1.5, 1, 2 ** 53 + 2, Number.MAX_VALUE]; + for (let i = 0; i < 2000; i++) { dv.setUint32(0, (Math.random() * 2 ** 32) >>> 0); dv.setUint32(4, (Math.random() * 2 ** 32) >>> 0); const x = dv.getFloat64(0); if (Number.isFinite(x)) xs.push(x); } + for (const x of xs) { + const [m, e] = floatWords(x); + const big = BigInt(m); + assert.ok((big < 0n ? -big : big) < 2n ** 53n && (big % 2n !== 0n), `${x}: m = ${m}`); + assert.equal(Number(big) * 2 ** Number(e), x, `${x}: ${m} * 2^${e}`); + } + assert.deepEqual(floatWords(-0), ['-0', '0']); + assert.deepEqual(floatWords(Infinity), ['1', '1024']); + assert.throws(() => floatWords(NaN)); +}); + +// A routine taking every kind of argument a reference can, and cases with the values a line could mangle +const EVERY = { params: ['FLOAT', 'INTEGER', 'BOOL', 'STRING', { k: 'LIST', of: { k: 'FLOAT' } }, { k: 'LIST', of: { k: 'STRING' } }].map((k) => ({ type: typeof k === 'string' ? { k } : k })), returns: { k: 'STRING' } }; +const CASES = [ + { args: [0.1, -5, true, 'a b', [1e-16, -0, 5e-324], ['', '100%', 'x%20y']] }, + { args: [1e300, 4611686018427387904n, false, '', [], ['tab\there', 'end']] }, + { args: [-Infinity, 0, true, ' ', [2.5], []] }, +]; + +test('core\'s Mojo interpreter runs a harness with every kind of argument and gets each value back exactly', () => { + const ref = { source: `def every(x: Float64, n: Int, b: Bool, s: String, xs: List[Float64], ss: List[String]) -> String: + var out = String(x) + "|" + String(n) + "|" + String(b) + "|[" + s + "]|" + for i in range(len(xs)): + out += String(xs[i]) + "," + out += "|" + for i in range(len(ss)): + out += "[" + ss[i] + "]" + return out +` }; + const src = harness('mojo', ref, 'every', EVERY, CASES); + const ck = checkMojo(src); + assert.deepEqual(ck.diagnostics.filter((d) => d.severity === 'error').map((d) => `L${d.line} ${d.message}`), []); + const run = runMojo(ck.module, src, { input: [harnessInput('mojo', EVERY, CASES).replace(/\n$/, '')], maxSteps: 400000 }); + assert.ok(!run.refused, `the interpreter declines ${run.refused}`); + assert.equal(run.exitCode, 0, run.stdout); + assert.equal(run.stdout, [ + '0.1|-5|True|[a b]|1e-16,-0.0,5e-324,|[][100%][x%20y]', + '1e+300|4611686018427387904|False|[]||[tab\there][end]', + '-inf|0|True|[ ]|2.5,|', + ].join('\n') + '\n'); + // a line that holds more or fewer words than its cases announce stops the program + const short = runMojo(ck.module, src, { input: ['1 1 0'], maxSteps: 400000 }); + assert.notEqual(short.exitCode, 0); +}); + +test('a C harness gets each value back exactly, at -O0 and -O2', (t) => { + if (spawnSync('cc', ['--version']).status !== 0) { assert.ok(!process.env.CI, 'cc is not installed, and CI must run this'); t.skip('no cc'); return; } + const sub = { params: ['FLOAT', 'INTEGER', 'STRING', 'CHAR', { k: 'LIST', of: { k: 'FLOAT' } }].map((k) => ({ type: typeof k === 'string' ? { k } : k })), returns: { k: 'INTEGER' } }; + const ref = { source: `long every(double x, long n, const char *s, char c, const double *xs, long count) { + printf("%.17g %ld [%s] %c", x, n, s, c); + for (long i = 0; i < count; i++) printf(" %.17g", xs[i]); + printf(" | "); + return count; +} +` }; + const cases = [ + { args: [0.1, -5, 'a b%', 'Z', [1e-16, -0, 5e-324]] }, + { args: [-Infinity, 4611686018427387904n, '', ' ', []] }, + ]; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-harness-')); + try { + fs.writeFileSync(path.join(dir, 'every.c'), harness('c', ref, 'every', sub, cases)); + for (const level of ['-O0', '-O2']) { + const c = spawnSync('cc', ['-std=c11', '-Wall', '-Wextra', '-Werror', level, '-o', path.join(dir, 'every'), path.join(dir, 'every.c'), '-lm'], { encoding: 'utf8' }); + assert.equal(c.status, 0, c.stderr); + const r = spawnSync(path.join(dir, 'every'), [], { input: harnessInput('c', sub, cases), encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, '0.10000000000000001 -5 [a b%] Z 9.9999999999999998e-17 -0 4.9406564584124654e-324 | 3\n-inf 4611686018427387904 [] | 0\n', level); + } + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/tests/library.test.mjs b/tests/library.test.mjs new file mode 100644 index 0000000..b75d3b6 --- /dev/null +++ b/tests/library.test.mjs @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The library (library/, pplus-library): the registry core builds from it is current, every routine checks and +// runs as its record says, and USE behaves as docs/library.md describes: names resolve, runs step over the +// routines, the prover uses their contracts, and the Mojo translator writes each routine once. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { libraryModules, libraryRoutine } from '../dist/library/registry.js'; +import { studyOf, runPseudo, provePseudo } from '../dist/services/pseudo.js'; +import { pseudoToMojo } from '../dist/sync/pseudo-to-mojo.js'; +import { checkMojo } from '../dist/mojo/check.js'; +import { runMojo } from '../dist/mojo/interp.js'; + +test('the registry is generated from the library checkout and is up to date', () => { + assert.ok(fs.existsSync('library/modules'), 'library/ is not checked out: git submodule update --init'); + const r = spawnSync(process.execPath, ['scripts/library.mjs', 'emit', '--check'], { encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr || r.stdout); +}); + +test('the task that asks godbolt about the Mojo references finds the pin, and would ask the pinned compiler', () => { + // library-verify-mojo read the pin from where the corpus used to live and stopped before asking anything + const sources = spawnSync(process.execPath, ['scripts/library.mjs', 'mojo-sources'], { encoding: 'utf8', maxBuffer: 1 << 26 }); + assert.equal(sources.status, 0, sources.stderr); + const py = spawnSync('python3', ['--version'], { encoding: 'utf8' }); + // no python3 here: the pipelines' own tests are skipped the same way, except under CI, where a skip hides a gap + if (py.status !== 0) { assert.ok(!process.env.CI, 'python3 is not installed, and CI must run this'); return; } + const pin = JSON.parse(fs.readFileSync('library/mojo/pin.json', 'utf8')); + const r = spawnSync('python3', ['scripts/library_godbolt.py', '--dry-run', '--force'], { input: sources.stdout, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + const asks = r.stdout.split('\n').filter((l) => l.includes('would ask')); + assert.ok(asks.length >= 16, `${asks.length} references would be asked`); + for (const l of asks) assert.ok(l.includes(`(${pin.godbolt.compiler})`), l); +}); + +test('every routine: one routine per file, its cases run, and its record is current and has no failing check', () => { + const mods = libraryModules(); + assert.ok(mods.length >= 6, `only ${mods.length} modules`); + let routines = 0; + for (const m of mods) { + for (const r of m.routines) { + routines++; + const st = studyOf(r.source); + assert.ok(st.ok, `${r.id}: ${st.diagnostics.filter((d) => d.severity === 'error').map((d) => d.message).join('; ')}`); + const subs = [...st.program.program.subs.values()]; + assert.deepEqual(subs.map((s) => s.name), [r.name], `${r.id} defines ${subs.map((s) => s.name)}`); + assert.ok(r.cases.length >= 2, `${r.id} has fewer than two cases`); + for (const c of r.cases) { + const tr = runPseudo(st, { call: c.call, maxSteps: 2000000 }); + assert.equal(tr.error, null, `${r.id}: ${c.call}: ${tr.error?.message}`); + assert.ok(c.result !== undefined, `${r.id}: ${c.call} has no recorded result: run pixi run library-check`); + } + const all = [...Object.entries(r.checks), ...r.references.flatMap((x) => Object.entries(x.checks).map(([k, v]) => [`${x.lang}/${x.level} ${k}`, v])), ...(r.proof ? [['proof', r.proof.check]] : [])]; + for (const [k, c] of all) assert.notEqual(c.status, 'fail', `${r.id} ${k}: ${c.detail}`); + for (const k of ['structure', 'pseudo', 'prover', 'fuzz']) assert.ok(r.checks[k], `${r.id} has no ${k} check recorded: run pixi run library-check`); + } + } + assert.ok(routines >= 16, `only ${routines} routines`); +}); + +test('USE: a module or one routine; unknown names are errors at the USE line', () => { + assert.ok(libraryRoutine('md.LennardJones')); + assert.ok(libraryRoutine('MD', 'lennardjones'), 'names ignore case'); + const ids = (src) => studyOf(src).diagnostics.filter((d) => d.severity === 'error').map((d) => `${d.id}@${d.line}`); + assert.deepEqual(ids('USE nosuchmodule\n'), ['unknown-library@1']); + assert.deepEqual(ids('USE md.NoSuchRoutine\n'), ['unknown-routine@1']); + assert.deepEqual(ids('USE md.MinimumImage\nFUNCTION MinimumImage(X : FLOAT) RETURNS FLOAT\n RETURN X\nENDFUNCTION\n'), ['library-clash@2']); + assert.deepEqual(ids('USE align.MatchScore\nOUTPUT LinearGapPenalty(2, -1)\n'), ['unknown-function@2'], 'USE of one routine brings in only that one'); + assert.deepEqual(ids('IF TRUE THEN\n USE md\nENDIF\n'), ['use-place@2']); +}); + +test('a run steps over a library routine, and a broken REQUIRE is reported at the call', () => { + const st = studyOf('USE md\nOUTPUT MinimumImage(2.3, 1.0)\nOUTPUT MinimumImage(1.0, 0.0)\n'); + const tr = runPseudo(st, {}); + assert.equal(tr.output[0], '0.2999999999999998'); + assert.ok(tr.steps.every((s) => s.line === null || s.line <= 3), 'a step carries a line of the library file'); + assert.ok(tr.steps.some((s) => s.kind === 'call' && /from the library \(md\.MinimumImage\)/.test(s.text))); + assert.equal(tr.error?.kind, 'require'); + assert.equal(tr.error.line, 3); + assert.match(tr.error.message, /In the library routine md\.MinimumImage/); +}); + +test('the prover uses a routine\'s REQUIRE at the call and its ENSURE afterwards, without proving it again', () => { + const r = provePseudo(studyOf('USE seq.HammingDistance\nFUNCTION F(S : STRING) RETURNS INTEGER\n ENSURE RESULT <= LENGTH(S)\n RETURN HammingDistance(S, S)\nENDFUNCTION\n')); + assert.deepEqual(r.units.map((u) => u.name), ['F'], 'library routines are not units of the program'); + const obs = r.units[0].obligations; + assert.equal(obs.find((o) => o.kind === 'require').status, 'proved'); + assert.equal(obs.find((o) => o.kind === 'ensure').status, 'proved', 'the ENSURE of HammingDistance is a fact after the call'); +}); + +test('the Mojo made from a program that uses the library holds each routine once, dependencies first, and agrees', () => { + const src = 'USE align\nOUTPUT NeedlemanWunschScore("GATTACA", "GCATGCU", 1, -1, -1), " ", MatchScore(\'A\', \'A\', 2, -1)\n'; + const mj = pseudoToMojo(src); + const defs = [...mj.source.matchAll(/^def (\w+)/gm)].map((m) => m[1]); + assert.deepEqual(defs, ['match_score', 'linear_gap_penalty', 'needleman_wunsch_score', 'main']); + assert.match(mj.source, /# From the Programming\+ library: align\.MatchScore/); + const ck = checkMojo(mj.source); + assert.deepEqual(ck.diagnostics.filter((d) => d.severity === 'error'), []); + const run = runMojo(ck.module, mj.source, { maxSteps: 4000000 }); + assert.equal(run.stdout.trim(), runPseudo(studyOf(src), {}).output.join('\n')); +}); diff --git a/tests/listings.test.mjs b/tests/listings.test.mjs new file mode 100644 index 0000000..a70e9c4 --- /dev/null +++ b/tests/listings.test.mjs @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// scripts/mojo/listings.mjs: godbolt's saved answers become x86-64 panes, and the program <-> exercise links hold. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { stripHeader, readHeader, findGolden, buildListing, buildAll, render } from '../scripts/mojo/listings.mjs'; +import { forSimulator, simulatorSetup } from '../dist/mojo/att.js'; +import { listingFunctions, mojoBuildArgs } from '../dist/services/godbolt.js'; +import { MOJO_BUILD_ARGS } from '../dist/mojo/generated/pin.js'; +import { parseAsm } from '../dist/services/asm.js'; + +const ROOT = new URL('..', import.meta.url).pathname; +const GOLDEN = ROOT + 'library/mojo/golden'; +const mods = { forSimulator, simulatorSetup, listingFunctions, mojoBuildArgs }; + +test('stripHeader drops what emit_ts.py drops, and says how many lines went', () => { + const text = '# origin: x\n# probe: y\n# mode: both\n\ndef main():\n pass\n'; + const { source, dropped } = stripHeader(text); + assert.equal(source, 'def main():\n pass\n'); + assert.equal(dropped, 4); // three header lines and the blank one after + assert.equal(stripHeader('# a comment, not a header key\ndef f(): pass').dropped, 0); +}); + +test('readHeader reads the keys verify.py knows and ignores other comments', () => { + const h = readHeader('# probe: q\n# mode: both\n# args: -O0 --debug-level=line-tables\n# a note\n# expect: pass\ndef main(): pass'); + assert.equal(h.mode, 'both'); + assert.equal(h.args, '-O0 --debug-level=line-tables'); + assert.equal(h.expect, 'pass'); + assert.equal(h.note, undefined); +}); + +test('a golden is used only for the exact source, mode and arguments it answered', () => { + const probe = fs.readFileSync(ROOT + 'library/mojo/probes/godbolt-line-tables-O0.mojo', 'utf8'); + const h = readHeader(probe); + const args = mojoBuildArgs(h.args ?? ''); + assert.ok(findGolden(GOLDEN, probe, h.mode, args), 'the committed answer for this probe is found'); + assert.equal(findGolden(GOLDEN, probe + ' ', h.mode, args), null, 'one extra character and the answer no longer applies'); + assert.equal(findGolden(GOLDEN, probe, 'run', args), null, 'a run-mode question is a different question'); + assert.equal(findGolden(GOLDEN, probe, h.mode, h.args ?? ''), null, 'its own arguments without the pin\'s are a different question'); +}); + +test('real godbolt answers become x86-64 panes the assembler accepts, with lines that point into the shown source', () => { + let made = 0; + for (const f of fs.readdirSync(ROOT + 'library/mojo/probes').filter((x) => x.endsWith('.mojo'))) { + const text = fs.readFileSync(ROOT + 'library/mojo/probes/' + f, 'utf8'); + if (!/^# mode: (asm|both)/m.test(text)) continue; + const r = buildListing(text, GOLDEN, mods); + if (r.why) continue; // e.g. built without -O0, so the optimizer inlined every function + made++; + const p = parseAsm(r.asm); + assert.ok(p.ok, `${f}: ${p.diagnostics[0]?.message}`); + assert.match(r.setup, /^#/, `${f}: the setup starts with the comment saying where the arguments arrive`); + const shown = stripHeader(text).source.split('\n').length; + for (const m of r.asm.matchAll(/; mojo line (\d+)/g)) assert.ok(+m[1] >= 1 && +m[1] <= shown, `${f}: line ${m[1]} is outside the ${shown}-line program the pane shows`); + } + assert.ok(made >= 1, 'at least one committed probe has a usable listing; if none does, the corpus lost its assembly'); +}); + +test('a program with no answer, or one that was never asked for a listing, says why instead of inventing one', () => { + assert.match(buildListing('# mode: both\n# args: -O0\ndef main(): print(1)\n', GOLDEN, mods).why, /no godbolt answer/); + assert.match(buildListing('def main(): print(1)\n', GOLDEN, mods).why, /mode: both/); +}); + +test('render keeps the header and interface, and writes programs in name order', () => { + const out = render({ zeta: { entry: 'z(Int)', asm: 'z:\n ret', setup: '# z', build: '' }, alpha: { entry: 'a(Int)', asm: 'a:\n ret', setup: '# a', build: '-O0' } }); + assert.match(out, /export interface MojoListing/); + assert.ok(out.indexOf('"alpha"') < out.indexOf('"zeta"')); + assert.ok(out.endsWith('};\n')); +}); + +test('every exercise that names a Mojo program has that program in library/mojo/local/', () => { + const dir = ROOT + 'src/exercises/'; + const missing = []; + for (const f of fs.readdirSync(dir).filter((x) => x.endsWith('.ts') && !x.startsWith('_'))) { + const m = /mojoProgram:\s*['"]([^'"]+)['"]/.exec(fs.readFileSync(dir + f, 'utf8')); + if (m && !fs.existsSync(ROOT + `library/mojo/local/${m[1]}.mojo`)) missing.push(`${f} -> library/mojo/local/${m[1]}.mojo`); + } + assert.deepEqual(missing, [], 'create the file, or remove `mojoProgram` from the exercise; until then `pixi run play` will not regenerate programs.ts, because the catalog would throw'); +}); + +// What an exercise's Hardware pane opens with. Measured on 2026-11-12: the whole of demo's program (every function +// the compiler kept at -O0, and the 492 KB location table its bounds checks address, placed byte by byte in the +// setup) made listings.ts 2.9 MB, and with it `node --test tests/*.test.mjs` passed a 3 GB memory cap. +const LISTING_LINES = 400, LISTINGS_BYTES = 64 * 1024; + +test('the committed listings.ts is what the tool writes from the stored answers, built as they were asked, and small', () => { + const { listings } = buildAll(ROOT + 'library/mojo/local', GOLDEN, mods); + const committed = fs.readFileSync(ROOT + 'src/mojo/generated/listings.ts', 'utf8'); + assert.ok(committed === render(listings), 'src/mojo/generated/listings.ts is not what scripts/mojo/listings.mjs writes from the stored answers: run `pixi run play <name> --no-verify`, or `node scripts/mojo/listings.mjs` after a build'); + assert.ok(Object.keys(listings).length >= 1, 'at least one exercise program has a stored listing'); + for (const [name, l] of Object.entries(listings)) { + const own = readHeader(fs.readFileSync(ROOT + `library/mojo/local/${name}.mojo`, 'utf8')).args ?? ''; + assert.equal(l.build, mojoBuildArgs(own), `${name}: the arguments its answer was asked with, the pin's first`); + assert.ok(l.build.startsWith(MOJO_BUILD_ARGS), `${name}: ${l.build}`); + assert.ok(parseAsm(l.asm).ok, `${name}: the stored listing assembles`); + const lines = l.asm.split('\n').length; + assert.ok(lines <= LISTING_LINES, `${name}: ${lines} lines of x86-64; an exercise's pane holds its function and what that calls, at most ${LISTING_LINES} lines`); + } + assert.ok(committed.length <= LISTINGS_BYTES, `listings.ts is ${committed.length} bytes, over ${LISTINGS_BYTES}`); +}); diff --git a/tests/matho.test.mjs b/tests/matho.test.mjs new file mode 100644 index 0000000..1324f5e --- /dev/null +++ b/tests/matho.test.mjs @@ -0,0 +1,186 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// matho (equations, written in LaTeX) → pseudo: formulas from statistics, probability, physics, chemistry +// and linear algebra, each run through the translation and checked against a value worked out by hand; plus +// the refusals, which have to say what finite procedure to write instead. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { parseMatho, runMatho, lowerMatho, defaultCallMatho, splitEquations, tokenizeMatho } from '../dist/services/matho.js'; +import { studyOf } from '../dist/services/pseudo.js'; + +const value = (src, call) => { + const r = parseMatho(src); + const errs = r.diagnostics.filter((d) => d.severity === 'error'); + assert.ok(r.ok, errs.map((d) => `L${d.line} ${d.id}: ${d.message}`).join(' | ') + '\n' + r.program.pseudo); + const tr = runMatho(r, { call, maxSteps: 200000 }); + assert.equal(tr.error, null, tr.error?.message); + const m = /returned (.*)$/.exec(tr.output[tr.output.length - 1]); + return { v: Number(m[1]), r, tr }; +}; +const close = (a, b, eps = 1e-9) => assert.ok(Math.abs(a - b) < eps, `${a} is not within ${eps} of ${b}`); + +test('matho: mean and variance, the second referring to the first and hoisted out of the loop', () => { + const src = '\\bar{x} = \\frac{1}{n} \\sum_{i=0}^{n-1} x_i\n\\sigma^2 = \\frac{1}{n - 1} \\sum_{i=0}^{n-1} (x_i - \\bar{x})^2'; + const { v, r } = value(src, 'SIGMA_SQUARED(4, [2.0, 4.0, 4.0, 6.0])'); + close(v, 8 / 3); + assert.match(r.program.pseudo, /FUNCTION X_BAR\(N : INTEGER, X : LIST OF FLOAT\) RETURNS FLOAT/); + assert.match(r.program.pseudo, /XBar <- X_BAR\(N, X\)\n\s+Total <- 0\.0\n\s+FOR I <- 0 TO N/); + assert.ok(r.diagnostics.some((d) => d.id === 'matho-hoist')); + assert.ok(r.diagnostics.some((d) => d.id === 'matho-list' && /X is indexed/.test(d.message))); + assert.equal(value(src, 'X_BAR(4, [2.0, 4.0, 4.0, 6.0])').v, 4); +}); + +test('matho: a sum from 1 shifts the subscript to a 0-based position, and says so', () => { + const { v, r } = value('\\bar{x} = \\frac{1}{n} \\sum_{i=1}^{n} x_i', 'X_BAR(4, [1.0, 2.0, 3.0, 4.0])'); + assert.equal(v, 2.5); + assert.match(r.program.pseudo, /FOR I <- 1 TO N \+ 1\n\s+Total <- Total \+ X\[I - 1\]/); + assert.ok(r.diagnostics.some((d) => d.id === 'matho-one-based')); +}); + +test('matho: a sum with no bounds runs over the whole list', () => { + const { v, r } = value('S = \\sum_i x_i^2', 'S([1.0, 2.0, 3.0])'); + assert.equal(v, 14); + assert.match(r.program.pseudo, /FOR I <- 0 TO LENGTH\(X\)/); +}); + +test('matho: binomial probability, with the extra names becoming parameters and p renamed away from P', () => { + const { v, r } = value('P(k) = \\binom{n}{k} p^k (1 - p)^{n - k}', 'P(2, 4, 0.5)'); + close(v, 0.375); + assert.match(r.program.pseudo, /FUNCTION P\(K : INTEGER, N : INTEGER, P2 : FLOAT\)/); + assert.ok(r.diagnostics.some((d) => d.id === 'matho-case')); + assert.ok(r.diagnostics.some((d) => d.id === 'matho-extra-params')); +}); + +test('matho: physics formulas with subscripted constants and Greek letters', () => { + close(value('h(t) = v_0 t - \\frac{1}{2} g t^2', 'H(2.0, 10.0, 9.81)').v, 20 - 0.5 * 9.81 * 4); + close(value('\\mathrm{KE} = \\frac{1}{2} m v^2', 'KE(3.0, 4.0)').v, 24); + close(value('F = \\frac{G m_1 m_2}{r^2}', 'F(2.0, 3.0, 4.0, 2.0)').v, 6); + const { r } = value('F = \\frac{G m_1 m_2}{r^2}', 'F(2.0, 3.0, 4.0, 2.0)'); + assert.match(r.program.pseudo, /FUNCTION F\(G : FLOAT, M1 : FLOAT, M2 : FLOAT, R : FLOAT\)/); + close(value('\\omega = 2 \\pi f', 'OMEGA(0.5)').v, Math.PI); +}); + +test('matho: the normal density, with e^{...} as EXP and \\sqrt', () => { + const { v } = value('f(x) = \\frac{1}{\\sqrt{2\\pi\\sigma^2}} e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}', 'F(0.0, 1.0, 0.0)'); + close(v, 1 / Math.sqrt(2 * Math.PI)); +}); + +test('matho: chemistry: pH from a concentration, log bases', () => { + close(value('\\text{pH} = -\\log_{10} [H]', 'PH(0.001)').v, 3); + close(value('b = \\log_2 n', 'B(8.0)').v, 3); + close(value('y = \\ln x + \\log x', 'Y(10.0)').v, Math.log(10) + 1); +}); + +test('matho: factorial as a product loop, both as n! and as \\prod', () => { + assert.equal(value('n! = \\prod_{k=1}^{n} k', 'FACTORIAL(5)').v, 120); + const { v, r } = value('c = \\frac{n!}{k! (n-k)!}', 'C(5, 2)'); + assert.equal(v, 10); + assert.match(r.program.pseudo, /Factorial <- 1\.0/); + assert.match(r.program.pseudo, /Factorial2 <- 1\.0/); +}); + +test('matho: vectors: dot product, norm, the cosine of the angle', () => { + close(value('\\cos\\theta = \\frac{\\vec{a} \\cdot \\vec{b}}{\\|\\vec{a}\\| \\, \\|\\vec{b}\\|}', 'COS_THETA([1.0, 0.0], [0.0, 1.0])').v, 0); + const { r } = value('\\cos\\theta = \\frac{\\vec{a} \\cdot \\vec{b}}{\\|\\vec{a}\\| \\, \\|\\vec{b}\\|}', 'COS_THETA([1.0, 0.0], [1.0, 0.0])'); + assert.match(r.program.pseudo, /RETURN DOT\(A, B\) \/ \(NORM_OF\(A\) \* NORM_OF\(B\)\)/); + close(value('d = \\sqrt{\\sum_{i=0}^{n-1} (p_i - q_i)^2}', 'D(2, [0.0, 0.0], [3.0, 4.0])').v, 5); +}); + +test('matho: a matrix row times a vector, from a two-letter subscript', () => { + const { v, r } = value('y_i = \\sum_{j=0}^{n-1} A_{ij} x_j', 'Y(1, 2, [[1.0, 2.0], [3.0, 4.0]], [1.0, 1.0])'); + assert.equal(v, 7); + assert.match(r.program.pseudo, /FUNCTION Y\(I : INTEGER, N : INTEGER, A : LIST OF LIST OF FLOAT, X : LIST OF FLOAT\)/); + assert.match(r.program.pseudo, /A\[I\]\[J\] \* X\[J\]/); +}); + +test('matho: nested sums', () => { + const { v } = value('s = \\sum_{i=0}^{n-1} \\sum_{j=0}^{m-1} a_{ij}', 'S(2, 2, [[1.0, 2.0], [3.0, 4.0]])'); + assert.equal(v, 10); +}); + +test('matho: piecewise definitions become IF / ELSE IF / ELSE', () => { + const src = '\\mathrm{relu}(x) = \\begin{cases} 0 & \\text{if } x < 0 \\\\ x & \\text{otherwise} \\end{cases}'; + assert.equal(value(src, 'RELU(-2.0)').v, 0); + assert.equal(value(src, 'RELU(3.0)').v, 3); + const tri = 'f(x) = \\begin{cases} 0 & x < 0 \\\\ x & 0 \\le x < 1 \\\\ 2 - x & 1 \\le x \\le 2 \\\\ 0 & \\text{otherwise} \\end{cases}'; + assert.equal(value(tri, 'F(0.5)').v, 0.5); + assert.equal(value(tri, 'F(1.5)').v, 0.5); + assert.equal(value(tri, 'F(5.0)').v, 0); + assert.match(parseMatho(tri).program.pseudo, /ELSEIF 0 <= X AND X < 1 THEN/); +}); + +test('matho: a defined function is called by name later, with explicit and implicit arguments', () => { + const src = 'g(x) = x^2 + 1\nh(x) = 2 g(x) + c'; + const { v, r } = value(src, 'H(3.0, 1.0)'); + assert.equal(v, 21); + assert.match(r.program.pseudo, /RETURN 2 \* G\(X\) \+ C/); +}); + +test('matho: a sum over a set uses FOR EACH', () => { + const { v, r } = value('m = \\sum_{x \\in S} x', 'M([1.0, 2.0, 3.5])'); + assert.equal(v, 6.5); + assert.match(r.program.pseudo, /FOR EACH X IN S/); +}); + +test('matho: floors, ceilings, absolute values, max and min, mod', () => { + assert.equal(value('h = \\lfloor n / 2 \\rfloor', 'H(7)').v, 3); + assert.equal(value('h = \\lceil n / 2 \\rceil', 'H(7)').v, 4); + assert.equal(value('d = |a - b|', 'D(2.0, 5.0)').v, 3); + assert.equal(value('m = \\max(a, b) - \\min(a, b)', 'M(2.0, 5.0)').v, 3); + assert.equal(value('r = a \\bmod b', 'R(7, 3)').v, 1); +}); + +test('matho: the refusals say what to write instead, and the good equations still translate', () => { + const r = parseMatho('a = b^2\n\\int_0^1 f(x) dx\n\\sum_{i=0}^{\\infty} \\frac{1}{2^i}\n\\lim_{x \\to 0} \\frac{\\sin x}{x}\n\\frac{\\partial f}{\\partial x}'); + assert.equal(r.ok, false); + const errs = r.diagnostics.filter((d) => d.severity === 'error'); + assert.deepEqual(errs.map((d) => d.line), [2, 3, 4, 5]); + assert.ok(errs.every((d) => d.id === 'matho-refused')); + assert.match(errs[0].message, /sum it approximates/); + assert.match(errs[1].message, /upper bound/); + assert.match(errs[2].message, /not a procedure/); + assert.match(r.program.pseudo, /FUNCTION A\(B : FLOAT\)/); +}); + +test('matho: syntax problems are reported on the right line, with a sentence', () => { + const r = parseMatho('a = \\frac{1}{\nb = \\foo{x}\nc = x \\pm y\nd = \\begin{cases} 1 & x < 0 \\end{cases}'); + const errs = r.diagnostics.filter((d) => d.severity === 'error'); + assert.deepEqual(errs.map((d) => [d.line, d.id]), [[1, 'matho-syntax'], [2, 'matho-command'], [3, 'matho-pm'], [4, 'matho-cases-otherwise']]); + assert.match(errs[1].message, /\\foo is not a command this reader knows/); +}); + +test('matho: a chain of equalities and a bare comparison are refused as statements', () => { + const r = parseMatho('a = b = c\nx < 0'); + assert.deepEqual(r.diagnostics.filter((d) => d.severity === 'error').map((d) => d.id), ['matho-chain', 'matho-condition']); +}); + +test('matho: equations split on lines, continue on operators, and ignore document wrappers', () => { + const parts = splitEquations('\\begin{align}\na &= b\n + c \\\\\nd &= e\n\\end{align}\n\nf = \\begin{cases}\n1 & x < 0 \\\\\n0 & \\text{otherwise}\n\\end{cases}\n% a comment\n$$ g = h $$'); + assert.deepEqual(parts.map((p) => p.line), [2, 4, 7, 12]); + assert.match(parts[0].text, /a\s*= b\n\s*\+ c/); + assert.match(parts[2].text, /begin\{cases\}[\s\S]*end\{cases\}/); +}); + +test('matho: the pseudo it produces reads cleanly and lowers with a line map back to the equations', () => { + const r = parseMatho('\\bar{x} = \\frac{1}{n} \\sum_{i=0}^{n-1} x_i\n\n\\sigma^2 = \\frac{1}{n-1} \\sum_{i=0}^{n-1} (x_i - \\bar{x})^2'); + const st = studyOf(r.program.pseudo); + assert.ok(st.ok, st.diagnostics.map((d) => d.message).join(' | ')); + const low = lowerMatho(r, 'pseudo'); + assert.equal(low.target, 'pseudo'); + const from = new Set(low.map.map((m) => m.from)); + assert.deepEqual([...from], [1, 3]); + assert.equal(defaultCallMatho(r), 'SIGMA_SQUARED(4, [1.0, 2.0, 3.0, 4.0])'); + const tr = runMatho(r, {}); + assert.equal(tr.lang, 'matho'); + assert.ok(tr.hits.get(3) > 0, 'the trace is mapped back to equation lines'); +}); + +test('matho: the tokenizer marks commands, refused commands, numbers and names', () => { + const toks = tokenizeMatho('\\frac{1}{n} \\sum_{i=0}^{n-1} x_i \\int % note'); + const cls = (t) => toks.find((x) => x.text === t)?.cls; + assert.equal(cls('\\frac'), 'tok-keyword'); + assert.equal(cls('\\int'), 'tok-error'); + assert.equal(cls('1'), 'tok-number'); + assert.equal(cls('x'), 'tok-name'); + assert.equal(cls('% note'), 'tok-comment'); +}); diff --git a/tests/mojo-check.test.mjs b/tests/mojo-check.test.mjs new file mode 100644 index 0000000..4d8e152 --- /dev/null +++ b/tests/mojo-check.test.mjs @@ -0,0 +1,236 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Programming+'s Mojo checker, scored against the real compiler (godbolt's answers in library/mojo/golden, +// and the compiler's own diagnostic tests). Two rules never bend, and two numbers only go up: +// - no error on a program the compiler accepts (a false accusation teaches the wrong lesson) +// - no error on a rejected program that the compiler did not make there, in those words +// - starters matched exactly, and errors matching the compiler's own annotations: a ratchet +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import crypto from 'node:crypto'; +import { checkMojo } from '../dist/mojo/check.js'; +import { compilerDiagnostics } from '../dist/services/godbolt.js'; + +const R = new URL('..', import.meta.url).pathname; +const read = (p) => JSON.parse(fs.readFileSync(R + p, 'utf8')); +const golden = (key) => key && fs.existsSync(`${R}library/mojo/golden/${key}.json`) ? read(`library/mojo/golden/${key}.json`) : null; +const verdicts = new Map(read('library/mojo/golden/status.json').items.map((r) => [r.id, r])); +const errorsOf = (src) => checkMojo(src).diagnostics.filter((d) => d.severity === 'error'); + +/** What the compiler said, as (line, message): its <source> errors, and the one it gives no line. */ +function compilerSaid(g) { + const ds = compilerDiagnostics(g.result.buildStderr).filter((d) => d.severity === 'error').map((d) => ({ line: d.line, message: d.message })); + const whole = /: error: (module does not contain a 'main' function)/.exec(g.result.buildStderr); + if (whole) ds.push({ line: 1, message: whole[1] }); + return ds; +} + +const MATCHED_AT_LEAST = 43; // raise when a stage makes more starters match (2a: 11, 2b: 23, 2c: 43) +const ANNOTATIONS_AT_LEAST = 21; // 2a: 14, 2b: 18, 2c: 19, 4c: 21 + +test('no false accusations: every program the compiler accepts gets no error', () => { + const items = [...read('library/mojo/corpus/quest/manifest.json').items.map((i) => ['quest', i]), ...read('library/mojo/corpus/manual/manifest.json').items.map((i) => ['manual', i])]; + const accused = []; let checked = 0; + for (const [dir, it] of items) { + const g = golden(verdicts.get(it.id)?.key); + if (!g?.result.compiled) continue; + checked++; + for (const d of errorsOf(fs.readFileSync(`${R}library/mojo/corpus/${dir}/${it.file}`, 'utf8'))) accused.push(`${it.id}: line ${d.line}: ${d.message}`); + } + for (const f of fs.readdirSync(`${R}library/mojo/local`).filter((f) => f.endsWith('.mojo'))) for (const d of errorsOf(fs.readFileSync(`${R}library/mojo/local/${f}`, 'utf8'))) accused.push(`local/${f}: line ${d.line}: ${d.message}`); + // the stress programs: every one is meant to compile, so none may be accused, verified or not + for (const f of fs.readdirSync(`${R}library/mojo/stress`).filter((f) => f.endsWith('.mojo'))) { + const src = fs.readFileSync(`${R}library/mojo/stress/${f}`, 'utf8'); + if (/^# expect: compile-error$/m.test(src)) continue; // kept as checker tests: meant to fail + for (const d of errorsOf(src)) accused.push(`stress/${f}: line ${d.line}: ${d.message}`); + } + assert.ok(checked > 200, `only ${checked} accepted programs found; run pixi run mojo-verify`); + assert.deepEqual(accused, []); +}); + +test('rejected starters: what Programming+ says is what the compiler said', () => { + const wrong = [], matched = []; + for (const it of read('library/mojo/corpus/quest/manifest.json').items.filter((i) => i.id.endsWith('/starter'))) { + const g = golden(verdicts.get(it.id)?.key); + if (!g || g.result.compiled) continue; + const ours = errorsOf(fs.readFileSync(`${R}library/mojo/corpus/quest/${it.file}`, 'utf8')); + if (!ours.length) continue; + const theirs = compilerSaid(g); + const bad = ours.filter((d) => !theirs.some((t) => t.line === d.line && t.message === d.message)); + if (bad.length) wrong.push(`${it.id}: Programming+ said ${bad.map((d) => `line ${d.line}: ${d.message}`).join('; ')} | the compiler: ${theirs.map((t) => `line ${t.line}: ${t.message}`).join('; ')}`); + else matched.push(it.id); + } + assert.deepEqual(wrong, []); + assert.ok(matched.length >= MATCHED_AT_LEAST, `${matched.length} starters matched, fewer than ${MATCHED_AT_LEAST}: ${matched.join(' ')}`); +}); + +// The compiler's harness (MLIR's -verify-diagnostics) matches an annotation as a substring of the message. +test('the compiler\'s own diagnostic tests: every error Programming+ reports there is one they expect', () => { + const toRe = (a) => new RegExp(a.message.split(/(\{\{.*?\}\})/).map((p) => p.startsWith('{{') ? p.slice(2, -2) : p.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('')); + const wrong = []; let matched = 0; + for (const it of read('library/mojo/corpus/diagnostics/manifest.json').items) { + // run with the isolated stdlib or other local files: they know other names, and that mode + // does not check for code at file scope; and as tests, none of them has a main + const special = it.run.some((r) => /isolated|%S\/|-I\s/.test(r)); + const skip = new Set(['no-main', ...(special ? ['unknown-name', 'implicit-declaration', 'file-scope'] : [])]); + for (const d of errorsOf(fs.readFileSync(`${R}library/mojo/corpus/diagnostics/${it.file}`, 'utf8')).filter((d) => !skip.has(d.id))) { + if (it.annotations.some((a) => a.kind === 'error' && a.line === d.line && (a.regex ? toRe(a).test(d.message) : d.message.includes(a.message)))) matched++; + else wrong.push(`${it.id}: line ${d.line}: ${d.message}`); + } + } + assert.deepEqual(wrong, []); + assert.ok(matched >= ANNOTATIONS_AT_LEAST, `${matched} matched annotations, fewer than ${ANNOTATIONS_AT_LEAST}`); +}); + +test('the checks, one by one, in the compiler\'s words', () => { + const say = (src) => errorsOf(src).map((d) => `${d.line}:${d.col} ${d.message}`); + assert.deepEqual(say("def main():\nprint(1)\n").sort(), ["1:12 body must not be empty; use 'pass' or check that the lines below are indented", "2:1 expressions must not appear at file scope; move this into a function body"]); + assert.deepEqual(say('def f(a, b: Int):\n pass\ndef main():\n pass\n'), ['1:7 argument type must be specified']); + assert.deepEqual(say('def main():\n x = 1\n print(x)\n'), ["2:5 implicit declaration of 'x' is not allowed; add 'var' to declare a new name"]); + assert.deepEqual(say('def main():\n print(sqrt(2.0))\n'), ["2:11 use of unknown declaration 'sqrt'; did you mean to import it from 'std.math'? Add 'from std.math import sqrt'"]); + assert.deepEqual(say('def main():\n var count = 1\n print(coun)\n'), [], 'near a declared name the compiler suggests one; Programming+ says nothing rather than guess'); + assert.deepEqual(say('struct S:\n def f() -> Int:\n return 1\ndef main():\n pass\n'), ['2:9 self argument must be present in instance method']); + assert.deepEqual(say('struct S:\n def __init__(x: Int, out self):\n pass\ndef main():\n pass\n'), [], 'out self may come after other parameters'); + assert.deepEqual(say('def boot():\n pass\n'), ["1:1 module does not contain a 'main' function"]); + assert.deepEqual(say('def main():\n var y = 0123\n'), ["2:13 decimal integer literals must not use leading zeros; add '0o' for octal literals"]); + // code the parser cannot read: a note, and no other finding that could be wrong + const unread = checkMojo('def main():\n var x = 1 2\n y = 2\n').diagnostics; + assert.ok(unread.some((d) => d.severity === 'note' && /cannot read this yet/.test(d.message))); + assert.ok(!unread.some((d) => d.id === 'implicit-declaration'), 'no name checks on a file the parser could not fully read'); +}); + +test('Stage 2b: signatures, conventions, raises, moves, constructors, in the compiler\'s words', () => { + const say = (src) => errorsOf(src).map((d) => `${d.line}:${d.col} ${d.message}`); + const notes = (src) => checkMojo(src).diagnostics.filter((d) => d.severity === 'note').map((d) => `${d.line}:${d.col} ${d.message}`); + const f = 'def area(w: Int, h: Int) -> Int:\n return w * h\n\n\n'; + assert.deepEqual(say(f + 'def main():\n print(area(2))\n'), ["6:15 invalid call to 'area': missing required argument: 'h'"]); + assert.deepEqual(notes(f + 'def main():\n print(area(2))\n'), ['1:5 function declared here']); + assert.deepEqual(say(f + 'def main():\n print(area(2, 3, 4))\n'), ["6:22 invalid call to 'area': unexpected argument"]); + assert.deepEqual(say('def go(n: Int, *, fast: Bool) -> Int:\n return n\n\n\ndef main():\n print(go(1, True))\n'), ["6:13 invalid call to 'go': missing required argument: 'fast'"], 'missing wins over unexpected'); + assert.deepEqual(say('def make(x: Int, out result: Int):\n result = x\n\n\ndef main():\n print(make(1))\n'), [], 'an out parameter is filled by the callee'); + const r = 'def read(n: Int) raises -> Int:\n return n\n\n\n'; + assert.deepEqual(say(r + 'def main():\n print(read(1))\n'), ['6:15 cannot call function that may raise in a context that cannot raise']); + assert.deepEqual(notes(r + 'def main():\n print(read(1))\n'), ["6:15 try surrounding the call in a 'try' block", "5:5 or mark surrounding function as 'raises'"]); + assert.deepEqual(say(r + 'def main():\n try:\n print(read(1))\n except e:\n print(e)\n'), []); + assert.deepEqual(say(r + 'def main() raises:\n print(read(1))\n'), []); + assert.deepEqual(say('def safe(n: Int) raises Never -> Int:\n return n\n\n\ndef main():\n print(safe(1))\n'), [], 'raises Never cannot raise'); + assert.deepEqual(say('def main():\n var s = input()\n print(s)\n'), ['2:18 cannot call function that may raise in a context that cannot raise'], 'input() raises (harvested from the stdlib)'); + assert.deepEqual(say('def add(total: Int, x: Int):\n total += x\n\n\ndef main():\n add(1, 2)\n'), ['2:5 expression must be mutable for in-place operator destination']); + assert.deepEqual(say('def add(mut total: Int, x: Int):\n total += x\n\n\ndef main():\n var t = 0\n add(t, 2)\n'), []); + assert.deepEqual(say('def main():\n var xs: List[Int] = [1, 2]\n for x in xs:\n x += 1\n'), ['4:9 expression must be mutable for in-place operator destination']); + assert.deepEqual(say('def main():\n var xs: List[Int] = [1, 2]\n for ref x in xs:\n x += 1\n'), [], 'for ref binds a mutable reference'); + assert.deepEqual(say('def main():\n comptime for _ in range(2):\n _ = 1\n'), [], '_ binds nothing'); + const st = 'struct C(Copyable, Movable):\n var n: Int\n\n def __init__(out self, n: Int):\n self.n = n\n\n def bump(self):\n self.n += 1\n\n\n'; + assert.deepEqual(say(st + 'def main():\n var c = C(0)\n c.bump()\n'), ['8:13 expression must be mutable for in-place operator destination'], 'at the dot of self.n'); + assert.deepEqual(say(st + 'def main():\n var c = C(1, 2)\n'), ['8:13 expression must be mutable for in-place operator destination', '12:14 no matching function in initialization'], 'bump still mutates an immutable self'); + assert.deepEqual(say('struct P(Copyable, Movable):\n var x: Int\n var y: Int\n\n def __init__(out self):\n pass\n\n\ndef main():\n var p = P()\n'), ["5:9 'self.x' is uninitialized at the implicit return from this function"], 'only the first unset field'); + assert.deepEqual(say('def main():\n var a: List[Int] = [1]\n var b = a^\n a.append(2)\n print(len(b))\n'), ["4:13 use of uninitialized value 'a'"]); + assert.deepEqual(say('def main():\n var a: List[Int] = [1]\n var b = a^\n a = [2]\n print(len(a), len(b))\n'), [], 'assigned again before the use'); +}); + +test('Stage 2c: types, in the compiler\'s words, and silence where a type is not certain', () => { + const say = (src) => errorsOf(src).map((d) => `${d.line}:${d.col} ${d.message}`); + const S = 'struct P(Copyable, Movable):\n var x: Int\n\n def __init__(out self, x: Int):\n self.x = x\n\n\n'; + assert.deepEqual(say(S + 'def main():\n var p = P(1)\n print(p.y)\n'), ["10:12 'P' value has no attribute 'y'"]); + assert.deepEqual(say(S + 'def main():\n var p = P(1)\n var q = p.copy()\n print(q.x)\n'), [], 'Copyable gives copy()'); + assert.deepEqual(say(S + 'def main():\n var a = P(1)\n var b = P(2)\n print(a + b)\n'), ["11:13 'P' does not implement the '__add__' method"]); + assert.deepEqual(say(S + 'def main():\n var a = P(1)\n var b = P(2)\n print(a == b)\n'), ["11:13 'P' does not implement the '__eq__' method"]); + assert.deepEqual(say(S.replace('(Copyable, Movable)', '(Copyable, Movable, Equatable)') + 'def main():\n var a = P(1)\n var b = P(2)\n print(a == b)\n'), [], "Equatable's default __eq__ compares the fields"); + assert.deepEqual(say(S + 'def main():\n var a = P(1)\n print(-a)\n'), ["10:11 'P' does not implement the '__neg__' method"]); + assert.deepEqual(say(S + 'def main():\n var a = P(1)\n print(a[0])\n'), ["10:12 'P' is not subscriptable, it does not implement the `__getitem__`/`__setitem__` methods"]); + const copy = checkMojo(S + 'def main():\n var a = P(1)\n var b = a\n print(b.x)\n').diagnostics.find((d) => d.id === 'implicit-copy'); + assert.equal(`${copy.line}:${copy.col} ${copy.message}`, "10:13 value of type 'P' cannot be implicitly copied, it does not conform to 'ImplicitlyCopyable'"); + assert.deepEqual(copy.question.options.map((o) => o.text), ['^', '.copy()'], "the compiler's two fixes"); + assert.deepEqual(say('def take(var xs: List[Int]) -> Int:\n return len(xs)\n\n\ndef main():\n var xs: List[Int] = [1]\n print(take(xs))\n'), ["7:16 value of type 'List[Int]' cannot be implicitly copied, it does not conform to 'ImplicitlyCopyable'"], 'even at its last use'); + assert.deepEqual(say('def main():\n var s = String("a")\n var t = s\n print(t)\n'), [], 'String is implicitly copyable'); + assert.deepEqual(say('def main():\n var xs: List[Int] = [1]\n ref v = xs\n print(v[0])\n'), [], 'ref binds, it does not copy'); + assert.deepEqual(say('def main():\n var n: Int = "5"\n'), ['2:18 cannot implicitly convert \'StringLiteral["5"]\' value to \'Int\'']); + assert.deepEqual(say('def f() -> Int:\n return None\n\n\ndef main():\n print(f())\n'), ["2:12 cannot implicitly convert 'None' value to 'Int'"]); + assert.deepEqual(say('def avg(t: Int, n: Int) -> Float64:\n return t / n\n\n\ndef main():\n print(avg(1, 2))\n'), ["2:14 cannot implicitly convert 'Int' value to 'Float64'"], 'Int / Int is an Int'); + assert.deepEqual(say('def main():\n var n = 10\n var x = 2.5\n print(n * x)\n'), ["4:15 invalid call to '__mul__': value passed to 'rhs' cannot be converted from 'Float64' to 'Int'"]); + assert.deepEqual(say('def main():\n var n = 10\n print(n * 2.5)\n'), [], 'a literal converts'); + assert.deepEqual(say('def main():\n var xs = List()\n xs.append(1)\n'), ['2:18 no matching function in initialization']); + assert.deepEqual(say('def main():\n var xs: List[Int] = List()\n xs.append(1)\n'), [], 'the annotation says what it holds'); + assert.deepEqual(say('def main():\n var d = Dict[String, Int]()\n d["a"] = 1\n print(d["a"])\n'), ['4:12 cannot call function that may raise in a context that cannot raise'], 'reading a key raises; setting one does not'); + const C = 'struct C(Copyable, Movable):\n var deg: Int\n\n def __init__(out self, deg: Int):\n self.deg = deg\n\n\ndef show(t: C):\n print(t.deg)\n\n\n'; + assert.deepEqual(say(C + 'def main():\n show(25)\n'), ["13:10 invalid call to 'show': value passed to 't' cannot be converted from 'IntLiteral[25]' to 'C'"]); + assert.deepEqual(say(C.replace(' def __init__', ' @implicit\n def __init__') + 'def main():\n show(25)\n'), [], 'an @implicit constructor converts'); +}); + +test('lifetime warnings: each is the compiler\'s, at its line and column', () => { + const programs = []; + for (const [dir, mf] of [['quest', 'quest'], ['manual', 'manual']]) for (const it of read(`library/mojo/corpus/${mf}/manifest.json`).items) programs.push([it.id, `library/mojo/corpus/${dir}/${it.file}`]); + for (const f of fs.readdirSync(`${R}library/mojo/probes`)) programs.push([`probes/${f.replace('.mojo', '')}`, `library/mojo/probes/${f}`]); + for (const f of fs.readdirSync(`${R}library/mojo/stress`)) programs.push([`stress/${f.replace('.mojo', '')}`, `library/mojo/stress/${f}`]); + const wrong = []; let matched = 0; + for (const [id, file] of programs) { + const g = golden(verdicts.get(id)?.key); + if (!g?.result.compiled) continue; + const theirs = [...g.result.buildStderr.matchAll(/<source>:(\d+):(\d+): warning: (.*)/g)].map((m) => ({ line: +m[1], col: +m[2], message: m[3] })); + for (const d of checkMojo(fs.readFileSync(R + file, 'utf8')).diagnostics.filter((d) => d.severity === 'warning')) { + if (theirs.some((t) => t.line === d.line && t.col === d.col && t.message === d.message)) matched++; + else wrong.push(`${id}: line ${d.line}:${d.col}: ${d.message}`); + } + } + assert.deepEqual(wrong, []); + assert.ok(matched >= 9, `${matched} warnings matched, fewer than 9`); +}); + +test('Stage 4c: a value read where it may be uninitialized, on any path, in the compiler\'s words', () => { + const say = (src) => errorsOf(src).map((d) => `${d.line}:${d.col} ${d.message}`); + const T = 'def take(var xs: List[Int]):\n pass\n\n\n'; + // moved out on one path, read after the branch: at the ( of the call that reads it (probes/check-use-after-move-on-one-path) + assert.deepEqual(say(T + 'def run(flag: Bool):\n var a: List[Int] = [1]\n if flag:\n take(a^)\n print(len(a))\n\n\ndef main():\n run(True)\n'), ["9:14 use of uninitialized value 'a'"]); + assert.deepEqual(say(T + 'def run(flag: Bool):\n var a: List[Int] = [1]\n if flag:\n take(a^)\n else:\n print(len(a))\n\n\ndef main():\n run(True)\n'), [], 'read only on the path that did not move it'); + assert.deepEqual(say(T + 'def run(flag: Bool):\n var a: List[Int] = [1]\n if flag:\n take(a^)\n a = [2]\n print(len(a))\n\n\ndef main():\n run(True)\n'), [], 'assigned again on the path that moved it'); + // declared without a value + assert.deepEqual(say('def f() -> Int:\n var a: Int\n return a\n\n\ndef main():\n print(f())\n'), ["3:12 use of uninitialized value 'a'"]); + assert.deepEqual(say('def f(c: Bool) -> Int:\n var a: Int\n if c:\n a = 1\n else:\n a = 2\n return a\n\n\ndef main():\n print(f(True))\n'), [], 'assigned on every path'); + assert.deepEqual(say('def f(c: Bool) -> Int:\n var a: Int\n while True:\n if c:\n a = 1\n break\n return a\n\n\ndef main():\n print(f(True))\n'), [], 'while True ends only through its breaks'); +}); + +test('rejected stress programs: whatever Programming+ reports there is what the compiler said', () => { + const wrong = []; let matched = 0, current = 0; + for (const f of fs.readdirSync(`${R}library/mojo/stress`).filter((f) => f.endsWith('.mojo'))) { + const src = fs.readFileSync(`${R}library/mojo/stress/${f}`, 'utf8'); + const g = golden(verdicts.get(`stress/${f.replace('.mojo', '')}`)?.key); + // a golden answers the exact source godbolt saw: an edited file waits for the next verify + if (!g || g.result.compiled || g.sourceSha256 !== crypto.createHash('sha256').update(src).digest('hex')) continue; + current++; + const theirs = compilerSaid(g); + for (const d of errorsOf(src)) { + if (theirs.some((t) => t.line === d.line && t.message === d.message)) matched++; + else wrong.push(`stress/${f}: line ${d.line}: ${d.message}`); + } + } + assert.deepEqual(wrong, []); + if (current >= 2) assert.ok(matched >= 5, `${matched} matched, fewer than the 5 the lambda programs give`); +}); + +test('a function type as an ordinary parameter\'s type: the compiler\'s errors, at its positions', () => { + const say = (src) => errorsOf(src).map((d) => `${d.line}:${d.col} ${d.message}`); + const src = 'def twice(f: def(Int) -> Int, x: Int) -> Int:\n return f(f(x))\n\n\ndef inc(n: Int) -> Int:\n return n + 1\n\n\ndef main():\n print(twice(inc, 1))\n'; + assert.deepEqual(say(src), [ + "2:13 invalid call to '__call__': value cannot be converted from type value '$0' to an instance of '$0'; did you mean to instantiate '$0'?", + "2:15 invalid call to '__call__': value cannot be converted from type value '$0' to an instance of '$0'; did you mean to instantiate '$0'?", + "10:17 invalid call to 'twice': value passed to 'f' cannot be converted from 'def inc(n: Int) thin -> Int' to 'def(Int) -> Int'", + ]); + assert.deepEqual(say('def twice[F: def(Int) -> Int, //](f: F, x: Int) -> Int:\n return f(f(x))\n\n\ndef inc(n: Int) -> Int:\n return n + 1\n\n\ndef main():\n print(twice(inc, 1))\n'), [], 'through a type parameter, as the manual does'); +}); + +test('a t-string is a TString, not a String, and the compiler names it by where its values live (stress/string-builder)', () => { + const say = (src) => errorsOf(src).map((d) => `${d.line}:${d.col} ${d.message}`); + assert.deepEqual(say('def label(name: String, score: Int) -> String:\n return t"{name}: {score}"\n\n\ndef main():\n print(label("a", 1))\n'), ["2:12 cannot implicitly convert 'TString[origin_of(anonymous*, name), origin_of(__call_result_tmp__)]' value to 'String'"]); + assert.deepEqual(say('def main():\n var n = 3\n var s: String = t"n={n}"\n print(s)\n'), ["3:21 cannot implicitly convert 'TString[origin_of(n), origin_of(__call_result_tmp__)]' value to 'String'"]); + // each answer below is the pinned compiler's, asked on this machine: a name once however often it is read, anonymous* once per read + // of a number or Bool parameter passed by value, and the list sorted by character code + assert.deepEqual(say('def f(a: String) -> String:\n return t"{a}{a}"\n\n\ndef main():\n print(f("a"))\n'), ["2:12 cannot implicitly convert 'TString[origin_of(a), origin_of(__call_result_tmp__)]' value to 'String'"]); + assert.deepEqual(say('def f(zed: String, Abe: String, k: Int) -> String:\n var loc = 2.5\n var s = String("q")\n return t"{zed}{k}{Abe}{loc}{s}{k}"\n\n\ndef main():\n print(f("a", "b", 1))\n'), + ["4:12 cannot implicitly convert 'TString[origin_of(Abe, anonymous*, anonymous*, loc, s, zed), origin_of(__call_result_tmp__)]' value to 'String'"]); + assert.deepEqual(say('def f(mut m: Int, var v: Int, b: Bool) -> String:\n return t"{m}{v}{b}"\n\n\ndef main():\n var q = 1\n print(f(q, 2, True))\n'), + ["2:12 cannot implicitly convert 'TString[origin_of(anonymous*, m, v), origin_of(__call_result_tmp__)]' value to 'String'"]); + assert.deepEqual(say('def f(xs: List[Int]) -> String:\n return t"{len(xs)}"\n\n\ndef main():\n print(f([1]))\n'), [], 'an interpolated call: where its value lives is not modeled, so nothing is said'); + assert.deepEqual(say('def main():\n var n = 3\n print(t"n={n}")\n var s = String(t"n={n}")\n print(s)\n'), [], 'print takes a t-string, and String() makes one a String'); + assert.deepEqual(say('def f() -> String:\n return t"{1 + 2}"\n\n\ndef main():\n print(f())\n'), [], 'an interpolated literal: its TString type is not verified, so nothing is said'); +}); diff --git a/tests/mojo-robustness.test.mjs b/tests/mojo-robustness.test.mjs new file mode 100644 index 0000000..4e42ce1 --- /dev/null +++ b/tests/mojo-robustness.test.mjs @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Mojo answers any text with diagnostics and any run with an output, a stop or a refusal: never with an exception of +// the engine underneath, a run without end, or an allocation the size of the program's mistake. Each test is a shape +// that once threw, hung or ran out of memory (a program can arrive in a share link), proven just past its limit with +// inputs small enough to keep the test quick (docs/limits.md lists every limit). +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { parseModule, MAX_NESTING, MAX_TREE } from '../dist/mojo/parse.js'; +import { checkMojo } from '../dist/mojo/check.js'; +import { runMojo, callMojo, RUN_LIMITS } from '../dist/mojo/interp.js'; +import { practiceDiagnostics } from '../dist/mojo/practices.js'; +import { mojoProperties } from '../dist/mojo/properties.js'; +import { mojoFunctions, mojoRoutine } from '../dist/mojo/flowir.js'; +import { tokenizeMojo } from '../dist/mojo/tokenize.js'; +import { lifetimes } from '../dist/mojo/lifetimes.js'; + +const mainOf = (body) => `def main():\n${body.split('\n').map((l) => ' ' + l).join('\n')}\n`; +const fnOf = (body) => `def f(x: Int) -> Int:\n${body.split('\n').map((l) => ' ' + l).join('\n')}\n\n` + mainOf('print(f(1))'); +const notes = (src) => checkMojo(src).diagnostics.filter((d) => d.id === 'unread'); +const errors = (src) => checkMojo(src).diagnostics.filter((d) => d.severity === 'error'); +const run = (src, opts = {}) => runMojo(parseModule(src), src, { record: false, ...opts }); + +/** Every stage a share link reaches, on one text: none may throw. */ +function everyStage(src) { + tokenizeMojo(src); + const { module } = checkMojo(src); + practiceDiagnostics(src, module); + for (const n of mojoFunctions(module).slice(0, 3)) mojoRoutine(module, n); + runMojo(module, src, {}); + mojoProperties(module, src, 5); + return module; +} + +test('nesting past MAX_NESTING is one unread note where the limit is passed, never an error: the compiler accepts 5,000 levels', () => { + assert.equal(MAX_NESTING, 200); + const deep = [ + mainOf(`print(${'('.repeat(30000)}1${')'.repeat(30000)})`), + mainOf(`print(${'('.repeat(30000)}1`), + mainOf(`var a = ${'['.repeat(5000)}1${']'.repeat(5000)}`), + mainOf(`print(${'abs('.repeat(5000)}1${')'.repeat(5000)})`), + mainOf(`print(${'-'.repeat(30000)}1)`), + mainOf(`print(${'not '.repeat(30000)}True)`), + mainOf(`print(1${' ** 1'.repeat(30000)})`), + mainOf(`print(${'1 if True else '.repeat(30000)}0)`), + mainOf(`var f = ${'lambda: '.repeat(30000)}1`), + mainOf(`var x = ${'[y for y in '.repeat(5000)}[1]${']'.repeat(5000)}`), + mainOf(`var x: ${'ref[a] '.repeat(30000)}Int = 1`), + `def main():\n${Array.from({ length: 2000 }, (_, k) => ' '.repeat(k + 1) + 'if True:').join('\n')}\n${' '.repeat(2001)}print(1)\n`, + ]; + for (const src of deep) { + const n = notes(src); + assert.equal(n.length, 1, src.slice(0, 60)); + assert.match(n[0].message, /^nested more than 200 levels deep; Programming\+ cannot read this yet/); + assert.equal(n[0].severity, 'note'); + assert.deepEqual(errors(src), [], 'never an error: the compiler accepts what is merely deep'); + const m = everyStage(src); + assert.ok(m.unread > 0); + assert.match(runMojo(m, src, {}).refused, /cannot read yet/); + } + // just under it, the same shapes read, check and run + const ok = mainOf(`print(${'('.repeat(150)}1${')'.repeat(150)})`); + assert.deepEqual(notes(ok), []); + assert.equal(run(ok).stdout, '1\n'); +}); + +test('a tree taller than MAX_TREE (a long chain, a long elif) is one unread note; the compiler accepts a 10,000-term sum and 5,000 elif arms', () => { + assert.equal(MAX_TREE, 1000); + for (const src of [ + fnOf(`return x${' + 1'.repeat(5000)}`), + mainOf(`print(True${' and True'.repeat(5000)})`), + mainOf(`var s = String("a")\nprint(s${'.copy()'.repeat(5000)})`), + mainOf(`var x = [0]\nprint(x${'[0]'.repeat(5000)})`), + fnOf(`if x == 0:\n return 0\n${Array.from({ length: 5000 }, (_, k) => `elif x == ${k + 1}:\n return ${k}`).join('\n')}\nelse:\n return -1`), + ]) { + const n = notes(src); + assert.equal(n.length, 1); + assert.match(n[0].message, /^more than 1000 levels deep once read/); + assert.deepEqual(errors(src), []); + everyStage(src); + } + // under it, every stage walks the tree and the run answers + assert.equal(run(fnOf(`return x${' + 1'.repeat(900)}`)).stdout, '901\n'); + const elifs = fnOf(`if x == 0:\n return 0\n${Array.from({ length: 900 }, (_, k) => `elif x == ${k + 1}:\n return ${k}`).join('\n')}\nelse:\n return -1`); + assert.deepEqual(notes(elifs), []); + everyStage(elifs); + assert.equal(run(elifs).stdout, '0\n'); +}); + +test('a long flat program reads, checks, gets its advice and runs without a stack or quadratic time', { timeout: 20000 }, () => { + everyStage(mainOf(Array.from({ length: 10000 }, (_, k) => `var x${k} = ${k}`).join('\n') + '\nprint(x0)')); + everyStage(`@fieldwise_init\nstruct S:\n${Array.from({ length: 10000 }, (_, k) => ` var f${k}: Int`).join('\n')}\n\n` + mainOf('print(1)')); + assert.equal(run(mainOf(Array.from({ length: 10000 }, () => 'print(1)').join('\n'))).stdout.length, 20000); + // highlighting a 30,000-operator line is linear (it looked each token's neighbor up by searching the line) + tokenizeMojo(mainOf(`print(${'1 if True else '.repeat(30000)}0)`)); +}); + +test('loops nested deep: the liveness and moved-value analyses give up past their budget instead of taking 2^depth', { timeout: 20000 }, () => { + const nested = (n) => `def main():\n var x = 1\n${Array.from({ length: n }, (_, k) => ' '.repeat(k + 4) + 'while x < 0:').join('\n')}\n${' '.repeat(n + 4)}x = x + 1\n print(x)\n`; + const src = nested(60); + assert.deepEqual(errors(src), []); + const life = lifetimes(parseModule(src).body[0]); + assert.ok(life.unmodeled.has('x'), 'given up: every name is unmodeled, so nothing is destroyed on a guess'); + assert.equal(run(src).stdout, '1\n'); + // a shallow one is still analyzed + assert.ok(!lifetimes(parseModule(nested(3)).body[0]).unmodeled.has('x')); +}); + +test('a function of thousands of parameters, or a program of thousands of functions, gets no generated inputs past the first, not a stack overflow or a hang', () => { + const src = `def f(${Array.from({ length: 3000 }, (_, k) => `p${k}: Int`).join(', ')}) -> Int:\n return p0\n\n` + mainOf('print(1)'); + const [r] = mojoProperties(parseModule(src), src, 5); + assert.match(r.problem, /more than 64 parameters/); + const many = Array.from({ length: 101 }, (_, k) => `def f${k}(x: Int) -> Int:\n return x + ${k}\n`).join('\n') + mainOf('print(1)'); + const reports = mojoProperties(parseModule(many), many, 2); + assert.equal(reports.length, 101); + assert.equal(reports[99].problem, undefined); + assert.match(reports[100].problem, /first 100 functions/); +}); + +test('a value that grows without bound stops the run as its step limit does (limited), before memory runs away', () => { + const limited = (body, opts, re) => { + const r = run(mainOf(body), opts); + assert.equal(r.limited, true, body); + assert.equal(r.exitCode, -1); + assert.match(r.trace.error.message, re); + return r; + }; + // the real limits, where reaching them costs little + assert.equal(RUN_LIMITS.text, 10_000_000); + limited('var t = String("ab")\nfor i in range(40):\n t = t + t\nprint(len(t))', {}, /String of 16,777,216 characters, more than Programming\+ holds \(10,000,000\)/); + limited('var t = String("ab")\nfor i in range(40):\n t += t', {}, /String of 16,777,216/); + limited('var s = String("ab")\nprint(len(s * 1000000000))', {}, /String of 2,000,000,000 characters/); + limited('var xs = [1, 2]\nfor i in range(40):\n xs = xs + xs', {}, /List of 2,097,152 values, more than Programming\+ holds \(2,000,000\)/); + limited('var xs: List[Int] = [1, 2]\nfor i in range(40):\n xs.extend(xs)', {}, /List of 2,097,152 values/); + limited('var s = String("x") * 1000000\nwhile True:\n print(s)', {}, /Stopped after printing 10,000,000 characters/); + limited('var s = String("x") * 10000000\nvar n = 0\nwhile True:\n n += s.find("z")', {}, /searching through 100,000,000 characters and values/); + // a range is never listed past the step limit + limited('var t = 0\nfor i in range(1000000000):\n t += i', {}, /Stopped after 20,000 steps/); + // each limit, made small for the test, is checked where values grow + limited('var d = Dict[Int, Int]()\nfor i in range(100):\n d[i] = i', { limits: { values: 50 } }, /Dict of 51 values, more than Programming\+ holds \(50\)/); + limited('var d = Set[Int]()\nfor i in range(100):\n d.add(i)', { limits: { values: 50 } }, /Set of 51 values/); + limited('var xs = List[Int]()\nwhile True:\n xs.append(1)', { limits: { values: 50 } }, /List of 51 values/); + limited('var xs = List[Int]()\nfor i in range(1000):\n xs.append(i)\nfor k in range(100):\n var ys = xs.copy()\n print(len(ys))', { limits: { made: 5000 } }, /making or copying 5,000 values/); + limited('var xs = List[Int]()\nfor i in range(1000):\n xs.append(i)\nfor k in range(100):\n var ys = List[Int](xs[0:1000])\n print(len(ys))', { limits: { made: 5000 } }, /making or copying 5,000 values/); + limited('var t = String("")\nfor i in range(100):\n t = t + "abcdefghij"', { limits: { work: 10000 } }, /searching through 10,000 characters/); + // std.math's gcd of an even number and a much larger odd one subtracts about 10^15 times in Mojo 1.2.0, and comb loops k times + const math = (body) => run(`from std.math import gcd, comb\n\n\ndef main():\n ${body}\n`); + for (const body of ['print(gcd(6, 4503599627370597))', 'print(comb(100000000, 50000000))']) { + const r = math(body); + assert.equal(r.limited, true, body); + assert.match(r.trace.error.message, /more than Programming\+ follows in one call/); + } + assert.equal(math('print(gcd(6, 4503599))').stdout, '1\n'); + limited('var a = 1\nvar b = 2\nwhile True:\n a += 1', { record: true, limits: { recorded: 100 } }, /recording 100 values of variables/); + // a lower limit can be asked for, not a higher one + assert.equal(run(mainOf('print((String("ab") * 3).byte_length())'), { limits: { text: 1e12 } }).stdout, '6\n'); + // the same program, small, prints what it always printed + assert.equal(run(mainOf('var t = String("ab")\nfor i in range(3):\n t = t + t\nprint(t.byte_length())')).stdout, '16\n'); + assert.equal(run(mainOf('var xs: List[Int] = [1, 2]\nfor i in range(3):\n xs.extend(xs)\nprint(len(xs))')).stdout, '16\n'); +}); + +test('what the engine underneath would refuse is declined, not thrown', () => { + const refused = (body, re) => { const r = run(mainOf(body)); assert.match(r.refused ?? '', re, body); }; + refused('print(1 << 100)', /a shift by more than 64 bits/); + refused('var f: Float64 = 1.0\nprint(Int(f / 0.0))', /converting inf to Int/); + refused('print(1.0e999999 * 2.0)', /exponent past 10,000/); + // [[[...]]] evaluated its first item twice per level: 2^40 evaluations + assert.equal(run(mainOf(`var a = ${'['.repeat(40)}1${']'.repeat(40)}\nprint(len(a))`)).stdout, '1\n'); + // and a call that cannot finish says so + const src = 'def f(n: Int) -> Int:\n return f(n + 1)\n\n' + mainOf('print(f(1))'); + assert.equal(callMojo(parseModule(src), src, 'f', [{ k: 'int', v: 1n, width: 64, signed: true, type: 'Int' }]).kind, 'limit'); +}); + +test('the trace shows a long value cut short, not copied whole into every step', () => { + const src = mainOf('var s = String("x") * 2000\nvar xs = List[Int]()\nfor i in range(30):\n xs.append(i)\nvar d = Dict[Int, Int]()\nfor i in range(30):\n d[i] = i\nprint(s.byte_length())'); + const r = runMojo(parseModule(src), src, {}); + const vars = r.trace.states.at(-1).frames[0].vars; + assert.match(vars.s.text, /^"x{1000}…" \(2,000 characters\)$/); + assert.match(vars.xs.text, /, … 10 more\]$/); + assert.match(vars.d.text, /^\{0: 0, 1: 1, .*, … 10 more\}$/); + assert.equal(r.stdout, '2000\n', 'print still writes the whole value'); +}); diff --git a/tests/mojo-run.test.mjs b/tests/mojo-run.test.mjs new file mode 100644 index 0000000..3a990a7 --- /dev/null +++ b/tests/mojo-run.test.mjs @@ -0,0 +1,262 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Programming+'s Mojo interpreter, judged against the real compiler: for every program godbolt +// compiled and ran (library/mojo/golden), the interpreter either declines ("cannot run this yet") or +// prints exactly what godbolt's run printed and ends the same way. It never disagrees. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import crypto from 'node:crypto'; +import { parseModule } from '../dist/mojo/parse.js'; +import { runMojo } from '../dist/mojo/interp.js'; + +const R = new URL('..', import.meta.url).pathname; +const read = (p) => JSON.parse(fs.readFileSync(R + p, 'utf8')); +const verdicts = new Map(read('library/mojo/golden/status.json').items.map((r) => [r.id, r])); +const golden = (key) => key && fs.existsSync(`${R}library/mojo/golden/${key}.json`) ? read(`library/mojo/golden/${key}.json`) : null; +const run = (src, input = []) => runMojo(parseModule(src), src, { input, maxSteps: 200000 }); +/** + * Programs where the pinned compiler's own answer is a bug of the compiler, each with what it does. The interpreter is + * held to what the program means there, not to the crash, and this test holds the difference: when a new pin answers + * as the interpreter does, the entry fails the test and comes out. + */ +const COMPILER_BUGS = { + 'probes/struct-field-loop-copy': 'Mojo 1.2.0 frees a struct\'s List field twice when it is given a copy inside a loop and nothing reads the struct afterwards: the built program crashes on a later pass (exit 139)', +}; + +const AGREE_AT_LEAST = 287; // raise as the interpreter learns more (3a: 171, 4a: 180, 4b: 182, 4c: 191, 7a: 201, 7b: 203; 265 before P12, P12: 287) + +test('the interpreter never disagrees with godbolt, and agrees on at least the ratchet', () => { + const programs = []; + for (const [dir, mf] of [['quest', 'quest'], ['manual', 'manual']]) for (const it of read(`library/mojo/corpus/${mf}/manifest.json`).items) programs.push([it.id, `library/mojo/corpus/${dir}/${it.file}`]); + for (const f of fs.readdirSync(R + 'library/mojo/probes')) programs.push([`probes/${f.replace('.mojo', '')}`, `library/mojo/probes/${f}`]); + for (const f of fs.readdirSync(R + 'library/mojo/local')) programs.push([`local/${f.replace('.mojo', '')}`, `library/mojo/local/${f}`]); + for (const f of fs.readdirSync(R + 'library/mojo/stress')) programs.push([`stress/${f.replace('.mojo', '')}`, `library/mojo/stress/${f}`]); + const disagree = []; let agree = 0; + for (const [id, file] of programs) { + const g = golden(verdicts.get(id)?.key); + // only godbolt runs to compare with; and none with input: the interpreter hands input() a line per call, as a terminal + // does, and on godbolt's pipe the first input() takes every line (probes/input-lines; docs/trust.md) + if (!g || !g.result.compiled || !g.result.executed || g.mode !== 'run' || g.stdin) continue; + const src = fs.readFileSync(R + file, 'utf8'); + if (g.sourceSha256 && g.sourceSha256 !== crypto.createHash('sha256').update(src).digest('hex')) continue; // an edited file waits for the next verify + const r = runMojo(parseModule(src), src, { maxSteps: 200000, record: false }); + if (r.refused || r.limited) continue; + const norm = (s) => s.replace(/\r\n/g, '\n').replace(/\n+$/, ''); + const same = norm(r.stdout) === norm(g.result.stdout) && r.exitCode === g.result.exitCode; + if (COMPILER_BUGS[id]) { assert.ok(!same, `${id} is listed as a compiler bug, and the compiler now answers as the interpreter does: take it off COMPILER_BUGS`); continue; } + if (same) agree++; + else disagree.push(`${id}: printed ${JSON.stringify(norm(r.stdout).slice(0, 120))} and exited ${r.exitCode}; godbolt printed ${JSON.stringify(norm(g.result.stdout).slice(0, 120))} and exited ${g.result.exitCode}`); + } + assert.deepEqual(disagree, []); + assert.ok(agree >= AGREE_AT_LEAST, `${agree} programs agree, fewer than ${AGREE_AT_LEAST}`); +}); + +/** + * Probes that hold Mojo the translator writes (src/sync/pseudo-to-mojo.ts): the interpreter must run each one, not + * decline it, and print what godbolt printed. A decline here is the Run view refusing the translator's own output. + */ +const TRANSLATOR_SPELLINGS = [ + 'probes/list-literal-into-place', // xs = [1, 2] into a List variable, field, element or Dict entry, then xs.append(...) + 'probes/float-place-int-literal', // a whole-number literal landing in a Float64 place + 'probes/list-slice-copy', // SLICE(L, a, b): List[Int](l[a:b]) + 'probes/list-slice-end-oob', 'probes/list-slice-negative', 'probes/list-slice-reversed', + 'probes/string-ord-chr', // ORD and CHR + 'probes/set-discard-dict-pop', // DELETE: s.discard(x), _ = m.pop(k, default) + 'probes/math-gcd-lcm-comb', // GCD, LCM, CHOOSE + 'probes/math-float-classify', // IS_INF, IS_NAN, IS_FINITE + 'probes/math-pi-literal', // PI(): std.math's pi, folded exactly beside other literals + 'probes/math-copysign-nextafter-ulp', 'probes/math-fma-hypot', // COPYSIGN, NEXT_AFTER, ULP, FMA, HYPOT + 'probes/string-atof-decimal', // TO_FLOAT of a STRING: atof(s) + 'probes/string-codepoint-oob', 'probes/string-codepoint-negative', // S[I] past either end + 'probes/math-inf-nan', // INF() and NAN(): inf[DType.float64](), nan[DType.float64]() +]; +/** Probes of Mojo's match statement (literal, alternative, capture and _ patterns, guards), which must run too. */ +const MATCH = ['probes/match-statement', 'probes/match-literal-capture-guard']; + +test('the Mojo the translator writes runs: every probe that holds one of its spellings runs and agrees with godbolt', () => { + for (const id of [...TRANSLATOR_SPELLINGS, ...MATCH]) { + const g = golden(verdicts.get(id)?.key); + assert.ok(g?.result?.executed, `${id} has no stored run (pixi run mojo-verify)`); + const src = fs.readFileSync(`${R}library/mojo/${id}.mojo`, 'utf8'); + const r = runMojo(parseModule(src), src, { maxSteps: 200000, record: false }); + assert.equal(r.refused, null, `${id}: declined (${r.refused})`); + const norm = (s) => s.replace(/\r\n/g, '\n').replace(/\n+$/, ''); + assert.equal(norm(r.stdout), norm(g.result.stdout), id); + assert.equal(r.exitCode, g.result.exitCode, id); + } +}); + +test('the trace has the shape the stepping views read', () => { + const r = run('def main():\n var total = 0\n for i in range(3):\n total += i\n print(total)\n'); + const t = r.trace; + assert.equal(r.stdout, '3\n'); + assert.equal(t.lang, 'mojo'); + assert.equal(t.states.length, t.steps.length + 1, 'a state before the first step, and one after each'); + const step = t.steps.find((s) => s.text.startsWith('total += i')); + assert.equal(step.line, 4); + assert.deepEqual(step.writes.map((w) => w.name), ['total']); + const last = t.states.at(-1).frames.find((f) => f.name === 'main'); + assert.equal(last.vars.total.text, '3'); + assert.equal(last.vars.total.type, 'Int'); + assert.equal(t.hits.get(4), 3, 'the loop body ran three times'); + const loopState = t.states[t.steps.findIndex((s) => s.kind === 'loop') + 1]; + assert.deepEqual(loopState.loops.map((l) => [l.kind, l.line, l.pass]), [['For', 3, 1]]); +}); + +test('conventions: mut aliases the caller, var copies, ^ moves and empties the source', () => { + assert.equal(run('def bump(mut n: Int):\n n += 1\n\n\ndef main():\n var x = 1\n bump(x)\n print(x)\n').stdout, '2\n'); + assert.equal(run('def grow(var xs: List[Int]) -> Int:\n xs.append(9)\n return len(xs)\n\n\ndef main():\n var a: List[Int] = [1]\n print(grow(a.copy()), len(a))\n').stdout, '2 1\n', 'the copy grew, not a'); + const moved = run('def main():\n var a: List[Int] = [1, 2]\n var b = a^\n print(len(b))\n'); + assert.equal(moved.stdout, '2\n'); + const after = moved.trace.states.at(-1).frames[0].vars; + assert.equal(after.a.text, '(moved out)'); +}); + +test('a numeric literal beside a Float32 is a Float32', () => { + // the pinned compiler's answer for these two lines, asked on this machine (pixi run mojo-local holds generated/50-float32 to it) + const src = 'def main():\n var x: Float32 = 0.1\n print(x == 0.1, Float64(x) == 0.1, x + 0.1, x * 3.0, 0.1 + x, 1.0 - x, x < 0.1, x / 3)\n print(Float32(0.1) >= x, x >= 0.1, 2 * x, x - 1)\n'; + assert.equal(run(src).stdout, 'True False 0.2 0.3 0.2 0.9 False 0.033333335\nTrue True 0.2 -0.9\n'); + // two Float64 values and a literal are untouched + assert.equal(run('def main():\n var y = 0.1\n print(y + 0.2, y == 0.1)\n').stdout, '0.30000000000000004 True\n'); +}); + +test('a float to a whole power is computed by squaring, as Mojo does; to any other power the interpreter declines', () => { + // probes/float-pow-whole, the pinned compiler's answer: 1.1 ** 20 is 6.727499949325609 (the correctly rounded power ends in 611) + const whole = run('def main():\n var x = 2.5\n var y = 1.1\n var n = 3\n print(x ** 2, x ** 3.0, x ** -1.0, y ** 7, y ** 20, y ** 20.0, y ** -3, x ** n)\n'); + assert.equal(whole.stdout, '6.25 15.625 0.4 1.9487171000000012 6.727499949325609 6.727499949325609 0.7513148009015775 15.625\n'); + // probes/float-pow-accuracy: Mojo prints 1.4142135623734946 for 2.0 ** 0.5, which no rule here reproduces + const frac = run('def main():\n var two = 2.0\n print(two ** 0.5)\n'); + assert.match(frac.refused ?? '', /power that is not a whole number/); + assert.equal(run('def main():\n var f: List[Float64] = [1, 2.5]\n print(f)\n').stdout, '[1.0, 2.5]\n', 'a list in brackets takes its items to the element type'); +}); + +test('round() keeps the sign of a value it rounds to zero', () => { + // probes/float-round-sign, the pinned compiler's answer; the interpreter printed 0.0 for the first two + assert.equal(run('def main():\n var a = -0.3\n var b = -0.5\n var c = 0.3\n var d = -0.0\n print(round(a), round(b), round(c), round(d))\n').stdout, '-0.0 -0.0 0.0 -0.0\n'); + assert.equal(run('def main():\n var a = 2.5\n var b = 3.5\n var c = -2.5\n print(round(a), round(b), round(c))\n').stdout, '2.0 4.0 -2.0\n'); +}); + +test('it ends the way the compiled program does', () => { + const oob = run('def main():\n var l: List[Int] = [1, 2, 3]\n var i = 3\n print(l[i])\n'); + assert.equal(oob.stdout, 'At: /app/example.mojo:4:12: Assert Error: index 3 is out of bounds, valid range is 0 to 2\n'); + assert.equal(oob.exitCode, 132); + const raised = run('def main() raises:\n print("before")\n raise Error("boom")\n'); + assert.equal(raised.stdout, 'before\n'); + assert.equal(raised.exitCode, 1); + assert.match(raised.stderr, /Unhandled exception caught during execution: boom/); + const eof = run('def main() raises:\n var s = input()\n print(s)\n'); + assert.match(eof.stderr, /Unhandled exception caught during execution: EOF/, 'input() at the end of input raises EOF'); + assert.equal(run('def main() raises:\n var s = input()\n print(atol(s) + 1)\n', ['41']).stdout, '42\n'); + const loop = run('def main():\n var x = 0\n while True:\n x += 1\n'); + assert.equal(loop.limited, true, 'an endless loop is inconclusive, not an exit'); +}); + +const LOUD = 'struct L(Copyable, Movable):\n var n: String\n\n def __init__(out self, n: String):\n self.n = n\n print("init", n)\n\n def __init__(out self, *, copy: Self):\n self.n = copy.n + "2"\n print("copy", copy.n)\n\n def __init__(out self, *, deinit move: Self):\n self.n = move.n^\n print("move", self.n)\n\n def __deinit__(deinit self):\n print("bye", self.n)\n\n\n'; + +test('Stage 4: a value is destroyed at its last use, and the trace shows it', () => { + const r = run(LOUD + 'def main():\n var a = L("a")\n var b = L("b")\n print("using", a.n)\n print("end")\n'); + // b is never read: it dies as its declaration ends; a dies when the print that borrows a.n returns + assert.equal(r.stdout, 'init a\ninit b\nbye b\nusing a\nbye a\nend\n'); + const destroyed = r.trace.steps.filter((s) => s.kind === 'destroy').map((s) => s.text); + assert.deepEqual(destroyed, ['b is destroyed', 'a is destroyed']); + const last = r.trace.states.at(-1).frames.find((f) => f.name === 'main').vars; + assert.equal(last.a.text, '(destroyed)'); +}); + +test('Stage 4: moves and copies run their constructors where the compiler runs them', () => { + const r = run(LOUD + 'def take(var t: L):\n print("took", t.n)\n\n\ndef main():\n var a = L("a")\n var c = a.copy()\n take(c^)\n var d = a^\n print("kept", d.n)\n'); + // into a var argument, ^ moves no bytes; into a new variable the move constructor runs + assert.equal(r.stdout, 'init a\ncopy a\ntook a2\nbye a2\nmove a\nkept a\nbye a\n'); +}); + +test('Stage 4c: a value dies where the path ahead cannot read it', () => { + // a branch that does not read it: it dies as that branch begins (probes/life-branch-one-side) + const r = run(LOUD + 'def run(flag: Bool):\n var a = L("a")\n if flag:\n print("then", a.n)\n else:\n print("else")\n print("after")\n\n\ndef main():\n run(False)\n'); + assert.equal(r.stdout, 'init a\nbye a\nelse\nafter\n'); + // read in every pass: it dies when the loop ends (probes/life-loop-outer-read-inside) + assert.equal(run(LOUD + 'def main():\n var a = L("a")\n for i in range(2):\n print(i, a.n)\n print("after")\n').stdout, 'init a\n0 a\n1 a\nbye a\nafter\n'); +}); + +test('what it does not model, it declines rather than guesses', () => { + const tryBlock = run(LOUD + 'def main():\n var a = L("a")\n try:\n print(a.n)\n except e:\n print(e)\n'); + assert.match(tryBlock.refused, /does not follow yet/, 'a try\'s paths are not modeled'); + assert.match(tryBlock.trace.error.message, /cannot run this yet/); + assert.ok(run('from std.python import Python\n\n\ndef main() raises:\n var np = Python.import_module("numpy")\n print(np)\n').refused, 'Python interop is declined'); +}); + +test('Stage 4: the trace records each value\'s lifecycle for the Lifetimes view', () => { + const r = run(LOUD + 'def main():\n var a = L("x")\n var b = a.copy()\n var c = a^\n print(b.n, c.n)\n'); + const events = r.trace.steps.flatMap((s) => (s.life ?? []).map((e) => `${e.kind} #${e.id}${e.from ? ` from #${e.from}` : ''}${e.name ? ` as ${e.name}` : ''}`)); + // a move makes a new value (#3) from the old one (#1), which ends there; c and b die after the print, newest first + assert.deepEqual(events, ['made #1', 'named #1 as a', 'copied #2 from #1', 'named #2 as b', 'moved #3 from #1', 'named #3 as c', 'destroyed #3', 'destroyed #2']); +}); + +test('Stage 5: a Mojo function in the shapes the flowchart and data flow read', async () => { + const { mojoRoutine, mojoFunctions } = await import('../dist/mojo/flowir.js'); + const src = 'def bump(mut n: Int, by: Int):\n n += by\n\n\ndef main():\n var x = 1\n var xs: List[Int] = [1]\n if x > 0:\n bump(x, 2)\n elif x < 0:\n print("neg")\n else:\n pass\n for v in xs:\n print(v)\n xs.append(x)\n'; + const mod = parseModule(src); + assert.deepEqual(mojoFunctions(mod), ['bump', 'main']); + const r = mojoRoutine(mod, 'main'); + assert.deepEqual(r.body.map((s) => s.kind), ['Assign', 'Assign', 'If', 'ForEach', 'BuiltinCall']); + const ifs = r.body[2]; + assert.equal(ifs.branches.length, 2, 'an elif is a second branch of one If'); + assert.deepEqual(ifs.branches[0].body.map((s) => s.kind), ['MojoCall']); + assert.deepEqual(ifs.branches[0].body[0].outs.map((o) => o.name), ['x'], 'the argument to a mut parameter changes'); + assert.equal(r.body[3].var, 'v'); + assert.deepEqual(r.symbols, [{ name: 'xs', type: 'List[Int]' }]); + const loop = run('def main():\n for i in range(2):\n print(i)\n'); + assert.equal(loop.trace.steps.filter((s) => s.text === 'no more items').length, 1, 'the loop running out is a step, as a while loop\'s final test is'); +}); + +test('Stage 5: generated inputs find what stops a compiled Mojo program, and nothing else', async () => { + const { mojoProperties } = await import('../dist/mojo/properties.js'); + const src = 'def last(xs: List[Int]) -> Int:\n return xs[len(xs) - 1]\n\n\ndef ratio(a: Int, b: Int) -> Int:\n return a // b\n\n\ndef parse(s: String) raises -> Int:\n return atol(s)\n\n\ndef spin(n: Int) -> Int:\n var i = 0\n while i != n:\n i += 2\n return i\n\n\ndef grow(mut xs: List[Int]):\n xs.append(1)\n\n\ndef main():\n pass\n'; + const by = Object.fromEntries(mojoProperties(parseModule(src), src).map((r) => [r.name, r])); + assert.deepEqual(by.last.failures.map((f) => [f.kind, f.call]), [['bounds', 'last([])']], 'the smallest input that aborts'); + assert.match(by.last.failures[0].message, /index -1 is out of bounds/); + assert.deepEqual(by.ratio.failures, [], 'an Int division by zero is 0 in Mojo (probes/int-div-zero), not a failure'); + assert.deepEqual(by.parse.failures.map((f) => [f.kind, f.call]), [['raises-declared', 'parse("")']], 'one shape of error, its smallest input; it says raises'); + assert.deepEqual(by.spin.failures.map((f) => [f.kind, f.call]), [['limit', 'spin(1)']]); + assert.match(by.grow.problem, /passed as mut/); + assert.ok(by.last.checked >= 100); +}); + +test('Stage 7a: overloads, with, a failing assert, exact literals, block scope, SIMD', () => { + // an overload picked by count, then by type + assert.equal(run('def area(s: Int) -> Int:\n return s * s\n\n\ndef area(w: Int, h: Int) -> Int:\n return w * h\n\n\ndef show(x: Int):\n print("int", x)\n\n\ndef show(x: String):\n print("str", x)\n\n\ndef main():\n print(area(3), area(2, 5))\n show(1)\n show("a")\n').stdout, '9 10\nint 1\nstr a\n'); + // the student's context manager: __enter__, the body, __exit__; an error goes to __exit__(err), True swallows it + const cm = '@fieldwise_init\nstruct G(Copyable, Movable):\n var name: String\n\n def __enter__(self) -> Self:\n print("enter", self.name)\n return self.copy()\n\n def __exit__(self):\n print("exit", self.name)\n\n def __exit__(self, err: Error) -> Bool:\n print("exit with", err)\n return True\n\n\n'; + assert.equal(run(cm + 'def main() raises:\n with G("a"):\n print("body")\n with G("b"):\n raise Error("boom")\n print("after")\n').stdout, 'enter a\nbody\nexit a\nenter b\nexit with boom\nafter\n'); + // a failing assert_equal raises the compiler's AssertionError, catchable like any other (quest/MQ-950/starter) + const failed = run('from std.testing import assert_equal\n\n\ndef main() raises:\n assert_equal(150, 200)\n'); + assert.match(failed.stderr, /At \/app\/example\.mojo:5:17: AssertionError: `left == right` comparison failed:\n left: 150\n right: 200/); + // literal-only arithmetic is exact (manual: 3.0 * (4.0 / 3.0 - 1.0) is 1.0) + assert.equal(run('def main():\n var three = 3.0\n print(3.0 * (4.0 / 3.0 - 1.0), three * (4.0 / three - 1.0))\n').stdout, '1.0 0.9999999999999998\n'); + // var is block-scoped: an inner var shadows, and the outer one is untouched + assert.equal(run('def main():\n var num = 1\n if num == 1:\n var num = 2\n print(num)\n print(num)\n').stdout, '2\n1\n'); + // SIMD (probes/simd-basics), a scalar applying to every lane + assert.equal(run('def main():\n var v = SIMD[DType.int32, 4](1, 5, 1, 9)\n print(v + 1, v.reduce_max())\n').stdout, '[2, 6, 2, 10] 9\n'); +}); + +test('Stage 7b: closures capture as their capture lists say (the manual\'s closures chapter)', () => { + // imm holds a reference: a later change is seen (test_imm_sees_live_updates) + assert.equal(run('def main():\n var limit = 10\n\n def check(x: Int) {imm limit} -> Bool:\n return x < limit\n\n print(check(5))\n limit = 3\n print(check(5))\n').stdout, 'True\nFalse\n'); + // var is a snapshot taken when the closure is made (test_var_snapshot) + assert.equal(run('def main():\n var v = 42\n\n def frozen() {var v} -> Int:\n return v\n\n v = 999\n print(frozen(), v)\n').stdout, '42 999\n'); + // a default alone, and a default with an override (test_default_convention) + assert.equal(run('def main():\n var a = 1\n var b = 2\n var z = "snap"\n\n def mixed() {mut, var z} -> String:\n a += 10\n b += 20\n return String(a) + " " + String(b) + " " + z\n\n print(mixed())\n z = "changed"\n print(mixed())\n').stdout, '11 22 snap\n21 42 snap\n'); + // a lambda captures as imm: a later call sees new values (manual functions/lambda) + assert.equal(run('def main():\n var x = 3\n var f = lambda (k: Int) -> Int: k * x\n print(f(2))\n x = 5\n print(f(2))\n').stdout, '6\n10\n'); + // a named function passed as a value, called through the parameter + assert.equal(run('def twice(n: Int) -> Int:\n return n * 2\n\n\ndef apply(f: def(Int) -> Int, n: Int) -> Int:\n return f(n)\n\n\ndef main():\n print(apply(twice, 4))\n').stdout, '8\n'); + // without a capture list, using an outer variable: the default is not documented, so declined + assert.match(run('def main():\n var a = 1\n\n def f() -> Int:\n return a\n\n print(f())\n').refused, /without a capture list/); +}); + +test('what `pixi run mojo-fuzz --statements` found: the interpreter against the pinned compiler', () => { + // a list literal's first item was evaluated twice, so a call in it ran twice (the compiler prints this, Mojo 1.2.0) + assert.equal(run('def f(n: Int) -> Int:\n print("f", n)\n return n\n\n\ndef main():\n var xs: List[List[Int]] = [[f(1)], [3, f(2)]]\n print(len(xs), xs[1][1])\n').stdout, 'f 1\nf 2\n2 2\n'); + // a whole number appended or inserted into a List[Float64] becomes a Float64 (the compiler prints this, Mojo 1.2.0) + assert.equal(run('def main():\n var xs: List[Float64] = [1.5]\n xs.append(4)\n xs.insert(0, 2)\n var n = 3\n xs.append(Float64(n))\n print(xs)\n').stdout, '[2.0, 1.5, 4.0, 3.0]\n'); +}); diff --git a/tests/mojo.test.mjs b/tests/mojo.test.mjs new file mode 100644 index 0000000..814c6b4 --- /dev/null +++ b/tests/mojo.test.mjs @@ -0,0 +1,291 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The Mojo corpus as the app sees it: the generated modules agree with library/mojo/pin.json +// and the harvested corpus, and the Python pipeline's own tests pass. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { MOJO_VERSION, GODBOLT_BASE, GODBOLT_COMPILER, MOJO_SOURCES } from '../dist/mojo/generated/pin.js'; +import { MOJO_KEYWORDS, MOJO_SOFT_KEYWORDS, MOJO_REMOVED_WORDS, MOJO_DECORATORS, MOJO_PRELUDE, MOJO_PUNCTUATION } from '../dist/mojo/generated/tokens.js'; +import { Workspace } from '../dist/common/workspace.js'; +import { PROBLEMS, problemById, workForProblem } from '../dist/common/problems.js'; + +const pin = JSON.parse(fs.readFileSync(new URL('../library/mojo/pin.json', import.meta.url), 'utf8')); + +test('the generated pin is library/mojo/pin.json', () => { + assert.equal(MOJO_VERSION, pin.mojo.version); + assert.equal(GODBOLT_BASE, pin.godbolt.base); + assert.equal(GODBOLT_COMPILER, pin.godbolt.compiler); + assert.deepEqual(Object.keys(MOJO_SOURCES).sort(), Object.keys(pin.sources).sort()); +}); + +test('every document names the pinned Mojo version, and no other', () => { + // a bump moves the pin in library/mojo/pin.json; these are the sentences that have to move with it + const docs = ['README.md', 'AGENTS.md', ...fs.readdirSync(new URL('../docs', import.meta.url)).filter((f) => f.endsWith('.md')).map((f) => `docs/${f}`)]; + const wrong = []; + for (const d of docs) { + const text = fs.readFileSync(new URL(`../${d}`, import.meta.url), 'utf8'); + text.split('\n').forEach((line, i) => { + for (const m of line.matchAll(/(?:pinned to (?:\*\*)?(?:Mojo )?|verified on Mojo |with Mojo )(\d+\.\d+\.\d+)/g)) if (m[1] !== MOJO_VERSION) wrong.push(`${d}:${i + 1} says ${m[1]}`); + }); + } + assert.deepEqual(wrong, [], `the pin is Mojo ${MOJO_VERSION}`); + assert.match(fs.readFileSync(new URL('../README.md', import.meta.url), 'utf8'), new RegExp(`pinned to \\*\\*Mojo ${MOJO_VERSION.replace(/\./g, '\\.')}\\*\\*`)); + assert.match(fs.readFileSync(new URL('../pixi.toml', import.meta.url), 'utf8'), new RegExp(`^mojo = "==${MOJO_VERSION.replace(/\./g, '\\.')}"`, 'm'), 'pixi.toml installs the pinned compiler'); +}); + +test('the generated words are the pinned language', () => { + for (const w of ['def', 'struct', 'trait', 'var', 'ref', 'comptime', 'raise', 'Self']) assert.ok(MOJO_KEYWORDS.includes(w), w); + assert.ok(!MOJO_KEYWORDS.includes('fn'), 'fn is removed, not a keyword to highlight as valid'); + assert.equal(MOJO_REMOVED_WORDS.fn, 'def'); + assert.equal(MOJO_REMOVED_WORDS.read, 'imm'); + for (const w of ['imm', 'mut', 'out', 'deinit']) assert.ok(MOJO_SOFT_KEYWORDS.includes(w), w); + for (const d of ['fieldwise_init', 'always_inline', 'export']) assert.ok(MOJO_DECORATORS.includes(d), d); + for (const n of ['Int', 'String', 'List', 'Dict', 'Optional', 'print', 'input', 'range', 'len']) assert.ok(MOJO_PRELUDE.includes(n), n); + assert.ok(MOJO_PUNCTUATION.indexOf('**=') < MOJO_PUNCTUATION.indexOf('**'), 'longest spellings first'); +}); + +const py = spawnSync('python3', ['--version']); +// under CI a missing python3 fails this instead of skipping it (a skip in CI is how a gap hides); pixi brings python +test('the corpus pipeline tests pass (python3)', { skip: py.status !== 0 && !process.env.CI ? 'python3 is not installed' : false }, () => { + assert.equal(py.status, 0, 'python3 is not installed, and CI must run this'); + const r = spawnSync('python3', ['-m', 'unittest', 'discover', '-s', 'scripts/mojo/tests'], { encoding: 'utf8', cwd: new URL('..', import.meta.url).pathname }); + assert.equal(r.status, 0, r.stderr.split('\n').slice(-30).join('\n')); +}); + +/* --------------------------------------------------------------------------- + * Stage 1: the Mojo pane. Wherever possible these use godbolt's real answers + * (library/mojo/golden), so they test against what the compiler said, not what we assume. + * ------------------------------------------------------------------------- */ +import { tokenizeMojo } from '../dist/mojo/tokenize.js'; +import { removedSyntax } from '../dist/mojo/check.js'; +import { nextIndent } from '../dist/mojo/editing.js'; +import { mojoRequestBody, mojoRequestUrl, normalizeAnswer, compilerDiagnostics, listingFunctions, demangle, sourceOf as lineSource, BUILD_ARGS, BUILD_LEVELS, mojoBuildArgs } from '../dist/services/godbolt.js'; +import { intelLine, labelMap, forSimulator } from '../dist/mojo/att.js'; +import { parseAsm, runAsmProgram } from '../dist/services/asm.js'; +import { runMojo } from '../dist/mojo/interp.js'; +import { parseModule } from '../dist/mojo/parse.js'; +import { MOJO_PROGRAMS } from '../dist/mojo/generated/programs.js'; + +const ROOT = new URL('..', import.meta.url).pathname; +const goldens = fs.readdirSync(new URL('../library/mojo/golden/', import.meta.url)).filter((f) => /^[0-9a-f]{64}\.json$/.test(f)) + .map((f) => JSON.parse(fs.readFileSync(new URL(`../library/mojo/golden/${f}`, import.meta.url), 'utf8'))); +/** godbolt's answer for one corpus item, rebuilt as the raw JSON the browser receives. */ +function answerFor(item) { + const g = goldens.find((x) => x.firstItem === item); + assert.ok(g, `no golden for ${item}; run pixi run mojo-verify`); + const source = (a) => a.line ? { line: a.line, ...('file' in a ? { file: a.file } : {}), ...('main' in a ? { mainsource: a.main } : {}) } : null; + return { golden: g, raw: { ...g.raw, asm: (g.result.asm || []).map((a) => ({ text: a.text, source: source(a) })) } }; +} +const sourceOf = (item) => fs.readFileSync(new URL(`../library/mojo/${item.replace('probes/', 'probes/')}.mojo`, import.meta.url), 'utf8'); +function applyFix(src, f) { + const lines = src.split('\n'); const l = lines[f.line - 1]; + lines[f.line - 1] = l.slice(0, f.col - 1) + f.text + l.slice(f.endCol == null ? l.length : f.endCol - 1); + return lines.join('\n'); +} +const kinds = (line) => tokenizeMojo(line)[0].filter((t) => t.cls).map((t) => `${t.cls.replace('tok-', '')}:${t.text}`); + +test('highlighting agrees with the Mojo grammar\'s own tests', () => { + const { cases } = JSON.parse(fs.readFileSync(new URL('../library/mojo/corpus/syntax/cases.json', import.meta.url), 'utf8')); + assert.ok(cases.length >= 20); + for (const c of cases) { + const toks = tokenizeMojo(c.text)[0]; + let at = 0; const hit = toks.find((t) => { const ok = c.start >= at && c.start < at + t.text.length; at += t.text.length; return ok; }); + const isKeyword = hit?.cls === 'tok-keyword'; + const wantsKeyword = c.scopes.some((x) => /^(keyword|storage)\./.test(x)); + // syntax 1.1.0 removed (alias) is not a keyword any more, whatever the grammar says + const expected = c.dialect?.older ? false : wantsKeyword; + assert.equal(isKeyword, expected, `${c.file}:${c.line} ${JSON.stringify(c.slice)} in ${JSON.stringify(c.text)} wants ${c.scopes.join(' ')}`); + } +}); + +test('highlighting: docstrings span lines, ^ transfers or XORs, conventions are ownership words', () => { + const doc = tokenizeMojo('def f():\n """One\n two."""\n return 1'); + assert.ok(doc[1].some((t) => t.cls === 'tok-string') && doc[2].every((t) => !t.text.trim() || t.cls === 'tok-string')); + assert.ok(doc[3].some((t) => t.cls === 'tok-keyword' && t.text === 'return')); + assert.ok(kinds('consume(b^, c)').includes('transfer:^')); + assert.ok(kinds('var b = a^').includes('transfer:^')); + assert.ok(kinds('var m = f ^ 0xFF').includes('op:^')); + assert.deepEqual(kinds('def g(mut self, var x: Int, deinit other: T):').filter((k) => k.startsWith('convention')), ['convention:mut', 'convention:var', 'convention:deinit']); + assert.ok(kinds(' var name: String').includes('keyword:var'), 'a field declaration is not a convention'); + assert.ok(kinds('def h(read x: Int):').includes('error:read')); + assert.ok(kinds('var data = file.read()').includes('name:read'), 'read is only an error as a convention'); + assert.ok(kinds('fn old():').includes('error:fn')); + assert.ok(kinds('@fieldwise_init').includes('label:fieldwise_init')); + assert.ok(kinds('print(t"n={n * 2}")').includes('name:n'), 't-string interpolations highlight as code'); +}); + +test('removed syntax is caught before compiling, in the compiler\'s words, with its fix', () => { + const src = 'fn main():\n show(1)\n\ndef show(read x: Int):\n print(x)\n'; + const ds = removedSyntax(src); + assert.deepEqual(ds.map((d) => [d.line, d.message]), [[1, "'fn' has been removed; use 'def' instead"], [4, "'read' was removed; use 'imm'"]]); + let fixed = src; + for (const d of [...ds].reverse()) fixed = applyFix(fixed, d.fix); + assert.equal(fixed, 'def main():\n show(1)\n\ndef show(imm x: Int):\n print(x)\n'); + assert.deepEqual(removedSyntax(fixed), []); + // and it is exactly what the real compiler said about the fn probe + const real = compilerDiagnostics(goldens.find((g) => g.firstItem === 'probes/dialect-fn').result.buildStderr); + assert.equal(real[0].message, ds[0].message); + const realRead = compilerDiagnostics(goldens.find((g) => g.firstItem === 'probes/dialect-read').result.buildStderr); + assert.equal(realRead[0].message, ds[1].message); +}); + +test('Enter indents after a block opener and dedents after return', () => { + assert.equal(nextIndent('def f(x: Int) -> Int:', ' '), ' '); + assert.equal(nextIndent(' comptime for i in range(3):', ' '), ' '); + assert.equal(nextIndent(' return x # done', ' '), ' '); + assert.equal(nextIndent(' var y = 1', ' '), ' '); +}); + +test('the Compile button asks godbolt exactly what the corpus harness asks', () => { + const src = 'def main():\n print(input())\n'; + const py = spawnSync('python3', ['-c', `import sys, json; sys.path.insert(0, 'scripts/mojo')\nfrom mojocorpus import godbolt\nprint(json.dumps(godbolt.request_body(${JSON.stringify(src)}, 'both', '7\\n', '-O0')))`], { encoding: 'utf8', cwd: ROOT }); + if (py.status !== 0) return; // no python3 here; the pipeline tests cover the Python side + assert.deepEqual(mojoRequestBody(src, '7\n', '-O0'), JSON.parse(py.stdout)); + assert.equal(mojoRequestUrl(), `${pin.godbolt.base}/api/compiler/${pin.godbolt.compiler}/compile`); +}); + +test('compiler errors land on their lines, and its fix-its become one-click fixes', () => { + // The probes were asked in run mode, where errors come in buildResult; the button's mode puts them in stderr. Same text. + const stderrOf = (item) => goldens.find((g) => g.firstItem === item).result.buildStderr; + const fnDiag = compilerDiagnostics(stderrOf('probes/dialect-fn')); + assert.equal(fnDiag[0].line, 3); + assert.equal(applyFix(sourceOf('probes/dialect-fn'), fnDiag[0].fix).split('\n')[2], 'def main():'); + const copy = compilerDiagnostics(stderrOf('probes/check-implicit-copy')); + const err = copy.find((d) => d.severity === 'error'); + assert.match(err.message, /cannot be implicitly copied/); + assert.equal(err.origin, `the Mojo ${MOJO_VERSION} compiler`); + const fixes = err.question.options.map((o) => applyFix(sourceOf('probes/check-implicit-copy'), o).split('\n')[err.line - 1].trim()); + assert.deepEqual(fixes, ['var b = a^', 'var b = a.copy()']); + const alias = compilerDiagnostics(stderrOf('probes/check-aliasing')); + assert.match(alias[0].message, /aliasing values passed mutably to 's' argument/); + assert.ok(alias.slice(1).every((d) => d.severity === 'note')); + assert.deepEqual(compilerDiagnostics(stderrOf('probes/check-use-after-move')).map((d) => [d.severity, d.message]), + [['error', "use of uninitialized value 'a'"], ['note', "'a' declared here"]]); +}); + +test('the real listing of add, translated to Intel, runs on the model machine', () => { + const ans = normalizeAnswer(answerFor('probes/godbolt-opt-O0').raw); + const fns = listingFunctions(ans.asm); + assert.deepEqual(fns.filter((f) => f.kind === 'user').map((f) => f.name), ['add(Int, Int)', 'main()']); + assert.deepEqual(fns.filter((f) => f.kind === 'entry').map((f) => f.name), ['main']); + const add = fns.find((f) => f.name === 'add(Int, Int)'); + const sim = forSimulator(fns, add); + // since Mojo 1.2.0 a build carries line tables without being asked, so each instruction arrives with `; mojo line N` + const code = (l) => l.replace(/\s*;.*$/, '').trim(); + assert.deepEqual(sim.source.split('\n').map(code).filter((l) => l && !l.endsWith(':')), ['mov rax, rdi', 'add rax, rsi', 'ret']); + const run = runAsmProgram(parseAsm(sim.source), { setup: 'rdi = 2\nrsi = 3' }); + assert.ok(!run.error, run.error?.message); + assert.equal(run.output.at(-1), 'rax = 5 (0x5)'); + // main calls print, which the model machine cannot run: it assembles with a stub for print, steps up to the call, and stops there saying so + const main = forSimulator(fns, fns.find((f) => f.name === 'main()')); + assert.ok(main.refusedCalls.some((c) => /print/.test(c))); + const mp = parseAsm(main.source); + assert.ok(mp.ok, mp.diagnostics.map((d) => d.message).join('; ')); + const mr = runAsmProgram(mp, { setup: main.data.join('\n') }); + assert.ok(mr.count > 5, 'main steps its own instructions before the call'); + assert.match(mr.error.message, /print: standard library, not on the model machine/); +}); + +test('the optimized listing keeps no function of the user\'s, and one request returned listing and output', () => { + const ans = normalizeAnswer(answerFor('probes/godbolt-asm-and-run').raw); + assert.equal(ans.stdout, '5'); + assert.ok(ans.asm.length > 0 && ans.compiled && ans.executed && ans.exitCode === 0); + assert.deepEqual(listingFunctions(ans.asm).filter((f) => f.kind === 'user'), []); +}); + +test('AT&T to Intel: operands reverse, sizes become ptr, names become labels', () => { + const m = labelMap([{ label: '"example::add(::SIMD[DType.int, 1],::SIMD[DType.int, 1])"', name: 'add(Int, Int)', kind: 'user', lines: [] }]); + const cases = [ + ['movq %rdi, 16(%rsp)', 'mov qword ptr [rsp + 16], rdi'], + ['movl $0, -4(%rbp)', 'mov dword ptr [rbp - 4], 0'], + ['movzbl (%rdi,%rcx), %eax', 'movzx eax, byte ptr [rdi + rcx*1]'], + ['movslq 8(%rsi,%rax,4), %rdx', 'movsxd rdx, dword ptr [rsi + rax*4 + 8]'], + ['leaq static_string_ab(%rip), %rax', 'lea rax, [rip + static_string_ab]'], + ['imulq $3, %rsi, %rax', 'imul rax, rsi, 3'], + ['cmovneq %rcx, %rax', 'cmovne rax, rcx'], + ['callq "example::add(::SIMD[DType.int, 1],::SIMD[DType.int, 1])"@PLT', 'call add'], + ['jne .LBB2_2', 'jne .LBB2_2'], + ['retq', 'ret'], + ['cqto', 'cqo'], + ['.cfi_startproc', null], + ]; + for (const [att, intel] of cases) assert.equal(intelLine(att, m), intel, att); + assert.equal(demangle('"example::add(::SIMD[DType.int, 1],::SIMD[DType.float64, 1])"'), 'add(Int, Float64)'); +}); + +test('an exercise\'s Mojo is a verified program from the corpus', () => { + assert.equal(MOJO_PROGRAMS['count-base'].verified, true); + const p = PROBLEMS.find((x) => x.id === 'count-base'); + assert.equal(p.mojo.id, 'count-base'); + assert.equal(p.sources.mojo, MOJO_PROGRAMS['count-base'].source); + assert.ok(workForProblem(p).sources.mojo, 'the state carries the exercise\'s Mojo'); + for (const q of PROBLEMS) if (q.mojo) assert.ok(MOJO_PROGRAMS[q.mojo.id], `${q.id} names a Mojo program that does not exist`); + // every exercise with pseudo has its Mojo too, so the Mojo/pseudo switch always has both sides + for (const q of PROBLEMS) if (q.sources.pseudo) assert.ok(q.mojo, `${q.id} has pseudo but no Mojo program`); +}); + +test('each exercise\'s Mojo prints, in the interpreter, what the exercise expects of the pseudo', () => { + const norm = (s) => s.trim().replace(/^"(.*)"$/, '$1').replace(/^TRUE$/, 'True').replace(/^FALSE$/, 'False'); + for (const q of PROBLEMS) { + if (!q.mojo || !q.expect) continue; + const r = runMojo(parseModule(q.mojo.source), q.mojo.source, { maxSteps: 200000, record: false }); + if (r.refused) continue; // the stepper says why; the compiler is the judge + assert.equal(norm(r.stdout), norm(q.expect), `${q.id}: its Mojo printed ${JSON.stringify(r.stdout)}`); + } +}); + +test('the Compile button builds with exactly the flags a passing probe verified', () => { + const text = fs.readFileSync(new URL('../library/mojo/probes/godbolt-compile-button.mojo', import.meta.url), 'utf8'); + const args = /^# args: (.*)$/m.exec(text)[1].trim(); + assert.equal(BUILD_LEVELS.O0, args, 'change the probe and re-verify before changing the button'); + assert.equal(BUILD_ARGS.O0, mojoBuildArgs(args), 'the button sends the pin\'s build arguments, then its level\'s'); + const g = goldens.find((x) => x.firstItem === 'probes/godbolt-compile-button' && x.args === BUILD_ARGS.O0); + assert.ok(g, 'godbolt has answered the probe with exactly the arguments the button sends: pixi run mojo-verify'); + assert.equal(g.result.compiled && g.result.executed && g.result.exitCode, 0); + assert.equal(g.result.stdout.trim(), '5'); +}); + +test('a source line is the program\'s only when godbolt says it comes from the program\'s file', () => { + assert.deepEqual(lineSource({ line: 10, file: null }), { line: 10 }); + assert.deepEqual(lineSource({ line: 10 }), { line: 10 }); + assert.deepEqual(lineSource({ line: 10, file: '/app/example.mojo' }), { line: 10 }); + assert.deepEqual(lineSource({ line: 10, file: 'other', mainsource: true }), { line: 10 }); + assert.deepEqual(lineSource({ line: 218, file: '/opt/mojo/lib/std/io/io.mojo' }), { line: null, from: { file: 'io.mojo', line: 218 } }); + assert.deepEqual(lineSource({ line: 218, file: null, mainsource: false }), { line: null, from: { file: '?', line: 218 } }); + assert.deepEqual(lineSource(null), { line: null }); +}); + +test('with line tables, the program\'s own instructions point at the program\'s own lines', (t) => { + const g = goldens.find((x) => x.firstItem === 'probes/godbolt-compile-button'); + if (!g) return t.skip('no golden for probes/godbolt-compile-button'); + const marked = (g.result.asm || []).some((a) => 'file' in a || 'main' in a); + if (!marked) return t.skip("this golden predates file marking; run: pixi run mojo-verify-item 'probes/godbolt-*'"); + const fns = listingFunctions(normalizeAnswer(answerFor('probes/godbolt-compile-button').raw).asm); + const lines = (name) => [...new Set(fns.find((f) => f.name === name).lines.map((l) => l.line).filter((x) => x != null))]; + assert.deepEqual(lines('add(Int, Int)'), [5, 6]); + assert.deepEqual(lines('main()').sort(), [10, 9].sort()); + assert.ok(fns.find((f) => f.name === 'main()').lines.some((l) => l.from && l.from.file.endsWith('.mojo')), 'print\'s inlined defaults are marked as library code'); +}); + +test('Stage 6: a function\'s calls into the standard library are named, from the real listing', async () => { + const g = goldens.find((x) => x.firstItem === 'probes/godbolt-compile-button'); + if (!g) return; + const { libraryCalls } = await import('../dist/ui/views/compiler.js').catch(() => ({ libraryCalls: null })); + if (!libraryCalls) return; // the view needs a DOM to load; covered by the browser check + const fns = listingFunctions(normalizeAnswer(answerFor('probes/godbolt-compile-button').raw).asm); + assert.deepEqual(libraryCalls(fns.find((f) => f.name === 'main()')), ['print (std.io.io)']); + assert.deepEqual(libraryCalls(fns.find((f) => f.name === 'add(Int, Int)')), []); +}); + +test('Stage 6b: code sent to the x86-64 pane keeps each instruction\'s Mojo line', async () => { + const g = goldens.find((x) => x.firstItem === 'probes/godbolt-compile-button'); + if (!g) return; + const { forSimulator } = await import('../dist/mojo/att.js'); + const fns = listingFunctions(normalizeAnswer(answerFor('probes/godbolt-compile-button').raw).asm); + const { source } = forSimulator(fns, fns.find((f) => f.name === 'add(Int, Int)')); + const marks = source.split('\n').filter((l) => /; mojo line \d+/.test(l)).map((l) => Number(/; mojo line (\d+)/.exec(l)[1])); + assert.deepEqual([...new Set(marks)], [5, 6], 'the instructions of add carry lines 5 and 6'); +}); + diff --git a/tests/numeric.test.mjs b/tests/numeric.test.mjs new file mode 100644 index 0000000..6482628 --- /dev/null +++ b/tests/numeric.test.mjs @@ -0,0 +1,382 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Pseudo for numerical work: sized types, FLOAT32, conversions, the numerical built-ins, USE IEEE, and what the +// prover may and may not say about them. The corpus programs 49 to 52 show the same features in use; this file +// pins each rule on its own. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { studyOf, runPseudo, provePseudo } from '../dist/services/pseudo.js'; +import { fma, exactDecimal, nextAfter, ulp, roundDyadic, wrap } from '../dist/engines/pseudo/numeric.js'; +import { intRange, SIZED, formatFloat32 } from '../dist/engines/pseudo/lang.js'; + +const run = (src) => { + const st = studyOf(src); + assert.ok(st.ok, st.diagnostics.filter((d) => d.severity === 'error').map((d) => `L${d.line} ${d.id}: ${d.message}`).join(' | ')); + return runPseudo(st, {}); +}; +const out = (src) => { const r = run(src); assert.equal(r.error, null, r.error?.message); return r.output; }; +const errors = (src) => studyOf(src).diagnostics.filter((d) => d.severity === 'error').map((d) => d.id); + +test('the sized types have the ranges of their names (64-bit types cut at what a run holds exactly)', () => { + assert.deepEqual(intRange(SIZED.INT8), [-128, 127]); + assert.deepEqual(intRange(SIZED.UINT16), [0, 65535]); + assert.deepEqual(intRange(SIZED.INT32), [-2147483648, 2147483647]); + assert.deepEqual(intRange(SIZED.UINT32), [0, 4294967295]); + assert.deepEqual(intRange(SIZED.INT64), [-Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER]); +}); + +test('a FLOAT made into a whole number is one the lab holds exactly, and ROUND takes halves away from zero without rounding twice', () => { + // INT, ROUND, FLOOR and CEIL give an INTEGER, so a result past 2^53 - 1 stops the run as any other does (mojo-fuzz found + // INT(6.02e23) going on as 6.02e23, and the Mojo made from it printing something else) + for (const f of ['INT', 'ROUND', 'FLOOR', 'CEIL']) { + const r = run(`DECLARE X : FLOAT <- 6.02e23\nOUTPUT ${f}(X)`); + assert.equal(r.error?.kind, 'overflow', f); + assert.deepEqual(out(`DECLARE X : FLOAT <- 9007199254740991.0\nOUTPUT ${f}(X)`), ['9007199254740991'], f); + } + assert.equal(run('USE IEEE\nOUTPUT INT(INF())').error?.kind, 'overflow', 'infinity is no whole number, with USE IEEE or without'); + assert.deepEqual(out('OUTPUT ROUND(2.5), " ", ROUND(-2.5), " ", ROUND(0.5), " ", ROUND(1.5), " ", ROUND(-0.5), " ", ROUND(2.4999)'), ['3 -3 1 2 -1 2']); + // 0.49999999999999994 + 0.5 rounds to 1.0, and 4503599627370497.0 + 0.5 to the even neighbor: neither is the nearest whole number + assert.deepEqual(out('OUTPUT ROUND(0.49999999999999994), " ", ROUND(-0.49999999999999994), " ", ROUND(4503599627370497.0), " ", ROUND(-4503599627370497.0)'), ['0 0 4503599627370497 -4503599627370497']); + // never minus zero: an INTEGER has one zero + assert.deepEqual(out('OUTPUT INT(-0.5), " ", CEIL(-0.5), " ", ROUND(-0.2)'), ['0 0 0']); + assert.deepEqual(out('OUTPUT ROUND(7), " ", FLOOR(-2.3), " ", CEIL(2.1), " ", INT(-2.7)'), ['7 -3 3 -2']); +}); + +test('literals: underscores, hexadecimal and binary', () => { + assert.deepEqual(out('OUTPUT 1_000_000, " ", 0xFF, " ", 0b1010, " ", 1_000.5'), ['1000000 255 10 1000.5']); + assert.ok(errors('OUTPUT 1__0').includes('bad-number')); + assert.ok(errors('OUTPUT 0xFG').includes('bad-number')); +}); + +test('a sized result that leaves its range stops the run; WRAP wraps on purpose', () => { + const r = run('DECLARE X : INT8 <- 127\nX <- X + 1\n'); + assert.equal(r.error?.kind, 'overflow'); + assert.equal(r.error.line, 2); + assert.match(r.error.message, /INT8 holds -128 to 127/); + assert.deepEqual(out('DECLARE X : INT8 <- 127\nOUTPUT WRAP(X + 1, INT8), " ", WRAP(-1, UINT8), " ", WRAP(70000, UINT16)'), ['-128 255 4464']); + assert.equal(run('DECLARE U : UINT8 <- 0\nU <- U - 1\n').error?.kind, 'overflow'); + assert.equal(run('DECLARE X : INT8\nX <- -128\nX <- -X\n').error?.kind, 'overflow'); +}); + +test('storing, passing and returning check the range of the receiving type', () => { + assert.equal(run('DECLARE N : INTEGER <- 300\nDECLARE B : UINT8\nB <- N\n').error?.kind, 'overflow'); + const src = 'FUNCTION Id(X : UINT8) RETURNS UINT8\n RETURN X\nENDFUNCTION\nDECLARE N : INTEGER <- 256\nOUTPUT Id(N)\n'; + const r = run(src); + assert.equal(r.error?.kind, 'overflow'); + assert.match(r.error.message, /Parameter X of Id/); + assert.equal(run('FUNCTION Big(N : INTEGER) RETURNS INT16\n RETURN N * 1000\nENDFUNCTION\nOUTPUT Big(40)\n').error?.kind, 'overflow'); +}); + +test('conversions: a FLOAT loses its fraction toward zero, and a value that does not fit stops the run', () => { + assert.deepEqual(out('OUTPUT INT32(-2.9), " ", UINT8(200), " ", INT64(2.5e9), " ", FLOAT32(0.1) == FLOAT32(0.1)'), ['-2 200 2500000000 TRUE']); + assert.deepEqual(errors('OUTPUT INT8(200)'), ['literal-range']); // a literal is checked before any run + assert.equal(run('DECLARE N : INTEGER <- 200\nOUTPUT INT8(N)').error?.kind, 'overflow'); + assert.equal(run('DECLARE F : FLOAT <- 3.0e9\nOUTPUT INT32(F)').error?.kind, 'overflow'); +}); + +test('the checker: widths and precisions do not mix silently, and literals take the type beside them', () => { + assert.deepEqual(errors('DECLARE A : INT8 <- 1\nDECLARE B : INT32 <- 2\nOUTPUT A + B'), ['mixed-width']); + assert.deepEqual(errors('DECLARE A : FLOAT32 <- 1.0\nDECLARE B : FLOAT <- 2.0\nOUTPUT A * B'), ['mixed-precision']); + assert.deepEqual(errors('DECLARE A : INT8\nA <- 300'), ['literal-range']); + assert.deepEqual(errors('DECLARE A : INT8 <- 1\nDECLARE B : INT32 <- 2\nB <- A'), ['type']); + assert.deepEqual(errors('DECLARE A : FLOAT32\nDECLARE D : FLOAT <- 0.5\nA <- D'), ['type']); + assert.deepEqual(errors('DECLARE A : FLOAT32\nDECLARE D : FLOAT <- 0.5\nA <- FLOAT32(D)'), []); + assert.deepEqual(errors('DECLARE A : FLOAT32\nA <- 0.1'), []); + assert.deepEqual(errors('OUTPUT INT8'), ['type-as-value']); + assert.deepEqual(errors('OUTPUT MAX_VALUE(INT64)'), ['exact-limit']); + assert.deepEqual(errors('OUTPUT WRAP(5, UINT64)'), ['wrap-64']); + assert.deepEqual(errors('DECLARE X : INTEGER <- 1\nOUTPUT EPSILON(INT32)'), ['type']); +}); + +test('FLOAT32: results round to single precision, a literal next to one is a single, and it prints as the shortest decimal', () => { + assert.deepEqual(out('DECLARE X : FLOAT32 <- 0.1\nOUTPUT X, " ", X == 0.1, " ", FLOAT64(X) == 0.1, " ", X + 0.2'), ['0.1 TRUE FALSE 0.3']); + assert.deepEqual(out('DECLARE X : FLOAT32 <- 16777216.0\nOUTPUT X + 1.0, " ", FLOAT64(X) + 1.0'), ['16777216.0 16777217.0']); + assert.equal(formatFloat32(Math.fround(0.1)), '0.1'); + assert.equal(formatFloat32(Math.fround(1 / 3)), '0.33333334'); +}); + +test('exact helpers: FMA rounds once, EXACT shows every digit, ULP and NEXT_AFTER step by one value', () => { + assert.equal(fma(0.1, 10, -1), 5.551115123125783e-17); + assert.equal(0.1 * 10 - 1, 0); + assert.equal(fma(2, 3, 4), 10); + assert.equal(fma(Math.fround(0.1), 10, -1, true), Math.fround(1.4901161193847656e-8)); + assert.equal(exactDecimal(0.1), '0.1000000000000000055511151231257827021181583404541015625'); + assert.equal(exactDecimal(-2.5), '-2.5'); + assert.equal(exactDecimal(1e21), '1000000000000000000000.0'); + assert.equal(exactDecimal(Number.MIN_VALUE).length, 1076); // 4.94...e-324 has 1074 digits after the point + assert.equal(nextAfter(1, 2), 1 + Number.EPSILON); + assert.equal(nextAfter(0, -1), -Number.MIN_VALUE); + assert.equal(nextAfter(1, 2, true), 1 + 2 ** -23); + assert.equal(ulp(1), Number.EPSILON); + assert.equal(ulp(Number.MAX_VALUE), 2 ** 971); + assert.equal(ulp(1, true), 2 ** -23); + assert.equal(roundDyadic(3n, -1), 1.5); + assert.equal(roundDyadic((1n << 54n) + 1n, 0), 2 ** 54); // ties and below round to even + assert.equal(wrap(-129, 8, true), 127); + assert.deepEqual(out('OUTPUT FMA(0.1, 10.0, -1.0), " ", ULP(1.0) == EPSILON(FLOAT64), " ", EXACT(0.5)'), ['5.551115123125783e-17 TRUE 0.5']); +}); + +test('whole numbers bit by bit, two\'s complement for negative values', () => { + assert.deepEqual(out('OUTPUT BIT_AND(-1, 255), " ", BIT_OR(-8, 1), " ", BIT_XOR(5, 3), " ", BIT_NOT(5), " ", BIT_NOT(UINT8(5))'), ['255 -7 6 -6 250']); + assert.deepEqual(out('OUTPUT SHIFT_LEFT(3, 4), " ", SHIFT_RIGHT(-9, 1), " ", SHIFT_RIGHT(UINT8(255), 4), " ", GCD(-12, 18), " ", LCM(6, 8)'), ['48 -5 15 6 24']); + assert.equal(run('DECLARE B : UINT8 <- 200\nOUTPUT SHIFT_LEFT(B, 1)').error?.kind, 'overflow'); +}); + +test('the limits of each type', () => { + assert.deepEqual(out('OUTPUT MAX_VALUE(INT8), " ", MIN_VALUE(INT16), " ", MAX_VALUE(UINT32), " ", MIN_VALUE(UINT64)'), ['127 -32768 4294967295 0']); + assert.deepEqual(out('OUTPUT MAX_VALUE(FLOAT64) > 1e308, " ", EPSILON(FLOAT64), " ", MAX_VALUE(FLOAT32)'), ['TRUE 2.220446049250313e-16 3.4028235e38']); +}); + +test('domains: without USE IEEE the run stops where IEEE 754 would give NaN or an infinity; with it, the values flow', () => { + for (const [e, kind] of [['SQRT(-1.0)', 'value'], ['LOG(0.0)', 'value'], ['ASIN(2.0)', 'value'], ['1.0 / 0.0', 'divide'], ['MAX_VALUE(FLOAT64) * 2.0', 'overflow'], ['INF()', 'overflow'], ['LOG1P(-1.0)', 'value']]) { + assert.equal(run(`OUTPUT ${e}`).error?.kind, kind, e); + } + assert.deepEqual(out('USE IEEE\nOUTPUT SQRT(-1.0), " ", 1.0 / 0.0, " ", -1.0 / 0.0, " ", IS_NAN(NAN()), " ", NAN() == NAN()'), ['not a number infinity -infinity TRUE FALSE']); + // whole-number division by zero still stops: IEEE 754 is about FLOAT values + assert.equal(run('USE IEEE\nOUTPUT 1 DIV 0').error?.kind, 'divide'); +}); + +test('names: a program may still call its variable E or its function GCD, and its own definition wins', () => { + assert.deepEqual(out('DECLARE E : FLOAT <- 2.0\nOUTPUT E * 2.0, " ", ROUND(E() * 100)'), ['4.0 272']); + assert.deepEqual(out('FUNCTION GCD(A : INTEGER, B : INTEGER) RETURNS INTEGER\n RETURN 7\nENDFUNCTION\nOUTPUT GCD(12, 18)'), ['7']); + assert.ok(errors('DECLARE LENGTH : INTEGER').includes('builtin-name')); + assert.ok(errors('DECLARE INT8 : INTEGER').length > 0); +}); + +test('the prover: sized arithmetic carries an overflow obligation; FLOAT32 is a function of its own; USE IEEE proves nothing', () => { + const report = (src) => provePseudo(studyOf(src)).units.flatMap((u) => u.obligations.map((o) => ({ ...o, unit: u.name, notes: u.notes }))); + const guarded = report('FUNCTION Inc(X : INT8) RETURNS INT8\n REQUIRE X < 127\n RETURN X + 1\nENDFUNCTION\n'); + assert.deepEqual(guarded.filter((o) => o.kind === 'overflow').map((o) => o.status), ['proved']); + const open = report('FUNCTION Inc(X : INT8) RETURNS INT8\n RETURN X + 1\nENDFUNCTION\n'); + assert.deepEqual(open.filter((o) => o.kind === 'overflow').map((o) => o.status), ['broken']); // the fuzzer tries 127 + const narrow = report('FUNCTION N(K : INTEGER) RETURNS UINT8\n REQUIRE 0 <= K AND K < 256\n RETURN K\nENDFUNCTION\n'); + assert.deepEqual(narrow.filter((o) => o.kind === 'overflow').map((o) => o.status), ['proved']); + const loop = report('FUNCTION Count(N : UINT8) RETURNS UINT8\n DECLARE C : UINT8 <- 0\n FOR I <- 0 TO N\n INVARIANT C == I\n C <- C + 1\n NEXT I\n RETURN C\nENDFUNCTION\n'); + assert.ok(loop.filter((o) => o.kind === 'overflow').every((o) => o.status === 'proved'), JSON.stringify(loop.map((o) => [o.kind, o.status]))); + const half = report('FUNCTION Half(X : FLOAT32) RETURNS FLOAT32\n ENSURE RESULT == X / 2.0\n RETURN X / 2.0\nENDFUNCTION\n'); + assert.ok(half.find((o) => o.kind === 'ensure').status === 'proved'); + const roots = report('FUNCTION R(X : FLOAT) RETURNS FLOAT\n RETURN SQRT(X)\nENDFUNCTION\n'); + assert.deepEqual(roots.filter((o) => o.kind === 'argument').map((o) => o.status), ['broken']); + const ieee = report('USE IEEE\nFUNCTION F(X : INTEGER) RETURNS INTEGER\n REQUIRE X > 0\n RETURN 10 DIV X\nENDFUNCTION\n'); + assert.ok(ieee.length && ieee.every((o) => o.status !== 'proved')); + assert.ok(ieee[0].notes.some((n) => /USE IEEE/.test(n))); +}); + +test('the fuzzer reaches the edges of a sized type and generates FLOAT32 values', async () => { + const { checkProperties } = await import('../dist/engines/pseudo/testing.js'); + const st = studyOf('FUNCTION F(X : UINT8, Y : FLOAT32) RETURNS BOOL\n ASSERT X != 255\n ASSERT FLOAT32(Y) == Y\n RETURN TRUE\nENDFUNCTION\n'); + const fails = checkProperties(st.program.program, 'F', { count: 300 }).failures; + assert.ok(fails.some((f) => f.kind === 'assert' && f.line === 2), 'X = 255 is never tried'); + assert.ok(!fails.some((f) => f.line === 3), 'a generated FLOAT32 is not a single'); +}); + +test('a whole number past 2^53 - 1 does not enter a run: INPUT, TO_INTEGER, a FOR counter and SUM stop as + does', () => { + // DIV and MOD on such a number are computed in doubles and break what the prover knows about them + // (library/pseudo/prover/input-past-exact.pseudo: N MOD 11 came to 64). + const withInput = (src, input) => runPseudo(studyOf(src), { input }); + assert.equal(withInput('DECLARE N : INTEGER\nINPUT N\nOUTPUT N MOD 11', ['-516719406483505152']).error?.kind, 'overflow'); + assert.deepEqual(withInput('DECLARE N : INTEGER\nINPUT N\nOUTPUT N', ['9007199254740991']).output, ['9007199254740991']); + assert.equal(run('OUTPUT TO_INTEGER("-516719406483505152") MOD 11').error?.kind, 'overflow'); + assert.equal(run('FOR I <- 9007199254740990 TO 9007199254740991 STEP 2\n OUTPUT I\nNEXT I').error?.kind, 'overflow'); + assert.equal(run('DECLARE L : LIST OF INTEGER\nL <- [9007199254740991, 1, 1]\nOUTPUT SUM K IN [0, 3) : L[K]').error?.kind, 'overflow'); + assert.equal(run('DECLARE L : LIST OF FLOAT\nL <- [1E308, 1E308]\nOUTPUT SUM K IN [0, 2) : L[K]').error?.kind, 'overflow'); + // a FLOAT read by INPUT is a finite number, unless the program says USE IEEE + assert.equal(withInput('DECLARE F : FLOAT\nINPUT F\nOUTPUT F', ['Infinity']).error?.kind, 'input'); + assert.equal(withInput('USE IEEE\nDECLARE F : FLOAT\nINPUT F\nOUTPUT IS_INF(F)', ['Infinity']).error, null); +}); + +test('text stays what its type says: a CHAR is one character, and CHR gives no half of a surrogate pair', () => { + // the prover counts every CHAR as one character (library/pseudo/prover/chr-surrogate.pseudo) + assert.deepEqual(out("OUTPUT UPPER('ß'), \" \", LENGTH(UPPER('ß') & \"\"), \" \", UPPER('a'), \" \", UPPER(\"straße\")"), ['ß 1 A STRASSE']); + assert.equal(run('OUTPUT LENGTH(CHR(55357) & CHR(56832))').error?.kind, 'value'); + assert.deepEqual(out('OUTPUT LENGTH(CHR(128512) & "")'), ['1']); +}); + +test('RAND(X) is below X even for the smallest FLOAT, where the product would round up to X', () => { + assert.equal(run('DECLARE R : FLOAT\nFOR I <- 0 TO 20\n R <- RAND(5E-324)\n ASSERT R < 5E-324\nNEXT I').error, null); + assert.equal(run('DECLARE R : FLOAT\nFOR I <- 0 TO 20\n R <- RAND(-5E-324)\n ASSERT R > -5E-324\nNEXT I').error, null); +}); + +test('a type says what a value is: CLAMP and a set literal widen, a FLOAT32 answer is rounded, a sized list checks what it receives', () => { + // CLAMP(0, 0.5, 2.5) is 0.5: typed INTEGER (its first value's type) it reached an INTEGER parameter as 0.5 + assert.ok(errors('DECLARE N : INTEGER\nN <- CLAMP(0, 0.5, 2.5)').includes('type')); + assert.deepEqual(out('OUTPUT CLAMP(0, 0.5, 2.5), " ", CLAMP(7, 0, 5)'), ['0.5 5']); + // {1, 2.5} is a SET OF FLOAT, as [1, 2.5] is a LIST OF FLOAT + assert.ok(errors('DECLARE N : INTEGER\nFOR EACH X IN {1, 2.5}\n N <- X\nNEXT X').includes('type')); + // MAX of a FLOAT32 and a whole number is a FLOAT32, so the whole number is rounded to one + assert.deepEqual(out('DECLARE X : FLOAT32 <- 1.0\nDECLARE I : INTEGER <- 16777217\nOUTPUT MAX(X, I) == 16777216, " ", MIN(X, I) == 1'), ['TRUE TRUE']); + // APPEND and INSERT store a value as the list's own type + assert.deepEqual(out('DECLARE L : LIST OF FLOAT32\nDECLARE N : INTEGER <- 16777217\nAPPEND(L, N)\nINSERT(L, 0, N)\nOUTPUT L[0] == 16777216, " ", L[1] == 16777216'), ['TRUE TRUE']); + assert.equal(run('DECLARE L : LIST OF INT8\nDECLARE N : INTEGER <- 300\nAPPEND(L, N)').error?.kind, 'overflow'); +}); + +test('an ARRAY or HASHMAP given as an argument is checked value by value', () => { + const src = 'PROCEDURE Q(A : ARRAY[1] OF INTEGER)\n OUTPUT A[0]\nENDPROCEDURE\nPROCEDURE H(M : HASHMAP OF STRING : INTEGER)\n OUTPUT LENGTH(M)\nENDPROCEDURE'; + const st = studyOf(src); + assert.throws(() => runPseudo(st, { call: 'Q([2.5])' }), /ARRAY/); + assert.throws(() => runPseudo(st, { call: 'H({"a": 2.5})' }), /HASHMAP/); + assert.deepEqual(runPseudo(st, { call: 'Q([2])' }).output, ['2']); + assert.deepEqual(runPseudo(st, { call: 'H({"a": 2})' }).output, ['1']); +}); + +// ---- what the audit of the number model found (work/revive/interp-audit-report.md, section A) + +test('MOD is the exact remainder with the divisor\'s sign, past 2^53 in the product too', () => { + assert.deepEqual(out('OUTPUT 9007199254740991 MOD -3, " ", 9007199254740991 MOD (-2287), " ", (-8306069099471541) MOD 1859998429852587, " ", (-9007199254740991) MOD 3'), ['-2 -420 993923049791394 2']); + assert.deepEqual(out('OUTPUT -7 DIV 2, " ", -7 MOD 2, " ", 7 MOD -2, " ", 7 DIV -2, " ", 6 MOD -3, " ", -6 MOD 3'), ['-4 1 -1 -4 0 0']); +}); + +test('CHOOSE is exact for every count a run can hold', () => { + assert.deepEqual(out('OUTPUT CHOOSE(56, 23), " ", CHOOSE(57, 24), " ", CHOOSE(58, 23), " ", CHOOSE(56, 28), " ", CHOOSE(5, 2), " ", CHOOSE(5, 7), " ", CHOOSE(0, 0)'), + ['3167295784216200 7522327487513475 8799226775309880 7648690600760440 10 0 1']); + assert.equal(run('OUTPUT CHOOSE(60, 30)').error?.kind, 'overflow'); +}); + +test('a sum of whole numbers stops at a partial sum past 2^53 - 1, as + does', () => { + const head = 'DECLARE L : LIST OF INTEGER <- [9007199254740991, 2, -3]\n'; + assert.equal(run(`${head}OUTPUT SUM_OF(L)`).error?.kind, 'overflow'); + assert.equal(run(`${head}OUTPUT MEAN_OF(L)`).error?.kind, 'overflow'); + assert.deepEqual(out('OUTPUT SUM_OF([9007199254740990, -2, 3]), " ", MEAN_OF([1, 2, 6])'), ['9007199254740991 3.0']); +}); + +test('a literal no run could hold is a literal-range error', () => { + assert.ok(errors('OUTPUT 9007199254740993').includes('literal-range')); + assert.ok(errors('OUTPUT 99999999999999999999').includes('literal-range')); + assert.ok(errors('OUTPUT 1e400').includes('literal-range')); + assert.ok(errors('DECLARE X : FLOAT32 <- 1e39').includes('literal-range')); + assert.deepEqual(errors('DECLARE X : INT64 <- 9223372036854775807').filter((id) => id === 'literal-range').length, 1); + assert.deepEqual(out('OUTPUT 9007199254740991, " ", -9007199254740991, " ", 1e308'), ['9007199254740991 -9007199254740991 1.0e308']); +}); + +test('sized results that were not checked: ABS, GCD, BIT_NOT of a UINT64, ADD, a declared FOR counter', () => { + assert.equal(run('DECLARE X : INT8 <- -128\nOUTPUT ABS(X)').error?.kind, 'overflow'); + assert.equal(run('DECLARE X : INT8 <- -128\nOUTPUT GCD(X, INT8(0))').error?.kind, 'overflow'); + assert.equal(run('DECLARE A : UINT64 <- 85\nOUTPUT BIT_NOT(A)').error?.kind, 'overflow'); + assert.deepEqual(out('DECLARE X : INT8 <- -127\nDECLARE U : UINT8 <- 5\nOUTPUT ABS(X), " ", GCD(X, INT8(0)), " ", BIT_NOT(U), " ", BIT_NOT(X)'), ['127 127 250 126']); + assert.equal(run('DECLARE S : SET OF INT8\nDECLARE K : INTEGER <- 1000\nADD(S, K)').error?.kind, 'overflow'); + assert.deepEqual(out('DECLARE S : SET OF FLOAT32\nDECLARE K : INTEGER <- 16777217\nADD(S, K)\nOUTPUT S'), ['{16777216.0}']); + const r = run('DECLARE I : INT8\nFOR I <- 126 TO 130\n OUTPUT I\nNEXT I'); + assert.deepEqual(r.output, ['126', '127']); + assert.equal(r.error?.kind, 'overflow'); + assert.deepEqual(out('DECLARE I : INT8\nFOR I <- 125 TO 128\n OUTPUT I\nNEXT I'), ['125', '126', '127']); + // the prover asks the same obligation where the run now stops + const rep = provePseudo(studyOf('FUNCTION F(X : INT8) RETURNS INT8\n RETURN ABS(X)\nENDFUNCTION')); + assert.ok(rep.units.flatMap((u) => u.obligations).some((o) => o.kind === 'overflow' && /ABS\(X\) fits INT8/.test(o.label) && o.status !== 'proved')); +}); + +test('two different sized types do not mix in MIN, MAX or a list either', () => { + const head = 'DECLARE A : INT8 <- 1\nDECLARE B : INT16 <- 2\n'; + assert.ok(errors(`${head}OUTPUT MAX(A, B)`).includes('mixed-width')); + assert.ok(errors(`${head}OUTPUT MIN(A, 3, B)`).includes('mixed-width')); + assert.ok(errors(`${head}OUTPUT [A, B]`).includes('mixed-width')); + assert.deepEqual(out(`${head}OUTPUT MAX(A, 3), " ", MIN(A, INT8(B)), " ", [A, 3]`), ['3 1 [1, 3]']); +}); + +test('a decimal that becomes a FLOAT32 is rounded once, from its text', () => { + assert.deepEqual(out('DECLARE X : FLOAT32 <- 1.0000000596046448\nOUTPUT EXACT(X), " ", EXACT(FLOAT32(10780.08740234374999999999892))'), ['1.00000011920928955078125 10780.0869140625']); + assert.deepEqual(out('DECLARE X : FLOAT32 <- 0.1\nDECLARE Y : FLOAT32 <- -0.1\nOUTPUT EXACT(X), " ", EXACT(Y), " ", EXACT(FLOAT32(1e-45)), " ", EXACT(FLOAT32(16777217.0)), " ", X + 1.5e0'), + ['0.100000001490116119384765625 -0.100000001490116119384765625 0.00000000000000000000000000000000000000000000140129846432481707092372958328991613128026194187651577175706828388979108268586060148663818836212158203125 16777216.0 1.6']); +}); + +test('USE IEEE: infinities compare as numbers, not-a-number sorts last, FMA keeps an infinite addend', () => { + assert.deepEqual(out('USE IEEE\nOUTPUT INF() <= INF(), " ", INF() >= INF(), " ", -INF() <= -INF(), " ", INF() < INF(), " ", NAN() < 1.0, " ", NAN() > 1.0, " ", NAN() >= NAN()'), ['TRUE TRUE TRUE FALSE FALSE FALSE FALSE']); + assert.deepEqual(out('USE IEEE\nOUTPUT SORTED([3.0, NAN(), 1.0, 2.0])'), ['[1.0, 2.0, 3.0, not a number]']); + assert.deepEqual(out('USE IEEE\nOUTPUT FMA(1e308, 10.0, -INF()), " ", FMA(-1e308, 10.0, INF()), " ", FMA(INF(), 0.0, 1.0)'), ['-infinity infinity not a number']); +}); + +test('a whole number is never minus zero, and a FLOAT that is prints as -0.0', () => { + assert.deepEqual(out('OUTPUT 0 * (-1), " ", EXACT(TO_FLOAT(0 * (-1))), " ", COPYSIGN(1.0, 0 * (-1)), " ", EXACT(TO_FLOAT(0 DIV -5)), " ", EXACT(TO_FLOAT(-4 MOD 2)), " ", EXACT(TO_FLOAT(-0))'), ['0 0.0 1.0 0.0 0.0 0.0']); + assert.deepEqual(out('OUTPUT -0.0, " ", 0.0, " ", TO_STRING(-0.0), " ", EXACT(TO_FLOAT(TO_STRING(-0.0))), " ", 0.0 * -1.0'), ['-0.0 0.0 -0.0 -0.0 -0.0']); +}); + +test('stops say what is true: the limit that was passed, the sign, the built-in', () => { + assert.match(run('DECLARE X : FLOAT32 <- 3e38\nX <- X * 10.0').error.message, /3\.4 × 10\^38.*FLOAT32/); + assert.match(run('OUTPUT (-8.0) ^ 0.5').error.message, /negative number is raised to a power that is not whole/); + assert.match(run('OUTPUT BIT_XOR(-1, 9007199254740991)').error.message, /less than -9,007,199,254,740,991/); + assert.match(run('OUTPUT TO_INTEGER("-9007199254740993")').error.message, /less than -9,007,199,254,740,991/); + assert.match(run('DECLARE L : LIST OF INTEGER <- [1, 2]\nREMOVE(L, 5)').error.message, /^REMOVE\(L, 5\) asks for position 5/); +}); + +test('valid programs the checker used to turn away', () => { + assert.deepEqual(out('DECLARE L : LIST OF FLOAT\nL <- [1, 2]\nOUTPUT L'), ['[1.0, 2.0]']); + assert.deepEqual(out('DECLARE L : LIST OF FLOAT32\nAPPEND(L, 0.1)\nINSERT(L, 0, 0.7)\nOUTPUT EXACT(L[1]), " ", EXACT(L[0])'), ['0.100000001490116119384765625 0.699999988079071044921875']); + assert.deepEqual(out('DECLARE S : SET OF FLOAT32\nADD(S, 0.1)\nOUTPUT FLOAT32(0.1) IN S'), ['TRUE']); + assert.deepEqual(out('FUNCTION F() RETURNS INTEGER\n WHILE TRUE\n RETURN 1\n ENDWHILE\nENDFUNCTION\nOUTPUT F()'), ['1']); + assert.ok(errors('FUNCTION F(X : INTEGER) RETURNS INTEGER\n WHILE TRUE\n IF X > 0 THEN\n BREAK\n ENDIF\n RETURN 1\n ENDWHILE\nENDFUNCTION').includes('missing-return')); +}); + +test('USE IEEE: MIN and MAX with NaN, ULP of an infinity, and POW of 1 follow IEEE 754', () => { + // MIN and MAX: a NaN anywhere among the values gives NaN (754-2019 minimum and maximum), not only in first place + assert.deepEqual(out('USE IEEE\nOUTPUT MAX(1.0, NAN()), " ", MAX(NAN(), 1.0), " ", MIN(2.0, NAN(), 1.0), " ", MIN([3.0, NAN()]), " ", MAX(1.0, INF())'), ['not a number not a number not a number not a number infinity']); + // ULP: an infinity's gap is infinite; NaN's is NaN + assert.deepEqual(out('USE IEEE\nOUTPUT ULP(INF()), " ", ULP(-INF()), " ", IS_NAN(ULP(NAN())), " ", ULP(1.0) == EPSILON(FLOAT64)'), ['infinity infinity TRUE TRUE']); + assert.equal(ulp(Infinity), Infinity); + // pow: 1 to any power is 1, NaN included, and -1 to an infinite power is 1; JavaScript's ** gives NaN for both + assert.deepEqual(out('USE IEEE\nOUTPUT POW(1.0, NAN()), " ", 1.0 ^ NAN(), " ", POW(-1.0, INF()), " ", (-1.0) ^ -INF(), " ", POW(NAN(), 0.0), " ", IS_NAN(POW(2.0, NAN()))'), ['1.0 1.0 1.0 1.0 1.0 TRUE']); + // without USE IEEE nothing changes: such values cannot arise + assert.deepEqual(out('OUTPUT MAX(1.0, 2.5), " ", POW(1.0, 3.0), " ", ULP(1.0) == EPSILON(FLOAT64)'), ['2.5 1.0 TRUE']); +}); + +test('messages and small things: ^ with no answer, INPUT into a FLOAT32, SORTED of lists, deep calls', () => { + // 0.0 ^ -1 divides by zero and (-8.0) ^ 0.5 has no real answer: each says so (they were called overflows), and the + // bracket that makes -8.0 the base is kept (-8.0 ^ 0.5 is -(8.0 ^ 0.5)) + const z = run('OUTPUT 0.0 ^ -1'); + assert.equal(z.error?.kind, 'value'); + assert.match(z.error.message, /^0\.0 \^ -1 has no answer: zero raised to a negative power divides by zero/); + const n = run('OUTPUT (-8.0) ^ 0.5'); + assert.equal(n.error?.kind, 'value'); + assert.match(n.error.message, /^\(-8\.0\) \^ 0\.5 has no real answer/); + assert.deepEqual(out('OUTPUT -8.0 ^ 0.5 < 0.0, " ", (-2.0) ^ 2, " ", -2 ^ 2'), ['TRUE 4.0 -4']); + // BIT_NOT of a UINT64 is past what a run holds: the message gives the exact answer, not a rounded double + const b = run('DECLARE X : UINT64 <- 85\nOUTPUT BIT_NOT(X)'); + assert.equal(b.error?.kind, 'overflow'); + assert.match(b.error.message, /comes to 18,446,744,073,709,551,530 /); + // INPUT into a FLOAT32 rounds once, from the text, as a literal does (1.0000000596046448 lies just past a tie) + const st = studyOf('DECLARE X : FLOAT32\nINPUT X\nOUTPUT EXACT(X)'); + assert.deepEqual(runPseudo(st, { input: ['1.0000000596046448'] }).output, ['1.00000011920928955078125']); + assert.deepEqual(runPseudo(st, { input: ['-0.1'] }).output, [out('OUTPUT EXACT(FLOAT32(-0.1))')[0]]); + assert.deepEqual(runPseudo(studyOf('DECLARE X : FLOAT32\nINPUT X\nOUTPUT X'), { input: ['.5'] }).output, ['0.5']); + // SORTED of lists is in dictionary order, value by value, not the order of their text + assert.deepEqual(out('OUTPUT SORTED([[2], [1, 5], [10], [1]])'), ['[[1], [1, 5], [2], [10]]']); + // calls that go deeper than the engine underneath can follow stop on a line + const deep = run('FUNCTION F(N : INTEGER) RETURNS INTEGER\n IF N == 0 THEN\n RETURN 0\n ENDIF\n WHILE TRUE\n IF TRUE THEN\n IF TRUE THEN\n RETURN F(N - 1) + 1\n ENDIF\n ENDIF\n ENDWHILE\nENDFUNCTION\nOUTPUT F(398)'); + assert.equal(deep.error?.kind, 'depth'); + assert.equal(typeof deep.error.line, 'number'); +}); + +test('TO_FLOAT and INPUT read a decimal number, as the reference says, and nothing else', () => { + assert.deepEqual(out('OUTPUT TO_FLOAT("2.5"), " ", TO_FLOAT(" -1e3 "), " ", TO_FLOAT(".5"), " ", TO_FLOAT("7")'), ['2.5 -1000.0 0.5 7.0']); + for (const t of ['"0x10"', '"0b11"', '""', '"1,5"', '"Infinity"', '"NaN"']) assert.equal(run(`OUTPUT TO_FLOAT(${t})`).error?.kind, 'value', t); + assert.equal(run('OUTPUT TO_FLOAT("1e400")').error?.kind, 'overflow'); + assert.deepEqual(out('USE IEEE\nOUTPUT TO_FLOAT("-Infinity")'), ['-infinity']); + const st = studyOf('DECLARE X : FLOAT\nINPUT X\nOUTPUT X'); + assert.equal(runPseudo(st, { input: ['0x1A'] }).error?.kind, 'input'); + assert.deepEqual(runPseudo(st, { input: ['26'] }).output, ['26.0']); + // TO_INTEGER of a CHAR digit is the digit, not its character code + assert.deepEqual(out("OUTPUT TO_INTEGER('7') + 1"), ['8']); +}); + +test('DOT over whole numbers checks every product and partial sum, as * and + do', () => { + // the first product passes 2^53: it used to be rounded, and the total came back inside the range, wrong (261134297) + const r = run('OUTPUT DOT([94906267, 1], [94906267, -9007199254740991])'); + assert.equal(r.error?.kind, 'overflow'); + assert.equal(run('OUTPUT DOT([9007199254740991, 1], [1, 1])').error?.kind, 'overflow'); + assert.deepEqual(out('OUTPUT DOT([3, 4], [5, 6]), " ", DOT([1.5, 2.0], [2.0, 4.0]), " ", DOT([1, 2], [0.5, 0.25]), " ", DOT([9007199254740991, 1], [1, -1])'), ['39 11.0 1.0 9007199254740990']); +}); + +test('more stops and checker messages say what is true (claims ledger, diagnostics)', () => { + // INPUT of a FLOAT past the largest says so, rather than that the line was no FLOAT + const big = runPseudo(studyOf('DECLARE F : FLOAT\nINPUT F'), { input: ['1e400'] }); + assert.equal(big.error?.kind, 'input'); + assert.match(big.error.message, /read 1e400, which is past the largest FLOAT/); + // INTEGER's limits, and the 64-bit types' "beyond", not "more than", for a negative number + const msg = (src) => studyOf(src).diagnostics.filter((d) => d.severity === 'error').map((d) => d.message).join(' | '); + assert.match(msg('OUTPUT MAX_VALUE(INTEGER)'), /from -9,007,199,254,740,991 to 9,007,199,254,740,991/); + assert.match(msg('OUTPUT MIN_VALUE(INT64)'), /-9,223,372,036,854,775,808, beyond what this lab holds exactly/); + // a range too large to visit: the counted loop is offered only when it could run in the steps a run has + assert.doesNotMatch(run('FOR EACH I IN [0, 3000000)\n OUTPUT I\nNEXT I').error.message, /A counted loop/); + assert.match(run('FOR EACH I IN [0, 3000000)\n OUTPUT I\nNEXT I').error.message, /more passes than a run may take/); + // NORM_OF of one value stops where its square passes the largest FLOAT, about 1.34e154 + assert.deepEqual(out('OUTPUT NORM_OF([1.3e154])'), ['1.3e154']); + assert.equal(run('OUTPUT NORM_OF([1.4e154])').error?.kind, 'overflow'); +}); diff --git a/tests/practices.test.mjs b/tests/practices.test.mjs new file mode 100644 index 0000000..e9c760b --- /dev/null +++ b/tests/practices.test.mjs @@ -0,0 +1,147 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Stage 13a/13d: the best-practices rules. Every rule is held to its own example (the before and after both read, +// run, and print the same; applying the rule to the before gives the after), to the harvested standard-library +// index (every method a rewrite names exists), and to Modular's own tested programs (it must stay quiet there, +// within a small allowlist). Findings are always notes; applying every fix keeps a program passing the checker +// and printing what it printed. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { RULES, findPractices, applyPractices, applyQuickFix, practiceDiagnostics, PRACTICES_ORIGIN } from '../dist/mojo/practices.js'; +import { checkMojo } from '../dist/mojo/check.js'; +import { runMojo } from '../dist/mojo/interp.js'; +import { pseudoToMojo } from '../dist/sync/pseudo-to-mojo.js'; +import { programs } from './helpers/sync-programs.mjs'; +import { EXERCISES } from '../dist/exercises/index.js'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const root = path.join(here, '..'); +const errorsOf = (s) => checkMojo(s).diagnostics.filter((d) => d.severity === 'error'); +const run = (s) => { const m = checkMojo(s); return runMojo(m.module, s); }; + +test('each rule\'s example: both programs read, run, and print the same; the rule fires on the before and not the after', () => { + const ids = new Set(); + for (const r of RULES) { + assert.ok(!ids.has(r.id), `duplicate rule ${r.id}`); ids.add(r.id); + assert.ok(r.title && r.why && r.docUrl.startsWith('https://mojolang.org/docs/manual/'), r.id); + const { before, after, beforeRefused } = r.example; + if (beforeRefused) assert.ok(errorsOf(before).some((d) => d.message.includes(beforeRefused)), `${r.id}: the compiler should refuse the before`); + else assert.deepEqual(errorsOf(before), [], `${r.id}: before`); + assert.deepEqual(errorsOf(after), [], `${r.id}: after`); + const a = run(after); + assert.equal(a.refused, null, `${r.id}: the interpreter runs the after (${a.refused})`); + if (!beforeRefused) { + const b = run(before); + assert.equal(b.refused, null, `${r.id}: the interpreter runs the before (${b.refused})`); + assert.equal(a.stdout, b.stdout, `${r.id}: same output`); + } + assert.ok(findPractices(before).some((f) => f.rule === r.id), `${r.id}: fires on the before`); + assert.ok(!findPractices(after).some((f) => f.rule === r.id), `${r.id}: quiet on the after`); + const first = findPractices(before).find((f) => f.rule === r.id); + if (first.fix) assert.equal(applyPractices(before, new Set([r.id])).source, after, `${r.id}: the fix turns the before into the after`); + else assert.equal(r.id, 'comptime-constant', 'only a rule that cannot promise the rewrite goes without a fix'); + } +}); + +test('grounding: every standard-library method a rewrite names is in the harvested index', () => { + const index = JSON.parse(fs.readFileSync(path.join(root, 'library', 'mojo', 'corpus', 'stdtypes', 'stdtypes.json'), 'utf8')).structs; + let n = 0; + for (const r of RULES) for (const [type, method] of r.apis) { n++; assert.ok(index[type]?.methods.includes(method), `${r.id}: ${type}.${method}`); } + assert.ok(n >= 4); +}); + +function mojoFiles(dir, out = []) { + for (const f of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, f.name); + if (f.isDirectory()) mojoFiles(p, out); else if (p.endsWith('.mojo')) out.push(p); + } + return out; +} + +// What the rules say about Modular's own programs, which are tested and often written to teach one thing at a time +// (a variable declared apart from its value to show how `var` works, `imm` written out to name the convention). Each +// number is a reviewed count; a rule that starts firing more than this on code that is already good is too noisy. +const ALLOWED = { 'declare-then-assign': 10, 'augmented-assign': 6, 'argument-convention': 4, 'unneeded-raises': 3, 'comptime-constant': 3, 'codepoint-loop': 0, 'reserve': 2, 'transfer-last-use': 2, 'explicit-copy': 0 }; +test('false-positive gate: over library/mojo/corpus/manual/ the rules fire no more than the reviewed allowlist', () => { + const files = mojoFiles(path.join(root, 'library', 'mojo', 'corpus', 'manual')); + assert.ok(files.length >= 100, `${files.length} programs`); + const counts = {}; + for (const f of files) for (const x of findPractices(fs.readFileSync(f, 'utf8'))) counts[x.rule] = (counts[x.rule] ?? 0) + 1; + for (const r of RULES) assert.ok((counts[r.id] ?? 0) <= ALLOWED[r.id], `${r.id} fired ${counts[r.id] ?? 0} times on Modular's programs (allowed ${ALLOWED[r.id]})`); +}); + +test('findings are notes from "best practices", never errors or warnings, and a source that does not read gets none', () => { + for (const p of programs().filter((x) => x.id.startsWith('ex:'))) { + const r = pseudoToMojo(p.pseudo, { call: p.call }); + if ('unsupported' in r) continue; + for (const d of practiceDiagnostics(r.source)) { + assert.equal(d.severity, 'note', p.id); + assert.equal(d.origin, PRACTICES_ORIGIN); + assert.ok(d.id.startsWith('practice-') && d.docUrl && d.message, p.id); + } + } + assert.deepEqual(findPractices('def main(:\n var x: Int\n x = 1\n'), []); +}); + +test('count-base: the literal translation draws the declare-then-assign, += and codepoint-loop findings', () => { + const ex = EXERCISES.find((e) => e.id === 'count-base'); + const src = pseudoToMojo(ex.sources.pseudo, { call: ex.call }).source; + const found = new Set(findPractices(src).map((f) => f.rule)); + for (const id of ['declare-then-assign', 'augmented-assign', 'codepoint-loop']) assert.ok(found.has(id), id); + const loop = findPractices(src).find((f) => f.rule === 'codepoint-loop'); + assert.ok(loop.pseudoHint && /FOR EACH/.test(loop.pseudoHint)); + // the same code pasted anywhere (the Developer area) draws the same findings + assert.deepEqual(new Set(findPractices(src.split('\n').slice(1).join('\n')).map((f) => f.rule)), found); +}); + +test('idiomatic version: for every exercise the safe rewrites keep the program checking and printing the same', () => { + let changed = 0, compared = 0; + for (const p of programs().filter((x) => x.id.startsWith('ex:'))) { + const r = pseudoToMojo(p.pseudo, { call: p.call }); + if ('unsupported' in r) continue; + const { source, applied } = applyPractices(r.source); + if (!applied.length) { assert.equal(source, r.source); continue; } + changed++; + assert.ok(errorsOf(source).length <= errorsOf(r.source).length, `${p.id}: the rewrite introduced an error`); + const a = run(r.source), b = run(source); + if (a.refused === null && b.refused === null) { compared++; assert.equal(b.stdout, a.stdout, `${p.id}: same output`); } + else assert.equal(b.refused === null, a.refused === null, `${p.id}: the interpreter should run both or neither (${a.refused} / ${b.refused})`); + assert.ok(source.startsWith(r.source.split('\n')[0]), `${p.id}: the header comment stays`); + } + assert.ok(changed >= 5 && compared >= 5, `${changed} changed, ${compared} compared`); +}); + +test('Developer area: applying every fix to each library/mojo/local program keeps it checking and printing the same', () => { + const dir = path.join(root, 'library', 'mojo', 'local'); + let fired = 0; + for (const f of fs.readdirSync(dir).filter((x) => x.endsWith('.mojo'))) { + const src = fs.readFileSync(path.join(dir, f), 'utf8'); + const { source, applied } = applyPractices(src); + fired += applied.length; + assert.ok(errorsOf(source).length <= errorsOf(src).length, `${f}: an error appeared`); + const a = run(src), b = run(source); + if (a.refused === null) { assert.equal(b.refused, null, `${f}: ${b.refused}`); assert.equal(b.stdout, a.stdout, f); } + } + assert.ok(fired >= 1, 'the rules find something to say somewhere in the exercises\' own programs'); +}); + +test('a fix is the edit the editor makes: whole lines, from line to endLine', () => { + const src = 'a\nb\nc\nd\n'; + assert.equal(applyQuickFix(src, { label: '', line: 2, endLine: 3, col: 1, endCol: null, text: 'X\nY\nZ' }), 'a\nX\nY\nZ\nd\n'); +}); + +test('every exercise\'s generated Mojo has its findings pinned (UPDATE_SNAPSHOTS=1 rewrites them for review)', () => { + const file = path.join(here, 'snapshots', 'practices.json'); + const now = {}; + for (const p of programs().filter((x) => x.id.startsWith('ex:'))) { + const r = pseudoToMojo(p.pseudo, { call: p.call }); + if ('unsupported' in r) continue; + now[p.name] = findPractices(r.source).map((f) => `${f.line}${f.endLine ? '-' + f.endLine : ''} ${f.rule}`); + } + if (process.env.UPDATE_SNAPSHOTS) fs.writeFileSync(file, JSON.stringify(now, null, 2) + '\n'); + assert.ok(fs.existsSync(file), 'no snapshot; run with UPDATE_SNAPSHOTS=1 and review the new file'); + assert.deepEqual(now, JSON.parse(fs.readFileSync(file, 'utf8')), 'the findings for a generated program changed; if the change is wanted, run with UPDATE_SNAPSHOTS=1 and review the diff'); +}); diff --git a/tests/prover-limits.test.mjs b/tests/prover-limits.test.mjs new file mode 100644 index 0000000..b7e44cd --- /dev/null +++ b/tests/prover-limits.test.mjs @@ -0,0 +1,255 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The prover, the generated inputs that cross-check it and the fuzzer answer any program in bounded work and memory: +// each budget is counted (so it ends at the same point on every machine), and past it the answer is "not proved", +// never "proved" and never a missing obligation (docs/limits.md). +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { studyOf, provePseudo } from '../dist/services/pseudo.js'; +import { verify } from '../dist/engines/pseudo/vcgen.js'; +import { prove } from '../dist/engines/pseudo/solver.js'; +import { sym, num, cmp, ite, neg, add, key, show, TOO_LARGE, MAX_KEY } from '../dist/engines/pseudo/logic.js'; + +const DIST = fileURLToPath(new URL('../dist/', import.meta.url)); +const absOf = (n) => `${'ABS('.repeat(n)}X${')'.repeat(n)}`; +const fn = (params, body, contracts = '') => `FUNCTION F(${params}) RETURNS INTEGER\n${contracts}${body}\nENDFUNCTION\n`; +const nestedFor = (n, name = 'F') => fn('N : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${Array.from({ length: n }, (_, k) => `${' '.repeat(k + 1)}FOR I${k} <- 0 TO N`).join('\n')}\n${' '.repeat(n + 1)}ASSERT T >= 0\n${' '.repeat(n + 1)}T <- T + 1\n${Array.from({ length: n }, (_, k) => `${' '.repeat(n - k)}NEXT I${n - 1 - k}`).join('\n')}\n RETURN T`).replace('FUNCTION F(', `FUNCTION ${name}(`); +const program = (src) => { const s = studyOf(src); assert.ok(s.ok, JSON.stringify(s.diagnostics.slice(0, 2))); return s.program.program; }; + +/** Run a module in its own process with a small heap: running out of memory there fails the test, not the suite. */ +function inSmallHeap(code, heapMB = 128) { + const r = spawnSync(process.execPath, [`--max-old-space-size=${heapMB}`, '--input-type=module', '-e', code], { encoding: 'utf8', timeout: 60000, env: { ...process.env, DIST } }); + assert.equal(r.status, 0, `the child ended with ${r.status ?? r.signal}: ${(r.stderr || '').split('\n').filter(Boolean).slice(-3).join(' | ')}`); + return JSON.parse(r.stdout.trim().split('\n').pop()); +} + +test('nested ABS and a matho pane of 20 bars are proved in a 128 MB heap: open, too large to prove here, not out of memory', () => { + const out = inSmallHeap(` + const { studyOf, provePseudo } = await import(process.env.DIST + 'services/pseudo.js'); + const { Workspace } = await import(process.env.DIST + 'common/workspace.js'); + const src = 'FUNCTION F(X : INTEGER) RETURNS INTEGER\\n ENSURE RESULT >= 0\\n RETURN ${absOf(20)}\\nENDFUNCTION\\n'; + const r = provePseudo(studyOf(src)); + const ws = new Workspace({ sources: { matho: 'f(x) = ${'|'.repeat(20)}x${'|'.repeat(20)}' }, lang: 'matho' }); + const m = ws.proof(); + console.log(JSON.stringify({ counts: r.counts, ob: r.units[0].obligations.map((o) => [o.kind, o.status, o.reason]), matho: m && m.report.counts, fault: ws.proofFault })); + `); + assert.deepEqual(out.counts, { proved: 0, broken: 0, open: 1, total: 1 }); + assert.deepEqual(out.ob, [['ensure', 'open', TOO_LARGE]]); + assert.ok(out.matho, 'the matho pane has a proof'); + assert.equal(out.fault, null); +}); + +test('a term that shares its parts has a short key, made once per part: 30 nested ABS are keyed in a 128 MB heap', () => { + const out = inSmallHeap(` + const L = await import(process.env.DIST + 'engines/pseudo/logic.js'); + let t = L.sym('X', 'int'); + for (let i = 0; i < 30; i++) t = L.ite(L.cmp('>=', t, L.num(0)), t, L.neg(t)); + const k = L.key(t); + console.log(JSON.stringify({ len: k.length, again: L.key(t) === k, shown: L.show(t).length })); + `); + assert.ok(out.len <= MAX_KEY + 20, `a key of ${out.len} characters`); + assert.equal(out.again, true); + assert.ok(out.shown < 40000, `show wrote ${out.shown} characters`); + // terms written out in full keep their keys: equal terms built apart are equal, and different ones are not + const x = sym('X', 'int'); + const a = ite(cmp('>=', x, num(0)), x, neg(x)); + const b = ite(cmp('>=', x, num(0)), x, neg(x)); + assert.equal(key(a), key(b)); + assert.notEqual(key(a), key(ite(cmp('>', x, num(0)), x, neg(x)))); + assert.equal(key(add(a, num(1))), key(add(b, num(1)))); +}); + +test('prove: past its work the answer is unknown, too large to prove here, and never proved', () => { + const x = sym('X', 'int'); + const facts = [cmp('>=', x, num(0))]; + const goal = cmp('>=', ite(cmp('>=', x, num(0)), x, neg(x)), num(0)); + assert.equal(prove(facts, goal).status, 'proved'); + const cut = prove(facts, goal, { work: 20 }); + assert.equal(cut.status, 'unknown'); + assert.equal(cut.reason, TOO_LARGE); + // a goal that does not follow is not proved with any budget + for (const work of [5, 50, 500, undefined]) assert.equal(prove([], cmp('>=', x, num(0)), { work }).status, 'unknown'); +}); + +test('a subroutine past the checker\'s work is partial and says so: nothing in it proved, the same answer every time', () => { + const p = program(nestedFor(30)); + const r1 = verify(p, { budgetMs: 600000 }); + const r2 = verify(p, { budgetMs: 600000 }); + assert.equal(r1.units[0].partial, true); + const note = r1.units[0].notes.find((n) => n.kind === 'too-large'); + assert.match(note.text, /too large to prove here.*nothing in it is marked proved/); + assert.deepEqual(r2.units[0].obligations.map((o) => [o.kind, o.line, o.status]), r1.units[0].obligations.map((o) => [o.kind, o.line, o.status])); + const report = provePseudo(studyOf(nestedFor(30))); + assert.equal(report.counts.proved, 0); + // two levels of the same loops are proved as before + const small = verify(program(nestedFor(2))); + assert.equal(small.units[0].partial, false); + assert.ok(small.counts.proved >= 1); +}); + +test('past a subroutine\'s proving work, every obligation is still listed, open and too large to prove here', () => { + const n = 40; + const src = fn('X : INTEGER', `${Array.from({ length: n }, (_, k) => ` ASSERT ${absOf(12).replace('X)', `X - ${k})`)} >= 0`).join('\n')}\n RETURN X`); + const r = verify(program(src), { budgetMs: 600000 }); + const obs = r.units[0].obligations; + assert.equal(obs.length, n); + assert.ok(obs.every((o) => o.status === 'unknown')); + assert.ok(obs.every((o) => o.shown.reason === TOO_LARGE), JSON.stringify([...new Set(obs.map((o) => o.shown.reason))])); +}); + +test('past a program\'s work, the subroutines after it are partial and say so', () => { + const src = [nestedFor(30, 'A'), nestedFor(30, 'B'), nestedFor(30, 'C'), fn('X : INTEGER', ' RETURN X', ' ENSURE RESULT == X\n').replace('FUNCTION F(', 'FUNCTION D(')].join('\n'); + const r = verify(program(src), { budgetMs: 600000 }); + assert.deepEqual(r.units.map((u) => u.partial), [true, true, true, true]); + for (const u of r.units) assert.ok(u.notes.some((n) => n.kind === 'too-large'), u.name); + assert.equal(r.counts.proved, 0); +}); + +test('show writes at most a thousand parts of a term, and the facts shown with an obligation stay short', () => { + let t = sym('X', 'int'); + for (let i = 0; i < 12; i++) t = ite(cmp('>=', t, num(0)), t, neg(t)); + assert.ok(show(t).length < 20000); + assert.match(show(t), /…/); +}); + +/* ---------- the generated inputs that cross-check a proof (testing.js) ---------- */ + +test('checkProperties: a LIST OF a RECORD has no generator, and says so instead of throwing', async () => { + const { checkProperties } = await import('../dist/engines/pseudo/testing.js'); + const { propertiesPseudo } = await import('../dist/services/pseudo.js'); + const src = 'TYPE P\n DECLARE N : INTEGER\nENDTYPE\nFUNCTION Count(Ps : LIST OF P) RETURNS INTEGER\n RETURN LENGTH(Ps)\nENDFUNCTION\n'; + const r = checkProperties(program(src), 'Count'); + assert.equal(r.checked, 0); + assert.match(r.problem, /cannot make up values/); + assert.match(propertiesPseudo(studyOf(src))[0].problem, /cannot make up values/); +}); + +const quantified = (depth) => fn('X : INTEGER', ' RETURN X', ` ENSURE ${'FORALL K IN [0, X + 20) : '.repeat(depth)}TRUE\n`); + +test('checkProperties: maxWork bounds the steps and quantifier positions of all its runs, the same on every machine, and a run it cuts short is not a failure', async () => { + const { checkProperties } = await import('../dist/engines/pseudo/testing.js'); + const p = program(quantified(6)); + const a = checkProperties(p, 'F', { count: 150, maxWork: 200000 }); + const b = checkProperties(p, 'F', { count: 150, maxWork: 200000 }); + assert.equal(a.cutShort, true); + assert.deepEqual(a.failures, []); + // the last run may take what was left in steps and in positions both: never more than twice the budget + assert.ok(a.work >= 200000 && a.work <= 2 * 200000, `work ${a.work}`); + assert.deepEqual([a.checked, a.work], [b.checked, b.work]); + // a loop without end is still a failure where the run's own limit stops it, inside the budget + const forever = program(fn('X : INTEGER', ' WHILE TRUE\n X <- X + 1\n ENDWHILE\n RETURN X')); + assert.deepEqual(checkProperties(forever, 'F', { count: 5, maxWork: 1000000 }).failures.map((f) => f.kind), ['limit']); + assert.deepEqual(checkProperties(forever, 'F', { count: 5, maxWork: 20000 }).failures, []); +}); + +test('the Prove tab\'s cross-check of a routine whose ENSURE nests quantifiers ends within its budget of work', async () => { + const { CROSS_CHECK_ROUTINE } = await import('../dist/services/pseudo.js'); + const t0 = Date.now(); + const r = provePseudo(studyOf(quantified(8))); + assert.equal(r.counts.broken, 0); + assert.ok(CROSS_CHECK_ROUTINE <= 1_000_000); + // before the budget one input of this routine visited 5,000,000 positions (about 7 seconds) + assert.ok(Date.now() - t0 < 30000); +}); + +/* ---------- the fuzzer (Workspace.fuzz, the Tests view) ---------- */ + +test('Workspace.fuzz: the first 100 routines are fuzzed, and the rest say they were not', async () => { + const { Workspace, FUZZ_ROUTINES } = await import('../dist/common/workspace.js'); + const src = Array.from({ length: 130 }, (_, k) => fn('X : INTEGER', ` RETURN X + ${k}`).replace('FUNCTION F(', `FUNCTION F${k}(`)).join('\n'); + const reports = new Workspace({ sources: { pseudo: src }, lang: 'pseudo' }).fuzz({ budget: 100 }); + assert.equal(reports.length, 130); + assert.equal(FUZZ_ROUTINES, 100); + assert.ok(reports.slice(0, 100).every((r) => r.cases.length > 0 && !r.problem)); + assert.ok(reports.slice(100).every((r) => r.cases.length === 0 && /^Not tested: generated inputs cover the first 100 routines/.test(r.problem))); +}); + +test('Workspace.fuzz: a routine stops at its work, a program at its own, and each report says what was not run', async () => { + const { Workspace } = await import('../dist/common/workspace.js'); + const { WORK_PER_INPUT } = await import('../dist/common/fuzz.js'); + const heavy = (name) => quantified(3).replace('FUNCTION F(', `FUNCTION ${name}(`); + const ws = new Workspace({ sources: { pseudo: [heavy('A'), heavy('B'), heavy('C'), fn('X : INTEGER', ' RETURN X').replace('FUNCTION F(', 'FUNCTION D(')].join('\n') }, lang: 'pseudo' }); + const reports = ws.fuzz({ budget: 100 }); + const [a, b, c, d] = reports; + for (const r of [a, b]) { + assert.match(r.problem, /^Stopped after [\d,]+ steps of work, all one routine's inputs may take here/); + assert.ok(r.work <= 2 * 100 * WORK_PER_INPUT, `${r.name} took ${r.work}`); + assert.deepEqual(r.failures, []); + } + for (const r of [c, d]) assert.match(r.problem, /^Not tested: the routines before this one took all the work the tests of one program may take/); + // the same report again, on every machine + assert.deepEqual(new Workspace(ws.state).fuzz({ budget: 100 }).map((r) => [r.cases.length, r.work, r.problem]), reports.map((r) => [r.cases.length, r.work, r.problem])); +}); + +test('Workspace.fuzz: a failure found before the work runs out is kept, however far shrinking got', async () => { + const { fuzzPseudo } = await import('../dist/common/fuzz.js'); + const p = program(fn('X : INTEGER', ' DECLARE T : INTEGER\n T <- 0\n FOR I <- 0 TO 2000\n T <- T + 1\n NEXT I\n ASSERT X < 3\n RETURN T')); + const r = fuzzPseudo(p, 'F', { budget: 300, work: 100000 }); + assert.match(r.problem, /^Stopped after/); + assert.deepEqual(r.failures.map((f) => f.kind), ['assert']); +}); + +/* ---------- how long a pseudocode text is read (services/pseudo-study.ts) ---------- */ + +test('pseudo: a program past 2,000 lines or 100,000 characters is one located too-long error, and the rest is not read', async () => { + const { MAX_PSEUDO_LINES, MAX_PSEUDO_CHARS } = await import('../dist/services/pseudo-study.js'); + assert.deepEqual([MAX_PSEUDO_LINES, MAX_PSEUDO_CHARS], [2000, 100000]); + const body = (n) => fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${' T <- T + X\n'.repeat(n)} RETURN T`); + const tooLong = (s) => s.diagnostics.filter((d) => d.id === 'too-long'); + // 2,000 lines read; trailing blank lines past them do not count + const at = body(MAX_PSEUDO_LINES - 5); + assert.equal(at.split('\n').length, MAX_PSEUDO_LINES + 1); + assert.equal(studyOf(at + '\n\n \n').ok, true); + const long = studyOf(body(MAX_PSEUDO_LINES - 4)); + assert.equal(long.ok, false); + assert.deepEqual(long.diagnostics.map((d) => [d.id, d.line]), [['too-long', MAX_PSEUDO_LINES + 1]]); + assert.match(tooLong(long)[0].message, /more than 2,000 lines long, which is more than this tool reads/); + // a few long lines: cut at the line that passes 100,000 characters + const wide = fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${` T <- T${' + X'.repeat(390)}\n`.repeat(80)} RETURN T`); + const w = studyOf(wide); + const line = wide.split('\n').findIndex((_, i, ls) => ls.slice(0, i + 1).join('\n').length + 1 > MAX_PSEUDO_CHARS) + 1; + assert.deepEqual(tooLong(w).map((d) => d.line), [line]); + assert.match(tooLong(w)[0].message, /longer than 100,000 characters/); + // every pane says so, and nothing throws + const { Workspace } = await import('../dist/common/workspace.js'); + const ws = new Workspace({ sources: { pseudo: wide }, lang: 'pseudo' }); + assert.ok(ws.allDiagnostics().some((d) => d.id === 'too-long')); + assert.equal(ws.proof(), null); + assert.deepEqual(ws.tests(), []); +}); + +test('pseudo analysis: a name defined on more than 64 lines reaches its uses as before, read a word at a time', () => { + const src = fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${Array.from({ length: 80 }, (_, k) => ` IF X > ${k} THEN\n T <- T + 1\n ENDIF`).join('\n')}\n RETURN T`); + const s = studyOf(src); + assert.ok(s.ok); + const unit = s.program.analysis.units[0]; + const ret = unit.uses.find((u) => u.var === 'T' && u.line === 3 + 80 * 3 + 1); + // the RETURN is reached by T <- 0 and by every one of the 80 increments, the latest first + assert.deepEqual(ret.defs.map((d) => d.line), [...Array.from({ length: 80 }, (_, k) => 3 + 80 * 3 - 1 - 3 * k), 3]); +}); + +test('a value built from thousands of operations one inside another becomes an unknown, not a stack overflow', async () => { + const { MAX_DEPTH } = await import('../dist/engines/pseudo/vcgen.js'); + const { Workspace } = await import('../dist/common/workspace.js'); + // 60 lines of 390-term sums: a value 23,000 operations deep, which ran the prover past the engine's stack + const src = fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${` T <- T${' + X'.repeat(390)}\n`.repeat(60)} ASSERT X == X\n RETURN T`, ' REQUIRE X >= 0\n ENSURE RESULT >= 0\n'); + const r = verify(program(src), { budgetMs: 600000 }); + assert.equal(r.units[0].partial, false); + assert.match(r.units[0].notes.find((n) => n.kind === 'too-deep').text, new RegExp(`more than ${MAX_DEPTH} operations one inside another`)); + // what does not depend on the value is still proved, and what does is open + assert.deepEqual(r.units[0].obligations.map((o) => [o.kind, o.status]), [['assert', 'proved'], ['ensure', 'unknown']]); + const ws = new Workspace({ sources: { pseudo: src }, lang: 'pseudo' }); + assert.ok(!ws.allDiagnostics().some((d) => d.id === 'engine-stop')); + assert.equal(ws.proof().report.counts.total, 2); +}); + +test('a run of pseudo takes maxVisits, and pseudoToMatho\'s check that the equations give the same values uses it', async () => { + const { runPseudo } = await import('../dist/services/pseudo.js'); + const s = studyOf(quantified(3)); + const t = runPseudo(s, { call: 'F(3)', maxVisits: 1000 }); + assert.equal(t.error.kind, 'limit'); + assert.match(t.error.message, /positions visited by FORALL/); + assert.equal(runPseudo(s, { call: 'F(3)' }).error, null); +}); diff --git a/tests/prover-soundness.test.mjs b/tests/prover-soundness.test.mjs new file mode 100644 index 0000000..ebdf998 --- /dev/null +++ b/tests/prover-soundness.test.mjs @@ -0,0 +1,187 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The prover's "proved" must be a proof. Two gates hold it to that: +// +// 1. The regression programs in library/pseudo/prover/ each name, in a `// MUST-NOT-PROVE: <kind>@<line>` +// header, an obligation that a run of the program breaks. Each one was proved by an unsound +// rule once; the rule is fixed, and the obligation may never be proved again. A program whose +// fix was to the language instead (a run that now stops where it used to go on outside what a +// proof covers) names where it stops with `// RUN-STOPS: <runtime kind>@<line>`. +// 2. Across every exercise, every corpus program, every lesson solution and those regression +// programs, no obligation the prover marks proved is broken by the fuzzer (matched by kind, +// line and column) or by running the program as written (matched by kind and line). +// +// A regression program that needs input says so one line at a time, `// INPUT: <line>` (as library/pseudo/programs/ +// does), and one that is a call of a subroutine says `// CALL: <call>`. A program may name both an obligation that +// must not be proved and where its run stops, when the run reports the failure on another line than the obligation's. +// +// A failure here is a soundness bug in src/engines/pseudo/vcgen.js or solver.js: fix the rule, +// never the test. Add the program that found it to library/pseudo/prover/. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { studyOf, runPseudo } from '../dist/services/pseudo.js'; +import { verify } from '../dist/engines/pseudo/vcgen.js'; +import { checkProperties } from '../dist/engines/pseudo/testing.js'; +import { PROBLEMS } from '../dist/common/problems.js'; +import { LESSONS, solutionOf } from '../dist/curriculum/index.js'; + +const PROVER_DIR = 'library/pseudo/prover'; +const CORPUS_DIR = 'library/pseudo/programs'; + +/** The verifier's obligation kinds against the runtime error kinds a run or the fuzzer reports. */ +const RUNTIME_KIND = { + index: 'index', slice: 'index', key: 'key', divide: 'divide', argument: 'value', + require: 'require', ensure: 'ensure', assert: 'assert', + 'invariant-init': 'invariant', 'invariant-keep': 'invariant', + 'decreases-bound': 'decreases', 'decreases-step': 'decreases', +}; + +/** What a program's header says a run of it needs: its INPUT lines, and the call to make instead of the main program. */ +function runOptions(src) { + const input = [...src.matchAll(/^\/\/ INPUT: (.*)$/gm)].map((m) => m[1]); + const call = /^\/\/ CALL: (.*)$/m.exec(src)?.[1]; + return call ? { input, call } : { input }; +} + +/** The half of a contract an obligation is about, as a runtime error names it. */ +const PHASE = { 'invariant-init': 'init', 'invariant-keep': 'keep', 'decreases-bound': 'bound', 'decreases-step': 'step' }; + +function programs() { + const out = []; + for (const f of fs.readdirSync(PROVER_DIR).sort()) out.push({ name: `prover/${f}`, src: fs.readFileSync(path.join(PROVER_DIR, f), 'utf8') }); + for (const f of fs.readdirSync(CORPUS_DIR).sort()) out.push({ name: `corpus/${f}`, src: fs.readFileSync(path.join(CORPUS_DIR, f), 'utf8') }); + for (const p of PROBLEMS) if (p.sources.pseudo) out.push({ name: `exercise ${p.id}`, src: p.sources.pseudo }); + for (const L of LESSONS) { + if (L.solution?.pseudo) out.push({ name: `lesson ${L.id} solution`, src: solutionOf(L.solution) }); + if (L.example?.pseudo) out.push({ name: `lesson ${L.id} example`, src: L.example.pseudo }); + } + return out; +} + +/** Every proved obligation of a program that a run or the fuzzer breaks, as messages. */ +function unsoundIn(src) { + const st = studyOf(src); + if (!st.ok) return { parsed: false, bad: [], proved: [] }; + const program = st.program.program; + const r = verify(program, { budgetMs: 3000 }); + const proved = r.units.filter((u) => !u.partial).flatMap((u) => u.obligations.filter((o) => o.status === 'proved')); + const failures = []; + for (const sub of program.subs.values()) { + try { failures.push(...checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures); } catch { /* parameters it cannot generate */ } + } + let run = null; + try { run = runPseudo(st, runOptions(src)); } catch { /* a program that needs input it cannot have */ } + if (run?.error) failures.push({ kind: run.error.kind, line: run.error.line, col: undefined, phase: run.raw?.error?.phase, call: '(the program as written)', message: run.error.message }); + const bad = []; + for (const o of proved) { + const kind = RUNTIME_KIND[o.kind] || o.kind; + // a failure that says which half of a contract it broke (phase) counts against that obligation only + const hit = failures.find((f) => f.kind === kind && f.line === o.line && (f.col === undefined || f.col === o.col) && (!f.phase || !PHASE[o.kind] || f.phase === PHASE[o.kind])); + if (hit) bad.push(`${o.kind}@${o.line}:${o.col} "${o.label}" is marked proved, and ${hit.call} breaks it: ${hit.message}`); + } + return { parsed: true, bad, proved, obligations: r.units.flatMap((u) => u.obligations) }; +} + +test('regression programs: an obligation an unsound rule once proved is never proved again', () => { + const files = fs.readdirSync(PROVER_DIR).filter((f) => f.endsWith('.pseudo')); + assert.ok(files.length >= 5, 'the regression programs are missing'); + for (const f of files) { + const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8'); + const marks = [...src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => ({ kind: m[1], line: Number(m[2]) })); + const stops = [...src.matchAll(/^\/\/ RUN-STOPS: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => `${m[1]}@${m[2]}`); + assert.ok(marks.length || stops.length, `${f} has no MUST-NOT-PROVE or RUN-STOPS line`); + if (stops.length) { + const run = runPseudo(studyOf(src), runOptions(src)); + assert.ok(run.error && stops.includes(`${run.error.kind}@${run.error.line}`), `${f}: the run should stop at ${stops.join(' or ')}, and ${run.error ? `stopped at ${run.error.kind}@${run.error.line}` : 'it finished'}`); + } + const st = studyOf(src); + assert.ok(st.ok, `${f} does not parse`); + const r = verify(st.program.program, { budgetMs: 3000 }); + const all = r.units.flatMap((u) => u.obligations); + for (const m of marks) { + const o = all.filter((x) => x.kind === m.kind && x.line === m.line); + assert.ok(o.length, `${f}: there is no ${m.kind} obligation on line ${m.line}; the checker must ask the question before it can answer it`); + for (const x of o) assert.notEqual(x.status, 'proved', `${f}: ${m.kind}@${m.line} "${x.label}" is proved, and a run of this program breaks it`); + } + } +}); + +test('regression programs stay meaningful: each one, run or fuzzed, really breaks what it names', () => { + for (const f of fs.readdirSync(PROVER_DIR).filter((x) => x.endsWith('.pseudo'))) { + const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8'); + const st = studyOf(src); + const program = st.program.program; + const kinds = []; + try { const run = runPseudo(st, runOptions(src)); if (run.error) kinds.push(`${run.error.kind}@${run.error.line}`); } catch { /* none */ } + // a program that names where its run stops, and stops there, has shown its failure: the obligation it names sits on another line + const stops = [...src.matchAll(/^\/\/ RUN-STOPS: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => `${m[1]}@${m[2]}`); + if (stops.some((x) => kinds.includes(x))) continue; + for (const sub of program.subs.values()) { + try { for (const fl of checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures) kinds.push(`${fl.kind}@${fl.line}`); } catch { /* none */ } + } + for (const m of src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)) { + const want = `${RUNTIME_KIND[m[1]] || m[1]}@${m[2]}`; + assert.ok(kinds.includes(want), `${f}: nothing breaks ${m[1]}@${m[2]} (saw ${kinds.join(', ') || 'no failure'}), so the program no longer tests the rule`); + } + } +}); + +test('nothing the prover proves is broken by a run or by generated inputs', () => { + const problems = []; + let proved = 0; + let checked = 0; + for (const p of programs()) { + const r = unsoundIn(p.src); + if (!r.parsed) continue; + checked++; + proved += r.proved.length; + for (const b of r.bad) problems.push(`${p.name}: ${b}`); + } + assert.ok(checked >= 100, `only ${checked} programs were checked`); + assert.ok(proved >= 150, `only ${proved} proved obligations were checked`); + assert.deepEqual(problems, []); +}); + +test('canary: after a program that runs to its end, ASSERT 1 == 2 is never proved', () => { + // A false claim proved at a point every run reaches means the checker holds a contradiction + // there: some rule assumed a fact a run can break. This finds such a rule even when no + // program happens to assert something false after it (library/pseudo/prover/foreach-empty.pseudo). + const problems = []; + let canaries = 0; + for (const p of programs()) { + const st = studyOf(p.src); + if (!st.ok || /^\s*INPUT\b/m.test(p.src)) continue; + let run; + try { run = runPseudo(st, {}); } catch { continue; } + if (run.error || run.truncated) continue; + const lines = p.src.replace(/\s+$/, '').split('\n'); + const line = lines.length + 1; + const withCanary = studyOf(`${lines.join('\n')}\nASSERT 1 == 2\n`); + if (!withCanary.ok) continue; + const r = verify(withCanary.program.program, { budgetMs: 3000 }); + const main = r.units.find((u) => u.kind === 'main'); + const o = main?.obligations.find((x) => x.kind === 'assert' && x.line === line); + if (!o) continue; + canaries++; + if (o.status === 'proved') problems.push(`${p.name}: ASSERT 1 == 2 after the last line is proved`); + } + assert.ok(canaries >= 60, `only ${canaries} programs carried a canary`); + assert.deepEqual(problems, []); +}); + +test('a FUNCTION that RETURNS an ARRAY of written lengths hands back those lengths: known at the call, and checked in the run', () => { + // known at the call: the store into G has nothing left to prove (15-game-of-life showed it open) + const life = fs.readFileSync(path.join(CORPUS_DIR, '15-game-of-life.pseudo'), 'utf8'); + const r = verify(studyOf(life).program.program, { budgetMs: 3000 }); + assert.deepEqual(r.units.flatMap((u) => u.obligations).filter((o) => o.status !== 'proved').map((o) => `${o.kind}@${o.line} ${o.label}`), []); + // and the fact is one a run keeps: a function that would hand back another length stops at its RETURN + const wrong = 'FUNCTION Two(N : INTEGER) RETURNS ARRAY[2] OF INTEGER\n DECLARE L : LIST OF INTEGER\n FOR I <- 0 TO N\n APPEND(L, I)\n NEXT I\n RETURN L\nENDFUNCTION\nDECLARE A : ARRAY[2] OF INTEGER\nA <- Two(3)\nASSERT LENGTH(A) == 2\n'; + const st = studyOf(wrong); + assert.ok(st.ok, st.diagnostics.map((d) => d.message).join('; ')); + const run = runPseudo(st, {}); + assert.equal(`${run.error?.kind}@${run.error?.line}`, 'value@6', run.error?.message); + const obs = verify(st.program.program, { budgetMs: 3000 }).units.flatMap((u) => u.obligations); + assert.ok(obs.some((o) => o.line === 6 && o.status !== 'proved'), 'the RETURN of a list of unknown length is not proved to have 2 positions'); +}); diff --git a/tests/pseudo-declare-init.test.mjs b/tests/pseudo-declare-init.test.mjs new file mode 100644 index 0000000..21ed891 --- /dev/null +++ b/tests/pseudo-declare-init.test.mjs @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// DECLARE <name> : <type> <- <value>: one line that declares a name and gives it its starting value +// (src/services/pseudo-study.ts). The two-line form still reads exactly as before. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { studyOf, runPseudo } from '../dist/services/pseudo.js'; +import { splitDeclareInits } from '../dist/services/pseudo-study.js'; +import { pseudoToMojo } from '../dist/sync/pseudo-to-mojo.js'; +import { checkMojo } from '../dist/mojo/check.js'; + +const fn = (decls) => `FUNCTION COUNT_BASE(B : CHAR, T : STRING) RETURNS INTEGER\n${decls}\n FOR I <- 0 TO N\n IF T[I] == B THEN\n Total <- Total + 1\n ENDIF\n NEXT I\n RETURN Total\nENDFUNCTION\n`; +const ONE = fn(' DECLARE N : INTEGER <- LENGTH(T)\n DECLARE Total : INTEGER <- 0'); +const TWO = fn(' DECLARE N : INTEGER\n N <- LENGTH(T)\n DECLARE Total : INTEGER\n Total <- 0'); +const CALL = `COUNT_BASE('A', "GATTACA")`; + +test('the value is blanked out of the line with its columns kept, and found apart from strings and comments', () => { + const { source, inits } = splitDeclareInits('DECLARE S : STRING <- "x <- y" // a <- b\nDECLARE N : INTEGER'); + assert.equal(source.split('\n')[0].length, 'DECLARE S : STRING <- "x <- y" // a <- b'.length); + assert.equal(source.split('\n')[0], `DECLARE S : STRING ${' '.repeat(12)}// a <- b`); + assert.deepEqual(inits.map((i) => [i.line, i.names, i.value, i.valueCol]), [[1, ['S'], '"x <- y"', 23]]); +}); + +test('one line and two lines read, run, and answer the same, with no diagnostics', () => { + for (const src of [ONE, TWO]) { + const st = studyOf(src); + assert.ok(st.ok); + assert.deepEqual(st.diagnostics.filter((d) => d.severity !== 'note'), []); + assert.deepEqual(runPseudo(st, { call: CALL }).output, ['COUNT_BASE returned 3']); + } +}); + +test('the run shows the declaration and its starting value as steps of the same line', () => { + const tr = runPseudo(studyOf(ONE), { call: CALL }); + assert.deepEqual(tr.steps.filter((s) => s.line === 2).map((s) => s.text), ['N is INTEGER, with no value yet.', 'N becomes 7.']); +}); + +test('the Mojo is var x: T = value for the one-line form, and stays two lines for the two-line form', () => { + const one = pseudoToMojo(ONE, { call: CALL }).source; + assert.match(one, /^ {4}var n: Int = t\.count_codepoints\(\)$/m); + assert.match(one, /^ {4}var total: Int = 0$/m); + assert.doesNotMatch(one, /^ {4}n = /m); + assert.deepEqual(checkMojo(one).diagnostics.filter((d) => d.severity === 'error'), []); + const two = pseudoToMojo(TWO, { call: CALL }).source; + assert.match(two, /^ {4}var n: Int\n {4}n = t\.count_codepoints\(\)$/m); +}); + +test('a mistake in the form says what the form is', () => { + const msg = (src) => studyOf(src + '\n').diagnostics.filter((d) => d.severity === 'error').map((d) => d.message).join(' | '); + assert.match(msg('DECLARE A, B : INTEGER <- 0'), /names one variable/); + assert.match(msg('DECLARE A : INTEGER <-'), /nothing on the right of its arrow/); + assert.match(msg('DECLARE A <- 3'), /needs its type before a starting value/); + assert.match(msg('DECLARE A : INTEGER <- "x"'), /INTEGER values, and "x" is a STRING/); +}); + +test('a lone carriage return ends a line for the one-line form too (it used to throw)', () => { + for (const nl of ['\r', '\r\n', '\n']) { + const st = studyOf(`OUTPUT 1${nl}DECLARE Y : INTEGER <- 2${nl}OUTPUT Y${nl}`); + assert.ok(st.ok, JSON.stringify(nl)); + assert.deepEqual(runPseudo(st, {}).output, ['1', '2']); + } +}); diff --git a/tests/pseudo-explain.test.mjs b/tests/pseudo-explain.test.mjs new file mode 100644 index 0000000..9616c1a --- /dev/null +++ b/tests/pseudo-explain.test.mjs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The words a step carries ("IF X > 0 (X is 3) is TRUE", "ASSERT on line 4 is FALSE: ..., where the left side is 9") +// are made by looking at the condition a second time. Looking must not be running: a routine called in a condition +// runs once, a random number is drawn once, and nothing is shown twice. The statement fuzzer found the first case +// (work/release/progress.md): IF LENGTH(F(1)) > 0 ran F twice. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { studyOf, runPseudo } from '../dist/services/pseudo.js'; + +const run = (src, opts = {}) => { + const st = studyOf(src); + assert.ok(st.ok, st.diagnostics.filter((d) => d.severity === 'error').map((d) => `L${d.line} ${d.id}: ${d.message}`).join(' | ')); + return runPseudo(st, opts); +}; +const F = `FUNCTION F(N : INTEGER) RETURNS LIST OF INTEGER + OUTPUT "called ", N + RETURN [N, N] +ENDFUNCTION +`; +const texts = (r) => r.steps.map((s) => s.text).join('\n'); + +test('a routine called in an IF, WHILE or UNTIL condition runs once per test', () => { + assert.deepEqual(run(`${F}IF LENGTH(F(1)) > 0 THEN\n OUTPUT "yes"\nENDIF`).output, ['called 1', 'yes']); + assert.deepEqual(run(`${F}DECLARE I : INTEGER\nI <- 0\nWHILE F(I)[1] < 2\n I <- I + 1\nENDWHILE`).output, ['called 0', 'called 1', 'called 2']); + assert.deepEqual(run(`${F}DECLARE I : INTEGER\nI <- 2\nREPEAT\n I <- I - 1\nUNTIL F(I)[0] <= 0`).output, ['called 1', 'called 0']); +}); + +test('the parts of a condition that call nothing are still described with their values', () => { + const r = run(`${F}DECLARE XS : LIST OF INTEGER\nXS <- [5, 6]\nIF LENGTH(F(1)) > 0 AND LENGTH(XS) == 2 AND XS[1] == 6 THEN\n OUTPUT "yes"\nENDIF`); + assert.deepEqual(r.output, ['called 1', 'yes']); + assert.match(texts(r), /LENGTH\(XS\) is 2/); + assert.match(texts(r), /XS\[1\] is 6/); + assert.doesNotMatch(texts(r), /LENGTH\(F\(1\)\) is/); +}); + +test('a random number in a condition is drawn once, so the run is the same with and without the description', () => { + const src = `DECLARE XS : LIST OF INTEGER\nXS <- [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\nDECLARE N : INTEGER\nN <- 0\nWHILE XS[RANDINT(0, 10)] < 8\n N <- N + 1\nENDWHILE\nOUTPUT N, " ", RANDINT(0, 1000000)`; + const plain = `DECLARE XS : LIST OF INTEGER\nXS <- [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\nDECLARE N : INTEGER\nN <- 0\nDECLARE T : INTEGER\nT <- XS[RANDINT(0, 10)]\nWHILE T < 8\n N <- N + 1\n T <- XS[RANDINT(0, 10)]\nENDWHILE\nOUTPUT N, " ", RANDINT(0, 1000000)`; + for (const seed of [1, 2, 3, 4, 5]) assert.deepEqual(run(src, { seed }).output, run(plain, { seed }).output, `seed ${seed}`); +}); + +test('explaining a failed ASSERT shows what a routine printed once, and gives the values that failed', () => { + const r = run(`${F}ASSERT F(9)[1] == 8`); + assert.deepEqual(r.output, ['called 9']); + assert.match(texts(r), /ASSERT on line 5 is FALSE: F\(9\)\[1\] == 8, where the left side is 9 and the right side is 8/); +}); + +test('a failed ASSERT on a random number is explained without values, since a second draw would give other ones', () => { + const r = run('ASSERT RANDINT(0, 1000000) == -1'); + assert.match(texts(r), /ASSERT on line 1 is FALSE: RANDINT\(0, 1000000\) == -1\./); + assert.doesNotMatch(texts(r), /where the left side/); +}); diff --git a/tests/pseudo-robustness.test.mjs b/tests/pseudo-robustness.test.mjs new file mode 100644 index 0000000..a058095 --- /dev/null +++ b/tests/pseudo-robustness.test.mjs @@ -0,0 +1,319 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Pseudo answers any text with diagnostics and any run with an output or a stop: never with an exception of the +// engine underneath, a run without end, or an allocation the size of the program's mistake. Each test is a case +// that once threw, hung or ran out of memory (work/revive/interp-audit-report.md has the audit that found them). +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { studyOf, runPseudo, provePseudo } from '../dist/services/pseudo.js'; +import { pseudoToMojo } from '../dist/sync/pseudo-to-mojo.js'; +import { pseudoToLingo } from '../dist/sync/pseudo-to-lingo.js'; +import { pseudoToMatho } from '../dist/sync/pseudo-to-matho.js'; +import { SyncHub } from '../dist/sync/hub.js'; +import { checkProperties } from '../dist/engines/pseudo/testing.js'; +import { run as runProgram } from '../dist/engines/pseudo/interp.js'; + +const errors = (src) => studyOf(src).diagnostics.filter((d) => d.severity === 'error').map((d) => d.id); +const run = (src, opts = {}) => { + const st = studyOf(src); + assert.ok(st.ok, st.diagnostics.filter((d) => d.severity === 'error').map((d) => `L${d.line} ${d.id}: ${d.message}`).join(' | ')); + return runPseudo(st, opts); +}; +/** Every reader of pseudo text, on text that does not check: each answers, none throws. */ +const everyReader = (src) => { + studyOf(src); pseudoToMojo(src); pseudoToLingo(src); pseudoToMatho(src); + new SyncHub({ pseudo: src }).fill(); + new SyncHub({ pseudo: '' }).edit('pseudo', src); +}; + +test('an arrow that is not the assignment arrow is a character with no meaning, not a crash in the lexer', () => { + for (const c of ['→', '↑', '⇒', '↔']) { + assert.ok(errors(`X ${c} Y`).includes('bad-character'), c); + everyReader(`X ${c} Y`); + } + assert.deepEqual(run('DECLARE X : INTEGER\nX ← 3\nOUTPUT X').output, ['3']); +}); + +test('FOR EACH X IN with nothing after IN is an error where it is written', () => { + const src = 'FOR EACH B IN\n OUTPUT 1\nNEXT B\n'; + assert.ok(errors(src).includes('expected')); + everyReader(src); +}); + +test('nesting past what the readers can walk is one too-deep error', () => { + const deep = [ + `OUTPUT ${'('.repeat(5000)}1${')'.repeat(5000)}`, + `OUTPUT ${'('.repeat(5000)}1`, + `OUTPUT ${'['.repeat(5000)}1${']'.repeat(5000)}`, + `OUTPUT ${'-'.repeat(5000)}1`, + `OUTPUT ${'NOT '.repeat(5000)}TRUE`, + `OUTPUT 1${' ^ 1'.repeat(5000)}`, + `OUTPUT 1${' + 1'.repeat(5000)}`, + `OUTPUT TRUE${' AND TRUE'.repeat(5000)}`, + `OUTPUT "a"${' & "a"'.repeat(5000)}`, + `OUTPUT ${'ABS('.repeat(3000)}1${')'.repeat(3000)}`, + `DECLARE L : LIST OF INTEGER\nOUTPUT L${'[0]'.repeat(5000)}`, + `DECLARE X : ${'LIST OF '.repeat(5000)}INTEGER`, + `${'IF TRUE THEN\n'.repeat(3000)}OUTPUT 1\n${'ENDIF\n'.repeat(3000)}`, + `${'WHILE FALSE\n'.repeat(3000)}OUTPUT 1\n${'ENDWHILE\n'.repeat(3000)}`, + `${'REPEAT\n'.repeat(3000)}OUTPUT 1\n${'UNTIL TRUE\n'.repeat(3000)}`, + 'IF\n'.repeat(5000), + ]; + for (const src of deep) { + const ids = errors(src); + assert.ok(ids.includes('too-deep'), `${src.slice(0, 30)}: ${[...new Set(ids)].join(', ')}`); + assert.ok(ids.filter((id) => id === 'too-deep').length <= 2, `${src.slice(0, 30)}: one message, not one per level`); + everyReader(src); + } +}); + +test('brackets that each hold a long chain are one too-deep error, though each limit alone is kept', () => { + // 200 levels and 400 values are counted apart, and a chain leans left one level per value: these stay inside both + // and built trees 800 to 2,000 levels tall, which the checker ended on with the engine's own stack error + const chain = ' + 1'.repeat(398); + const levels = (n) => { let e = `1${chain}`; for (let i = 1; i < n; i++) e = `(${e})${chain}`; return e; }; + const deep = [ + `OUTPUT ${levels(2)}`, + `OUTPUT ${levels(3)}`, + `DECLARE X : INTEGER\nX <- ${levels(5)}\nOUTPUT X`, + `OUTPUT ${levels(40)}`, + `DECLARE L : LIST OF INTEGER\nOUTPUT L${'[0]'.repeat(398)}${chain}`, + `DECLARE L : LIST OF INTEGER\nL[${levels(1)}]${'[0]'.repeat(300)} <- 1`, + `OUTPUT ABS(${levels(1)})${chain}`, + `DECLARE X : INTEGER <- ${levels(3)}\nOUTPUT X`, + `OUTPUT TRUE${' AND TRUE'.repeat(300)} OR (${levels(1)}) > 0`.replace('OUTPUT TRUE', `OUTPUT (${levels(1)} > 0)`), + ]; + for (const src of deep) { + const ids = errors(src); + assert.ok(ids.includes('too-deep'), `${src.slice(0, 40)}: ${[...new Set(ids)].join(', ')}`); + assert.ok(ids.filter((id) => id === 'too-deep').length <= 2, `${src.slice(0, 40)}: one message, not one per level`); + everyReader(src); + } + // a tree as tall as the limit allows goes through every reader, a run and the prover + const tallest = `OUTPUT (1${' + 1'.repeat(298)})${' + 1'.repeat(101)}`; + assert.deepEqual(run(tallest).output, ['400']); + assert.ok(errors(`OUTPUT (1${' + 1'.repeat(298)})${' + 1'.repeat(102)}`).includes('too-deep'), 'one level more'); + everyReader(tallest); + provePseudo(studyOf(tallest)); +}); + +test('what is nested as deeply as the limit allows still reads, runs and translates', () => { + const parens = `OUTPUT ${'('.repeat(150)}1${')'.repeat(150)}`; + assert.deepEqual(run(parens).output, ['1']); + const sum = `OUTPUT 1${' + 1'.repeat(399)}`; + assert.deepEqual(run(sum).output, ['400']); + const ifs = `${'IF TRUE THEN\n'.repeat(150)}OUTPUT 1\n${'ENDIF\n'.repeat(150)}`; + assert.deepEqual(run(ifs).output, ['1']); + for (const src of [parens, sum, ifs]) { everyReader(src); provePseudo(studyOf(src)); } +}); + +test('a size no run could hold stops the run with a sentence instead of an allocation', () => { + const stops = (src, kind, re) => { const r = run(src); assert.equal(r.error?.kind, kind, src); assert.match(r.error.message, re); }; + stops('DECLARE L : LIST OF INTEGER <- FILL(1000000000, 0)', 'value', /more values than this lab gives one collection/); + stops('DECLARE A : ARRAY[5000000000] OF INTEGER', 'value', /more than this lab gives one collection/); + stops('DECLARE A : ARRAY[10000, 10000] OF INTEGER', 'value', /10000 by 10000/); + stops('FOR EACH I IN [0, 1000000000)\n OUTPUT I\nNEXT I', 'value', /too large to visit/); + assert.deepEqual(run('DECLARE L : LIST OF INTEGER <- FILL(1000, 7)\nOUTPUT LENGTH(L)').output, ['1000']); +}); + +test('SHIFT_LEFT by a count no number survives is an overflow, without building the number', () => { + for (const n of ['65', '1073741824', '2147483648', '9007199254740991']) assert.equal(run(`OUTPUT SHIFT_LEFT(1, ${n})`).error?.kind, 'overflow', n); + assert.deepEqual(run('OUTPUT SHIFT_LEFT(0, 2147483648), " ", SHIFT_LEFT(1, 52), " ", SHIFT_RIGHT(-1, 2147483648)').output, ['0 4503599627370496 -1']); +}); + +test('the positions quantifiers visit are counted: one inside another cannot run without end', () => { + const r = run('OUTPUT SUM I IN [0, 2000000) : SUM J IN [0, 2000000) : 1', { maxSteps: 200 }); + assert.equal(r.error?.kind, 'limit'); + assert.match(r.error.message, /positions visited by FORALL, EXISTS, SUM/); + assert.equal(run('ASSERT FORALL I IN [0, 300) : FORALL J IN [0, 300) : I + J >= 0\nOUTPUT 1', { maxSteps: 200 }).error?.kind, 'limit'); + assert.deepEqual(run('OUTPUT SUM I IN [0, 100) : SUM J IN [0, 100) : 1').output, ['10000']); +}); + +test('CHOOSE with a huge K ends at the first step that cannot fit', () => { + const t0 = performance.now(); + assert.equal(run('OUTPUT CHOOSE(9007199254740991, 4503599627370495)').error?.kind, 'overflow'); + assert.ok(performance.now() - t0 < 2000); +}); + +test('an error of the engine underneath is a stop of the run, never an exception from it', () => { + // a text doubled until the engine refuses its length: the lab's own limit comes first + const r = run('DECLARE S : STRING <- "ab"\nFOR I <- 0 TO 40\n S <- S & S\nNEXT I\nOUTPUT LENGTH(S)'); + assert.equal(r.error?.kind, 'value'); + assert.match(r.error.message, /longer than this lab holds/); +}); + +test('a record that holds itself, and an ARRAY of ARRAYs, are errors before any run', () => { + assert.ok(errors('TYPE Node\n DECLARE Value : INTEGER\n DECLARE Link : Node\nENDTYPE\nDECLARE N : Node').includes('recursive-type')); + assert.ok(errors('TYPE A\n DECLARE B1 : B\nENDTYPE\nTYPE B\n DECLARE A1 : ARRAY[2] OF A\nENDTYPE\nDECLARE X : A').includes('recursive-type')); + assert.deepEqual(run('TYPE Node\n DECLARE Value : INTEGER\n DECLARE Kids : LIST OF Node\nENDTYPE\nDECLARE N, M : Node\nN.Value <- 1\nM.Value <- 2\nAPPEND(N.Kids, M)\nOUTPUT N.Kids[0].Value').output, ['2']); + assert.ok(errors('DECLARE A : ARRAY[2] OF ARRAY[2] OF INTEGER').includes('array-of-array')); +}); + +test('an ARRAY keeps its declared shape, and its unset positions stop the built-ins that read them', () => { + assert.ok(errors('DECLARE M : ARRAY[2, 2] OF INTEGER\nM <- [1, 2, 3, 4]').includes('type')); + // lengths the text shows are an error before the run; others stop it (the next test) + assert.ok(errors('DECLARE A : ARRAY[3] OF INTEGER\nA <- [1, 2, 3, 4, 5]').includes('array-length')); + assert.ok(errors('DECLARE P : ARRAY[2, 2] OF INTEGER\nDECLARE Q : ARRAY[3, 3] OF INTEGER\nP <- Q').includes('array-length')); + assert.equal(run('DECLARE N : INTEGER <- 5\nDECLARE A : ARRAY[3] OF INTEGER\nDECLARE B : ARRAY[N] OF INTEGER\nA <- B').error?.kind, 'value'); + assert.deepEqual(run('DECLARE A : ARRAY[3] OF INTEGER\nA <- [3, 1, 2]\nSORT(A)\nA[0] <- 9\nOUTPUT A, " ", LENGTH(A), " ", MAX(A), " ", SORTED(A)').output, ['[9, 2, 3] 3 9 [2, 3, 9]']); + const head = 'DECLARE A : ARRAY[2] OF FLOAT\nDECLARE B : ARRAY[2] OF FLOAT\nA[0] <- 1.0\nB[0] <- 2.0\nB[1] <- 2.0\n'; + for (const call of ['OUTPUT DOT(A, B)', 'OUTPUT DOT(B, A)', 'OUTPUT MAX(A)', 'OUTPUT MIN(A)', 'OUTPUT SORTED(A)', 'SORT(A)']) assert.equal(run(head + call).error?.kind, 'unset', call); +}); + +test('the inputs provePseudo tries on a slow routine are held to a time budget', () => { + // 300 calls deep, then a loop: every run is slow, and 150 of them used to be waited for + const src = 'FUNCTION F(N : INTEGER, D : INTEGER) RETURNS INTEGER\n DECLARE T : INTEGER <- 0\n IF D < 300 THEN\n RETURN F(N, D + 1)\n ENDIF\n FOR I <- 0 TO 2000\n T <- T + 1\n NEXT I\n RETURN T\nENDFUNCTION\n'; + const st = studyOf(src); + const pr = checkProperties(st.program.program, 'F', { count: 150, budgetMs: 50 }); + assert.equal(pr.cutShort, true); + assert.ok(pr.checked < 150); + assert.equal(checkProperties(studyOf('FUNCTION G(N : INTEGER) RETURNS INTEGER\n RETURN N\nENDFUNCTION\n').program.program, 'G', { count: 20, budgetMs: 5000 }).cutShort, undefined); +}); + +/** Milliseconds `fn` takes. */ +const timed = (fn) => { const t0 = performance.now(); fn(); return performance.now() - t0; }; + +test('a long program is studied in time that grows with its length, not its square (B7)', () => { + // each case took seconds when the data flow was solved one statement per pass, over sets of every definition + const declares = Array.from({ length: 1000 }, (_, i) => `DECLARE V${i} : INTEGER <- ${i}\n`).join(''); + assert.ok(timed(() => assert.ok(studyOf(declares).ok)) < 2000, '1000 declarations'); + assert.ok(timed(() => new SyncHub({ pseudo: '' }).edit('pseudo', declares)) < 4000, 'an edit of 1000 declarations'); + const nested = `${Array.from({ length: 150 }, (_, i) => `FOR I${i} <- 0 TO 1\n`).join('')}OUTPUT 1\n${Array.from({ length: 150 }, (_, i) => `NEXT I${149 - i}\n`).join('')}`; + assert.ok(timed(() => assert.ok(studyOf(nested).ok)) < 2000, '150 nested FOR loops'); + const arms = 2000; + const chain = `DECLARE X : INTEGER <- 1\nIF X == 0 THEN\n OUTPUT X\n${Array.from({ length: arms }, (_, i) => `ELSEIF X == ${i + 1} THEN\n OUTPUT X\n`).join('')}ENDIF\n`; + let st; + assert.ok(timed(() => { st = studyOf(chain); }) < 2000, `an ELSEIF chain of ${arms} arms`); + // and the answer is the whole one: X has its value in every arm, however far down the chain + assert.deepEqual(st.diagnostics.filter((d) => /unset/.test(d.id)).map((d) => d.line), []); +}); + +test('English for a text of many routines takes time that grows with their number, not its square (B8)', () => { + const funcs = (n) => Array.from({ length: n }, (_, i) => `FUNCTION F${i}(X : INTEGER) RETURNS INTEGER\n RETURN X + ${i}\nENDFUNCTION\n`).join(''); + // the English reader takes 200 routines and phrases at most (docs/limits.md): up to there the text is said in full + let r; + assert.ok(timed(() => { r = pseudoToLingo(funcs(200)); }) < 1000, '200 functions'); + assert.ok(!('unsupported' in r)); + // past it the answer says why, and still comes at once: 2,000 functions took a minute when each was held against every other + assert.ok(timed(() => { r = pseudoToLingo(funcs(600)); }) < 3000, '600 functions'); + assert.match(JSON.stringify(r.unsupported), /more than 200 routines and phrases/); + // 2,000 of them are 6,000 lines, past what a pseudocode text is read to (docs/limits.md), and say that instead + assert.ok(timed(() => { r = pseudoToLingo(funcs(2000)); }) < 3000, '2000 functions'); + assert.match(JSON.stringify(r.unsupported), /more than 2,000 lines long/); +}); + + +test('a step costs the same at any call depth: frames that did not change are not pictured again (B9)', () => { + // 390 calls deep, then a loop of 12,000 passes in the deepest call + const src = 'FUNCTION F(N : INTEGER, D : INTEGER) RETURNS INTEGER\n DECLARE T : INTEGER <- 0\n IF D < 390 THEN\n RETURN F(N, D + 1)\n ENDIF\n FOR I <- 0 TO 12000\n T <- T + 1\n NEXT I\n RETURN T\nENDFUNCTION\nOUTPUT F(1, 0)\n'; + let tr; + assert.ok(timed(() => { tr = run(src); }) < 1500, '20,000 steps 391 frames deep'); + assert.equal(tr.error?.kind, 'limit'); + const [a, b] = [tr.states[tr.count - 2], tr.states[tr.count - 1]]; + assert.equal(a.frames.length, 392); + // the 391 frames under the one that runs are the same pictures from step to step; the running one is new + for (let i = 0; i < 391; i++) assert.equal(a.frames[i], b.frames[i], `frame ${i}`); + assert.notEqual(a.frames[391], b.frames[391]); + assert.notDeepEqual(a.frames[391].vars, b.frames[391].vars); + // a BYREF parameter shares its caller's slot: a write in the callee shows in the caller's frame at that step + const byref = run('PROCEDURE Bump(BYREF X : INTEGER)\n X <- X + 1\nENDPROCEDURE\nDECLARE A : INTEGER <- 1\nCALL Bump(A)\nOUTPUT A\n'); + const wrote = byref.steps.findIndex((s) => s.line === 2); + assert.deepEqual(byref.states[wrote + 1].frames.map((f) => f.vars.A?.value ?? f.vars.X?.value), [2, 2]); + assert.deepEqual(byref.states[wrote].frames.map((f) => f.vars.A?.value ?? f.vars.X?.value), [1, 1]); +}); + +test('a recording stops copying a large collection at its budget, says so, and the run goes on (B10)', () => { + // 50 passes over an ARRAY of 100, one position written per step: each of those steps copied the whole ARRAY + const src = 'DECLARE A : ARRAY[100] OF INTEGER\nFOR P <- 0 TO 50\n FOR I <- 0 TO 100\n A[I] <- P\n NEXT I\nNEXT P\nOUTPUT A[99]\nOUTPUT A[100]\n'; + const program = studyOf(src).program.program; + const whole = runProgram(program, { maxSteps: 100000 }); + assert.equal(whole.unrecorded, null); + const cut = runProgram(program, { maxSteps: 100000, maxRecorded: 5000 }); + const k = cut.unrecorded.after; + assert.ok(k > 10 && k < whole.count, `cut after ${k} of ${whole.count} steps`); + // the copies stop within one picture of the budget + assert.ok(cut.unrecorded.values > 5000 && cut.unrecorded.values <= 5000 + 2 * 102); + // one last step says so, and the steps and states before it are the ones the whole recording has + assert.equal(cut.count, k + 1); + assert.equal(cut.states.length, k + 2); + assert.equal(cut.steps[k].kind, 'unrecorded'); + assert.match(cut.steps[k].text, /not recorded/); + // (frame ids count up across runs, so they are compared from each run's first) + const same = (tr, x) => JSON.stringify(x, (key, v) => ((key === 'id' || key === 'frame') && typeof v === 'number' ? v - tr.states[0].frames[0].id : key === 'slot' ? undefined : typeof v === 'symbol' ? String(v) : v)); + assert.ok(same(cut, cut.steps.slice(0, k)) === same(whole, whole.steps.slice(0, k)), 'the steps before the cut'); + assert.ok(same(cut, cut.states.slice(0, k + 1)) === same(whole, whole.states.slice(0, k + 1)), 'the states before the cut'); + // the run itself went on to the end: its output, its stop, and the lines it ran are the whole run's + assert.deepEqual(cut.output, ['49']); + assert.equal(cut.error?.kind, 'index'); + assert.equal(cut.error.message, whole.error.message); + assert.deepEqual([...cut.hits], [...whole.hits]); + assert.equal(cut.work.steps, whole.work.steps); +}); + +test('an ARRAY parameter and an ARRAY RETURN keep their declared lengths, as a store does (C12)', () => { + const first = 'FUNCTION FIRST(A : ARRAY[3] OF INTEGER) RETURNS INTEGER\n RETURN A[0] + LENGTH(A)\nENDFUNCTION\n'; + // lengths the text shows: an error where it is written + assert.ok(errors(`${first}OUTPUT FIRST([7, 8])`).includes('array-length')); + assert.ok(errors('PROCEDURE P(BYREF A : ARRAY[3] OF INTEGER)\n A[0] <- 1\nENDPROCEDURE\nDECLARE B : ARRAY[5] OF INTEGER\nCALL P(B)').includes('array-length')); + assert.ok(errors('FUNCTION MAKE() RETURNS ARRAY[3] OF INTEGER\n DECLARE B : ARRAY[5] OF INTEGER\n RETURN B\nENDFUNCTION\nOUTPUT LENGTH(MAKE())').includes('array-length')); + assert.ok(errors('DECLARE L : LIST OF ARRAY[2] OF INTEGER\nAPPEND(L, [1, 2, 3])').includes('array-length')); + // lengths only the run knows: the run stops at the call or the RETURN + const stop = run(`${first}DECLARE L : LIST OF INTEGER\nAPPEND(L, 7)\nAPPEND(L, 8)\nOUTPUT FIRST(L)`); + assert.equal(stop.error?.kind, 'value'); + assert.equal(stop.error.line, 7); + assert.match(stop.error.message, /Parameter A of FIRST is an ARRAY of 3 positions, and the LIST given to it has 2/); + const back = run('FUNCTION MAKE(N : INTEGER) RETURNS ARRAY[3] OF INTEGER\n DECLARE B : ARRAY[N] OF INTEGER\n RETURN B\nENDFUNCTION\nOUTPUT LENGTH(MAKE(5))'); + assert.equal(back.error?.kind, 'value'); + assert.match(back.error.message, /the RESULT of MAKE is an ARRAY of 3 positions/); + // the right lengths run as before, a LIST becoming the ARRAY's positions + assert.deepEqual(run(`${first}OUTPUT FIRST([7, 8, 9])`).output, ['10']); + assert.deepEqual(run('FUNCTION MAKE(N : INTEGER) RETURNS ARRAY[3] OF INTEGER\n DECLARE B : ARRAY[N] OF INTEGER\n B[0] <- 1\n RETURN B\nENDFUNCTION\nOUTPUT LENGTH(MAKE(3))').output, ['3']); + // and the prover knows a parameter's lengths, and asks for them where the text does not show them + const pr = provePseudo(studyOf(`${first}DECLARE L : LIST OF INTEGER\nAPPEND(L, 7)\nOUTPUT FIRST(L)`)); + const obs = pr.units.flatMap((u) => u.obligations); + assert.equal(obs.find((o) => o.kind === 'index' && o.line === 2)?.status, 'proved'); + assert.notEqual(obs.find((o) => o.kind === 'argument' && o.line === 6)?.status, 'proved'); + assert.equal(obs.filter((o) => o.kind === 'argument').length, 1); +}); + +test('a SET holds single values: a LIST, an ARRAY or a record as a member is an error (C5, prover O)', () => { + // a run found such a member by identity, and it could change after it was added + for (const src of [ + 'DECLARE S : SET OF LIST OF INTEGER\nADD(S, [1, 2])\nASSERT [1, 2] IN S', + 'TYPE P\n DECLARE X : INTEGER\nENDTYPE\nDECLARE S : SET OF P', + 'DECLARE S : SET OF ARRAY[2] OF INTEGER', + 'DECLARE L : LIST OF INTEGER\nOUTPUT {L}', + 'DECLARE L : LIST OF LIST OF INTEGER\nOUTPUT {X IN L : LENGTH(X) > 0}', + 'TYPE Rows = SET OF LIST OF CHAR\nOUTPUT 1', + 'DECLARE S : SET OF SET OF INTEGER', + ]) assert.ok(errors(src).includes('bad-member'), src); + assert.deepEqual(run("DECLARE S : SET OF INT8\nDECLARE T : SET OF STRING\nADD(S, 3)\nADD(T, \"AC\")\nOUTPUT 3 IN S, \" \", \"AC\" IN T").output, ['TRUE TRUE']); +}); + +test('NaN or an infinity passed straight to run() for a FLOAT parameter is refused without USE IEEE (prover Q)', () => { + const body = 'PROCEDURE P(X : FLOAT, L : LIST OF FLOAT)\n ASSERT X == X\n OUTPUT X\nENDPROCEDURE\n'; + const program = studyOf(body).program.program; + for (const [x, l] of [[NaN, []], [Infinity, []], [-Infinity, []], [1.5, [NaN]], [1.5, [2, Infinity]]]) { + const r = runProgram(program, { call: { name: 'P', args: [x, l] } }); + assert.equal(r.error?.kind, 'value', `${x} ${l}`); + assert.match(r.error.message, /not a finite number/); + } + assert.deepEqual(runProgram(program, { call: { name: 'P', args: [1.5, [2]] } }).output, ['1.5']); + // with USE IEEE they are values like any other + const ieee = studyOf(`USE IEEE\n${body.replace(' ASSERT X == X\n', '')}`).program.program; + assert.deepEqual(runProgram(ieee, { call: { name: 'P', args: [Infinity, [NaN]] } }).output, ['infinity']); +}); + +test('a routine the program cannot call by name is still checked, with its own record', () => { + // named after a built-in, with a contract below its first line: the checker read `kind` of a record it never made, and threw + const builtin = 'FUNCTION LENGTH(X : INTEGER) RETURNS INTEGER\n DECLARE Y : INTEGER\n REQUIRE X > 0\n RETURN X\nENDFUNCTION'; + assert.deepEqual(errors(builtin), ['builtin-name', 'contract-place']); + everyReader(builtin); + // its parameters are its own: X is declared, Nope is not (before, both were reported) + assert.deepEqual(errors('PROCEDURE ABS(BYREF X : INTEGER)\n X <- Nope\nENDPROCEDURE'), ['builtin-name', 'undeclared']); + // a second routine of a name was checked with the first one's record: the first body twice, the second never + const twice = 'FUNCTION F(X : INTEGER) RETURNS INTEGER\n RETURN X\nENDFUNCTION\nFUNCTION F(A : STRING) RETURNS STRING\n RETURN A & Undeclared\nENDFUNCTION'; + const found = studyOf(twice).diagnostics.filter((d) => d.severity === 'error').map((d) => `${d.line} ${d.id}`); + assert.deepEqual(found, ['4 duplicate', '5 undeclared']); + everyReader(twice); +}); diff --git a/tests/reader-limits.test.mjs b/tests/reader-limits.test.mjs new file mode 100644 index 0000000..b89ce78 --- /dev/null +++ b/tests/reader-limits.test.mjs @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The readers a share link reaches besides Mojo (lingo, matho, speco, x86-64) answer any text with diagnostics, and +// the hub and the workspace never let a reader's exception out: each limit proven just past it with small inputs +// (docs/limits.md lists every limit; tests/mojo-robustness.test.mjs has Mojo's). +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readLingo } from '../dist/services/lingo.js'; +import { parseMatho } from '../dist/services/matho.js'; +import { parseSpec, toAscii, toMath } from '../dist/services/speco.js'; +import { parseAsm, runAsmProgram } from '../dist/services/asm.js'; +import { SERVICES } from '../dist/services/index.js'; +import { SyncHub, LINKED, FAULT_ID } from '../dist/sync/hub.js'; +import { Workspace } from '../dist/common/workspace.js'; + +const ids = (r) => r.diagnostics.filter((d) => d.severity === 'error').map((d) => d.id); +/** The app's path for a pane: a workspace opening it, and an edit through the hub. */ +const everyPath = (lang, src) => { + const ws = new Workspace({ sources: { [lang]: src }, lang }); + ws.allDiagnostics(); ws.trace(); ws.proof(); ws.lineMaps(); + new SyncHub({ pseudo: '' }).edit(lang, src); + new SyncHub({ [lang]: src }).fill(); +}; + +test('lingo: a sentence nested past 200 levels is one too-deep error, and one past 1,000 words and symbols is too-long', () => { + const deep = [ + `To f X:\n The answer is ${'('.repeat(300)}X${')'.repeat(300)}.`, + `To f X:\n If ${'not '.repeat(300)}X is 1, the answer is 1.\n The answer is 0.`, + `To f X:\n The answer is ${'the length of '.repeat(300)}X.`, + `To f X:\n The answer is ${'minus '.repeat(300)}X.`, + `To f X:\n Start with C at 0.\n${Array.from({ length: 250 }, (_, k) => ' ' + ' '.repeat(k) + 'If X is 1:').join('\n')}\n ${' '.repeat(250)}Add 1 to C.\n The answer is C.`, + ]; + for (const src of deep) { + const r = readLingo(src); + assert.equal(r.ok, false); + assert.deepEqual(ids(r).filter((x) => x === 'too-deep'), ['too-deep'], src.slice(0, 60)); + assert.match(r.diagnostics.find((d) => d.id === 'too-deep').message, /nested more than 200 levels deep/); + everyPath('lingo', src); + } + const long = readLingo(`To f X:\n The answer is X${' plus 1'.repeat(600)}.`); + assert.deepEqual(ids(long).filter((x) => x === 'too-long'), ['too-long']); + assert.match(long.diagnostics.find((d) => d.id === 'too-long').message, /more than 1,000 words and symbols/); + // a line continued under a step joins it one token at a time (push(...tokens) passed 150,000 arguments) + const continued = `To f X:\n The answer is X\n ${'+ 1 '.repeat(75000)}`; + assert.ok(ids(readLingo(continued)).includes('too-long')); + for (const src of [`To f X:\n The answer is ${'('.repeat(30000)}X${')'.repeat(30000)}.`, `To f X:\n The answer is X${' plus 1'.repeat(30000)}.`]) everyPath('lingo', src); + // every value is tried against every routine's first line: a program is read with its first 200 routines and phrases + const many = readLingo(Array.from({ length: 300 }, (_, k) => `To get x${k} of X:\n The answer is X.`).join('\n')); + assert.deepEqual(ids(many).filter((x) => x === 'too-long'), ['too-long']); + assert.equal(many.diagnostics.find((d) => d.id === 'too-long').line, 401); + // a program is read to its 2,000th line, and its pseudocode checked only when it is at most 2,000 lines long + const lines = readLingo(`To f X:\n Start with C at 0.\n${' Add 1 to C.\n'.repeat(30000)} The answer is C.`); + assert.match(lines.diagnostics.find((d) => d.id === 'too-long').message, /more than 2,000 lines long/); + const arms = readLingo(`To f X:\n If X is 0, the answer is 0.\n${Array.from({ length: 1500 }, (_, k) => ` Otherwise if X is ${k + 1}, the answer is ${k}.`).join('\n')}\n The answer is 1.`); + assert.match(arms.diagnostics.find((d) => d.id === 'too-long').message, /more than 2,000 lines of pseudocode/); + // just under the limits, the same shapes read + for (const src of [`To f X:\n The answer is ${'('.repeat(150)}X${')'.repeat(150)}.`, `To f X:\n The answer is X${' plus 1'.repeat(300)}.`]) { + assert.ok(!ids(readLingo(src)).some((x) => x === 'too-deep' || x === 'too-long')); + } + assert.equal(readLingo(`To f X:\n The answer is X${' plus 1'.repeat(300)}.`).ok, true); +}); + +test('matho: an equation nested past 200 levels is matho-too-deep, and one past 2,000 symbols matho-too-long', () => { + for (const src of [ + `f(x) = ${'('.repeat(300)}x${')'.repeat(300)}`, + `f(x) = ${'\\frac{1}{'.repeat(300)}x${'}'.repeat(300)}`, + `f(x) = ${'\\sqrt{'.repeat(300)}x${'}'.repeat(300)}`, + `f(x) = ${'-'.repeat(300)}x`, + `f(x) = ${'\\sin '.repeat(300)}x`, + `f(x) = x${'^{x'.repeat(300)}${'}'.repeat(300)}`, + ]) { + const r = parseMatho(src); + assert.deepEqual(ids(r), ['matho-too-deep'], src.slice(0, 40)); + assert.match(r.diagnostics[0].message, /nested more than 200 levels deep/); + everyPath('matho', src); + } + const long = parseMatho(`f(x) = x${' + 1'.repeat(1000)}`); + assert.deepEqual(ids(long), ['matho-too-long']); + for (const src of [`f(x) = ${'('.repeat(30000)}x${')'.repeat(30000)}`, `f(x) = x${' + 1'.repeat(30000)}`, `f(x) = ${'\\frac{1}{'.repeat(30000)}x`]) { + assert.deepEqual(ids(parseMatho(src)), ['matho-too-long']); + everyPath('matho', src); + } + assert.equal(parseMatho(`f(x) = ${'('.repeat(150)}x${')'.repeat(150)}`).ok, true); + // 600 factorials are 601 symbols, and 3,000 lines of pseudocode: not checked as pseudocode + const fact = parseMatho(`f(n) = n${'!'.repeat(600)}`); + assert.deepEqual(ids(fact), ['matho-too-long']); + assert.match(fact.diagnostics.at(-1).message, /more than 2,000 lines of pseudocode/); + // and a pane holds at most 500 equations (lowering 30,000 took 40 seconds) + const eqs = parseMatho(Array.from({ length: 30000 }, (_, k) => `f${k}(x) = x + ${k}`).join('\n')); + assert.deepEqual(ids(eqs), ['matho-too-long']); + assert.equal(eqs.diagnostics[0].line, 501); +}); + +test('speco: a claim of 30,000 nested bars reads in linear time (each bar trimmed and searched the whole claim)', { timeout: 10000 }, () => { + const big = '|{ K ∈ [0, X) : '.repeat(30000) + 'TRUE' + ' }|'.repeat(30000); + const t0 = performance.now(); + const doc = parseSpec(`SPEC F(X)\n ENSURE result = ${big}`); + assert.equal(doc.units[0].ensures.length, 1); + assert.ok(performance.now() - t0 < 5000); + toMath(toAscii(big)); + // and what it reads is what it read before + assert.equal(toAscii('|{ K ∈ [0, |T|) : T[K] = B }|'), 'LENGTH({ K IN [0, LENGTH(T)) : T[K] == B })'); + assert.equal(toAscii('X AND |T| = 0'), 'X AND LENGTH(T) == 0'); +}); + +test('x86-64: a setup region past 1 MiB is refused, and writing a large region at every step stops at 128 MiB of copies', () => { + const p = parseAsm('l:\n mov byte ptr [rdi], 1\n jmp l'); + assert.ok(p.ok); + for (const size of ['1048577', '4000000000', '-5', '0x7fffffffffffffff']) { + assert.throws(() => runAsmProgram(p, { maxSteps: 100, setup: `rdi = 0x1000\nmemory 0x1000 zero ${size}` }), /maps regions of 0 to 1,048,576 bytes/); + } + const ws = new Workspace({ sources: { asm: 'l:\n mov byte ptr [rdi], 1\n jmp l' }, lang: 'asm', setup: 'rdi = 0x1000\nmemory 0x1000 zero 4000000000' }); + assert.equal(ws.trace(), null); + assert.match(ws.runError, /1,048,576 bytes/); + // each step keeps its own copy of a region it writes: 1 MiB written at every step stops after 128 of them + const r = runAsmProgram(p, { maxSteps: 20000, setup: 'rdi = 0x1000\nmemory 0x1000 zero 1048576' }); + assert.match(r.error.message, /stopped: the steps so far wrote to memory regions totaling 128 MiB/); + assert.ok(r.count < 300); + // an endless loop stops at the step limit, as it always did + assert.equal(runAsmProgram(parseAsm('l:\n jmp l'), { maxSteps: 500 }).truncated, true); +}); + +test('a reader that throws leaves one engine-stop diagnostic on its pane; nothing throws out of the workspace or the hub', () => { + const src = 'To count the B bases in T:\n Start with Count at 0.\n For each letter c of T:\n If c is B, add 1 to Count.\n The answer is Count.\n'; + const parse = SERVICES.lingo.parse; + SERVICES.lingo.parse = () => { throw new TypeError('injected'); }; + try { + const ws = new Workspace({ sources: { lingo: src }, lang: 'lingo' }); + const ds = ws.diagnostics('lingo'); + assert.equal(ds.length, 1); + assert.equal(ds[0].id, FAULT_ID); + assert.match(ds[0].message, /could not read this pane \(injected\)/); + ws.allDiagnostics(); ws.trace(); ws.proof(); ws.lineMaps(); ws.callText(); ws.readsInput(); + } finally { SERVICES.lingo.parse = parse; } + + // the hub: a translator that throws marks the other panes stale with the reason, and keeps their text + const hub = new SyncHub({ pseudo: 'FUNCTION F(X : INTEGER) RETURNS INTEGER\n RETURN X\nENDFUNCTION\n', matho: 'f(x) = x' }); + hub.generateMatho = () => { throw new Error('injected'); }; + const r = hub.edit('pseudo', 'FUNCTION F(X : INTEGER) RETURNS INTEGER\n RETURN X + 1\nENDFUNCTION\n'); + assert.equal(r.authority, 'pseudo'); + assert.equal(r.sources.matho, 'f(x) = x'); + for (const l of LINKED.filter((x) => x !== 'pseudo')) assert.match(r.stale[l], /could not follow this change \(injected\)/); + assert.equal(r.mojo.diagnostics[0].id, FAULT_ID); + const filler = new SyncHub({ lingo: src }); + filler.lowerOne = () => { throw new Error('injected'); }; + const f = filler.fill(); + assert.equal(f.sources.lingo, src); + assert.match(f.stale.pseudo, /injected/); + + // the prover: a fault is said on the speco pane, and proof() is null as for a program that does not check + class Faulty extends Workspace { proofOf() { throw new Error('injected'); } } + const w = new Faulty({ sources: { pseudo: 'FUNCTION F(X : INTEGER) RETURNS INTEGER\n RETURN X\nENDFUNCTION\n' }, lang: 'pseudo' }); + assert.equal(w.proof(), null); + assert.equal(w.diagnostics('speco').at(-1).id, FAULT_ID); +}); diff --git a/tests/reference.test.mjs b/tests/reference.test.mjs new file mode 100644 index 0000000..6f9cf01 --- /dev/null +++ b/tests/reference.test.mjs @@ -0,0 +1,240 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The reference is only trustworthy if its examples run through the real parsers. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { REFERENCE, PSEUDO_REF, LINGO_REF, SPECO_REF, ASM_REF, MATHO_REF, explain, SPECO_SYMBOLS } from '../dist/common/reference.js'; +import { parseMatho, lowerMatho } from '../dist/services/matho.js'; +import { studyOf, runPseudo } from '../dist/services/pseudo.js'; +import { run as runProgram } from '../dist/engines/pseudo/interp.js'; +import { readLingo } from '../dist/services/lingo.js'; +import { parseSpeco, toAscii, toMath } from '../dist/services/speco.js'; +import { parseAsm } from '../dist/services/asm.js'; +import { KEYWORDS, BUILTINS, SOFT_BUILTINS } from '../dist/engines/pseudo/lang.js'; +import { OPS } from '../dist/engines/asm86/interp.js'; + +test('every pseudocode statement example parses and checks', () => { + const sec = PSEUDO_REF.sections.find((s) => s.id === 'pseudo-statements'); + for (const e of sec.entries) { + const st = studyOf(e.example); + const errs = st.diagnostics.filter((d) => d.severity === 'error'); + assert.equal(errs.length, 0, `${e.term}: ${errs.map((d) => `L${d.line} ${d.message}`).join(' | ')}`); + } +}); + +test('every keyword in the engine has a meaning in the reference, and vice versa', () => { + const sec = PSEUDO_REF.sections.find((s) => s.id === 'pseudo-keywords'); + const listed = new Set(sec.entries.map((e) => e.term)); + for (const k of KEYWORDS) assert.ok(listed.has(k), `keyword ${k} is missing from the reference`); + for (const e of sec.entries) assert.ok(e.meaning.length > 10, `${e.term} has no meaning`); + for (const b of Object.keys(BUILTINS)) assert.ok(explain('pseudo', b), `built-in ${b} has no hover`); +}); + +test('every lingo sentence example reads (inside a minimal routine where needed)', () => { + const failures = []; + // kinds for every name the examples mention, so a fragment reads the same way it does on the page + const kinds = 'T is a text. P is a text. S is a text. w is a text. A is a whole number. Score is a whole number.'; + const setup = [ + 'Start with Count at 0.', 'Start with Best at 0.', 'Start with Run at 0.', 'Start with Left at 3.', 'Start with window at "AC".', + 'Start with an empty list called Starts.', 'Start with an empty tally called Counts.', 'Start with an empty text called R.', + ]; + const defs = "The complement of b is 'T' when b is 'A', and 'N' otherwise."; + for (const sec of LINGO_REF.sections.filter((s) => s.id.startsWith('lingo-forms'))) { + for (const e of sec.entries) { + if (!e.example) continue; + const ex = e.example.trim(); + const examples = ex.includes(' ') && !/^To /.test(ex) && !/ when /.test(ex) ? ex.split(/\s{2,}/) : [ex]; + for (const one of examples) { + let src; + const head = 'To test T, P, S, w, A, B and Score:'; + const pre = setup.map((l) => ' ' + l).join('\n'); + if (/^The answer is/.test(one)) src = `${head}\n${pre}\n Start with i at 0.\n ${one}\n\n${kinds}`; + else if (/^To /.test(one)) src = `${one}\n${pre}\n The answer is Count.\n\n${kinds}`; + else if (/ when /.test(one) || /^The .* is /.test(one)) src = `To test P, T, S and b:\n Start with i at 0.\n If b is 'A', add 1 to i.\n The answer is the length of T plus the length of P plus the length of S plus i.\n\n${one}${/[.:]$/.test(one) ? '' : '.'}\n\nP is a text. T is a text. S is a text. b is a letter.`; + else if (/^(For|Repeat)/.test(one)) src = `${head}\n${pre}\n${/^For (each|every) .*\bi\b/.test(one) ? '' : ' Start with i at 0.\n'} ${one}\n Add 1 to Count.\n The answer is Count.\n\n${kinds}`; + else if (/^If /.test(one) && /:$/.test(one)) src = `${head}\n${pre}\n ${one}\n Add 1 to Count.\n The answer is Count.\n\n${kinds}`; + else if (sec.title === 'Tests') src = `${head}\n${pre}\n For each position i of T:\n Set b to the letter of T at i.\n ${/^If /.test(one) ? one : `If ${one}, add 1 to Count.`}\n The answer is Count.\n\n${kinds}`; + else if (sec.title === 'Values') src = `${head}\n${pre}\n For each position i of T:\n Set X to ${one}.\n The answer is Count.\n\n${kinds}`; + else src = `${head}\n${pre}\n Start with i at 0.\n${/\bb\b/.test(one) && !/\bB\b/.test(one) ? ' Set b to the letter of T at i.\n' : ''} ${one}\n The answer is Count.\n\n${/complement/.test(one) ? defs + '\n\n' : ''}${kinds}${/^Swap/.test(one) ? ' B is a whole number.' : ''}`; + const r = readLingo(src); + const tolerated = ['pronoun', 'kind-clash', 'unset-read', 'borrowed-input', 'which-letter', 'never-read', 'unreachable']; + const errs = r.diagnostics.filter((d) => d.severity === 'error' && !tolerated.includes(d.id)); + if (errs.length) failures.push(`${e.term} → "${one}": ${errs.map((d) => d.id + ': ' + d.message.slice(0, 100)).join(' | ')}`); + } + } + } + assert.deepEqual(failures, []); +}); + +test('every lingo ambiguity example raises exactly the question it documents', () => { + const sec = LINGO_REF.sections.find((s) => s.id === 'lingo-questions'); + for (const e of sec.entries) { + const r = readLingo(e.example); + assert.ok(r.diagnostics.some((d) => d.id === e.term), `${e.term}: got ${r.diagnostics.map((d) => d.id).join(', ')}`); + } +}); + +test('every speco example parses, and the symbol table round-trips', () => { + const sec = SPECO_REF.sections.find((s) => s.id === 'speco-forms'); + for (const e of sec.entries) { + const m = parseSpeco(e.example); + assert.ok(m.ok, `${e.term}: ${m.diagnostics.map((d) => d.message).join(' | ')}`); + } + for (const s of SPECO_SYMBOLS) { + if (s.math.length !== 1) continue; + assert.equal(toAscii(`A ${s.math} B`), `A ${s.ascii} B`); + assert.equal(toMath(`A ${s.ascii} B`), `A ${s.math} B`); + } +}); + +test('every instruction has a gloss and a hover; setup examples parse', () => { + for (const op of Object.keys(OPS)) assert.ok(explain('asm', op)?.body, `${op} has no gloss`); + for (const e of ASM_REF.sections.find((s) => s.id === 'asm-registers').entries) assert.ok(explain('asm', e.term), e.term); + assert.ok(parseAsm(' mov rax, [rbx]\n ret\n').ok); + assert.ok(REFERENCE.length === 5); +}); + +test('every pseudo idiom runs and prints what its note says', async () => { + const { runPseudo } = await import('../dist/services/pseudo.js'); + const { KEYWORDS: KW, BUILTINS: BI, CONTEXTUAL } = await import('../dist/engines/pseudo/lang.js'); + const sec = PSEUDO_REF.sections.find((s) => s.id === 'pseudo-idioms'); + assert.ok(sec.entries.length >= 8); + for (const e of sec.entries) { + const st = studyOf(e.example); + const errs = st.diagnostics.filter((d) => d.severity === 'error'); + assert.equal(errs.length, 0, `${e.term}: ${errs.map((d) => `L${d.line} ${d.message}`).join(' | ')}`); + const tr = runPseudo(st, { input: [], maxSteps: 200000 }); + assert.equal(tr.error, null, `${e.term}: ${tr.error?.message}`); + const out = tr.output.filter((l) => !/^\w+ returned /.test(l)); + assert.deepEqual(out, [e.note], e.term); + } + const reserved = PSEUDO_REF.sections.find((s) => s.id === 'pseudo-reserved'); + const words = new Set(reserved.entries.map((e) => e.term)); + for (const k of KW) assert.ok(words.has(k), `reserved list misses keyword ${k}`); + for (const k of Object.keys(CONTEXTUAL)) assert.ok(words.has(k), `reserved list misses ${k}`); + // the built-ins added for numerical work stay usable as names (a variable E, a FUNCTION GCD), so they are not listed + for (const k of Object.keys(BI)) assert.ok(SOFT_BUILTINS.has(k) ? !words.has(k) : words.has(k), SOFT_BUILTINS.has(k) ? `${k} can be a name, so it is not reserved` : `reserved list misses built-in ${k}`); +}); + +test('every matho example translates to pseudo that checks, and every refusal is refused', () => { + const forms = MATHO_REF.sections.find((s) => s.id === 'matho-forms'); + for (const e of forms.entries) { + const r = parseMatho(e.example); + const errs = r.diagnostics.filter((d) => d.severity === 'error'); + assert.ok(r.ok, `${e.term}: ${errs.map((d) => `L${d.line} ${d.id}: ${d.message}`).join(' | ')}\n${r.program.pseudo}`); + const st = studyOf(r.program.pseudo); + assert.ok(st.ok, `${e.term}: the pseudo does not check: ${st.diagnostics.filter((d) => d.severity === 'error').map((d) => d.message).join(' | ')}`); + } + for (const src of ['y = \\int_0^1 x dx', 's = \\sum_{i=0}^{\\infty} x_i', 'y = \\lim_{x \\to 0} x', 'y = \\partial f', 'y = a \\pm b', 'a = b = c']) { + const r = parseMatho(src); + assert.equal(r.ok, false, `${src} should be refused`); + assert.ok(r.diagnostics.some((d) => d.severity === 'error' && /matho-(refused|pm|chain)/.test(d.id)), src); + } +}); + +/** A run of `src` (which must check), with runPseudo's defaults. */ +const runOf = (src, opts = {}) => { + const st = studyOf(src); + assert.ok(st.ok, `${src}: ${st.diagnostics.filter((d) => d.severity === 'error').map((d) => d.message).join(' | ')}`); + return runPseudo(st, opts); +}; +const errorIds = (src) => studyOf(src).diagnostics.filter((d) => d.severity === 'error').map((d) => d.id); + +test('every row of "What stops a run" is a stop a run makes', () => { + // [program, the kind of stop, run options], one or more per row + const SHOWN = { + 'Position does not exist': [['DECLARE L : LIST OF INTEGER <- [1]\nOUTPUT L[1]', 'index']], + 'No value yet': [['DECLARE X : INTEGER\nOUTPUT X', 'unset']], + 'Missing key': [['DECLARE H : HASHMAP OF STRING : INTEGER\nOUTPUT H["a"]', 'key']], + 'Division by zero': [['DECLARE Y : INTEGER <- 0\nOUTPUT 1 DIV Y', 'divide'], ['DECLARE Y : FLOAT <- 0.0\nOUTPUT 1.0 / Y', 'divide'], ['OUTPUT 0.0 ^ -1', 'value']], + 'No answer': [['OUTPUT 2 ^ -1', 'value'], ['OUTPUT (-8.0) ^ 0.5', 'value'], ['OUTPUT SQRT(-1.0)', 'value'], ['OUTPUT TO_FLOAT("0x10")', 'value'], ['DECLARE L : LIST OF INTEGER\nOUTPUT MIN(L)', 'value']], + 'Wrong length': [['FUNCTION F(A : ARRAY[3] OF INTEGER) RETURNS INTEGER\n RETURN A[0]\nENDFUNCTION\nDECLARE L : LIST OF INTEGER <- [1, 2]\nOUTPUT F(L)', 'value'], ['OUTPUT DOT([1, 2], [3])', 'value']], + 'STEP 0': [['DECLARE S : INTEGER <- 0\nFOR I <- 0 TO 3 STEP S\n OUTPUT I\nNEXT I', 'value']], + 'Too large': [['OUTPUT 9007199254740991 + 1', 'overflow'], ['DECLARE X : INT8\nFOR X <- 120 TO 130\n OUTPUT X\nNEXT X', 'overflow'], ['DECLARE X : FLOAT32 <- 3e38\nX <- X * 10.0', 'overflow'], ['OUTPUT DOT([94906267, 1], [94906267, -9007199254740991])', 'overflow']], + 'Too much to hold': [['DECLARE L : LIST OF INTEGER <- FILL(2000001, 0)', 'value'], ['DECLARE S : STRING <- "ab"\nFOR I <- 0 TO 30\n S <- S & S\nNEXT I', 'value']], + 'Input': [['DECLARE X : FLOAT\nINPUT X', 'input', { input: ['0x1A'] }], ['DECLARE X : INTEGER\nINPUT X', 'input']], + 'A value from outside': [['PROCEDURE P(X : FLOAT)\n OUTPUT X\nENDPROCEDURE', 'value', { call: 'P(TRUE)' }]], + 'A broken promise': [['ASSERT 1 > 2', 'assert']], + 'No end in sight': [['WHILE TRUE\n OUTPUT 1\nENDWHILE', 'limit'], ['OUTPUT SUM I IN [0, 2000000) : SUM J IN [0, 2000000) : 1', 'limit']], + 'Too deep': [['FUNCTION F(N : INTEGER) RETURNS INTEGER\n RETURN F(N + 1)\nENDFUNCTION\nOUTPUT F(0)', 'depth']], + 'A data race': [['DECLARE T : INTEGER <- 0\nPARALLEL FOR I <- 0 TO 4\n T <- T + I\nNEXT I', 'race']], + }; + const sec = PSEUDO_REF.sections.find((s) => s.id === 'pseudo-stops'); + for (const e of sec.entries) { + assert.ok(SHOWN[e.term], `"${e.term}" has no program in this test`); + for (const [src, kind, opts] of SHOWN[e.term]) { + const r = opts?.call ? (() => { try { return runOf(src, opts); } catch (err) { return { error: { kind: 'value', message: err.message } }; } })() : runOf(src, opts); + assert.equal(r.error?.kind, kind, `${e.term}: ${src}`); + } + } + assert.deepEqual(Object.keys(SHOWN).sort(), sec.entries.map((e) => e.term).sort()); + // a value from outside that is NaN or an infinity, through the engine's own run() + const p = studyOf('PROCEDURE P(X : FLOAT)\n OUTPUT X\nENDPROCEDURE').program.program; + assert.equal(runProgram(p, { call: { name: 'P', args: [NaN] } }).error?.kind, 'value'); +}); + +test('the limits the reference lists are the ones the lab applies', () => { + const sec = PSEUDO_REF.sections.find((s) => s.id === 'pseudo-limits'); + const terms = sec.entries.map((e) => e.term); + for (const t of ['200 levels of nesting', '400 operands in a chain', '2,000,000 values', '10,000,000 characters', '20,000 steps', '400 calls deep', '10,000,000 recorded values']) assert.ok(terms.includes(t), t); + assert.ok(errorIds(`${'IF TRUE THEN\n'.repeat(201)}OUTPUT 1\n${'ENDIF\n'.repeat(201)}`).includes('too-deep')); + assert.deepEqual(errorIds(`${'IF TRUE THEN\n'.repeat(150)}OUTPUT 1\n${'ENDIF\n'.repeat(150)}`), []); + assert.ok(errorIds(`OUTPUT 1${' + 1'.repeat(400)}`).includes('too-deep')); + assert.deepEqual(runOf(`OUTPUT 1${' + 1'.repeat(399)}`).output, ['400']); + assert.ok(errorIds(`OUTPUT ((1${' + 1'.repeat(398)})${' + 1'.repeat(398)})${' + 1'.repeat(398)}`).includes('too-deep'), 'brackets that each hold a long chain'); + assert.equal(runOf('DECLARE A : ARRAY[2000001] OF INTEGER').error?.kind, 'value'); + assert.equal(runOf('DECLARE A : ARRAY[2000000] OF BOOL\nOUTPUT LENGTH(A)').error, null); + assert.equal(runOf('FOR I <- 0 TO 30000\nNEXT I').error?.kind, 'limit'); + assert.match(runOf('FOR I <- 0 TO 30000\nNEXT I').error.message, /20,000 steps/); + const deep = (n) => runOf(`FUNCTION F(N : INTEGER) RETURNS INTEGER\n IF N == 0 THEN\n RETURN 0\n ENDIF\n RETURN F(N - 1)\nENDFUNCTION\nOUTPUT F(${n})`); + assert.equal(deep(398).error, null); + assert.equal(deep(400).error?.kind, 'depth'); +}); + +test('the glosses the claims ledger found a run contradicting now say what a run does', () => { + const out = (src) => { const r = runOf(src); assert.equal(r.error, null, r.error?.message); return r.output; }; + const meaning = (id, term) => PSEUDO_REF.sections.flatMap((s) => s.entries).find((e) => e.term === term)?.meaning ?? (BUILTINS[term]?.doc || ''); + // CONSTANT: worked out once, built-ins and a random draw included; a variable is refused + assert.match(meaning('', 'CONSTANT <name> = <value>'), /worked out once/); + assert.deepEqual(out('CONSTANT A = 2 + 3\nCONSTANT B = LENGTH("ACGT")\nCONSTANT R = RAND(1.0)\nOUTPUT A, " ", B, " ", R == R'), ['5 4 TRUE']); + assert.ok(errorIds('DECLARE X : INTEGER <- 1\nCONSTANT A = X').length > 0); + // RAND: between 0 and X, never X; 0.0 for 0 + assert.match(BUILTINS.RAND.doc, /\(X, 0\] for X below 0/); + assert.deepEqual(out('OUTPUT RAND(0.0), " ", RAND(-5.0) <= 0.0, " ", RAND(-5.0) > -5.0'), ['0.0 TRUE TRUE']); + // the ranges of ATAN2, ATAN and TANH include their ends + assert.match(BUILTINS.ATAN2.doc, /\[-PI\(\), PI\(\)\]/); + assert.match(BUILTINS.ATAN.doc, /\[-PI\(\)\/2, PI\(\)\/2\]/); + assert.match(BUILTINS.TANH.doc, /\[-1, 1\]/); + assert.deepEqual(out('OUTPUT ATAN2(-0.0, -1.0) == -PI(), " ", ATAN(1.0e300) == PI() / 2, " ", TANH(20.0) == 1.0'), ['TRUE TRUE TRUE']); + // POW: 0 to a negative power stops + assert.match(BUILTINS.POW.doc, /nor 0 when Y is negative/); + assert.equal(runOf('OUTPUT POW(0.0, -1.0)').error?.kind, 'value'); + // TO_INTEGER of a CHAR digit is the digit; TO_FLOAT reads decimal text + assert.match(BUILTINS.TO_INTEGER.doc, /TO_INTEGER\('7'\) is 7/); + assert.deepEqual(out("OUTPUT TO_INTEGER('7'), \" \", TO_FLOAT(\"-1e3\")"), ['7 -1000.0']); + // UPPER and LOWER: a STRING may grow, a CHAR whose other case is two characters stays + assert.match(BUILTINS.UPPER.doc, /STRASSE/); + assert.deepEqual(out("DECLARE C : CHAR <- UPPER('ß')\nOUTPUT C, \" \", UPPER(\"straße\")"), ['ß STRASSE']); + // FLOAT32: arithmetic and stores round to a single; SQRT and its kind give a FLOAT + assert.match(meaning('', 'FLOAT32'), /SQRT, EXP and the other functions give a FLOAT/); + assert.match(meaning('', 'INTEGER'), /from -9,007,199,254,740,991 to 9,007,199,254,740,991/); + assert.match(meaning('', 'Division by zero'), /with USE IEEE, a FLOAT \/ gives an infinity/); + assert.deepEqual(out('DECLARE X : FLOAT32 <- 1.1\nOUTPUT EXACT(X ^ 3), " ", EXACT(SQRT(FLOAT32(2.0)))'), ['1.3310000896453857421875 1.4142135623730951454746218587388284504413604736328125']); + assert.ok(errorIds('DECLARE X : FLOAT32 <- 2.0\nDECLARE Y : FLOAT32\nY <- SQRT(X)').includes('type')); + // INTEGER: the run's range, both ends + assert.equal(runOf('OUTPUT -9007199254740991 - 1').error?.kind, 'overflow'); + // Division by zero: with USE IEEE a FLOAT / gives an infinity; DIV still stops + assert.deepEqual(out('USE IEEE\nOUTPUT 1.0 / 0.0'), ['infinity']); + assert.equal(runOf('USE IEEE\nOUTPUT 1 DIV 0').error?.kind, 'divide'); + // speco INVARIANT: in FOR EACH, a claim that names the loop's variable is not checked after the last pass; in a + // FOR the exit check sees the counter at the value that ended the loop + const inv = SPECO_REF.sections.flatMap((s) => s.entries).find((e) => e.term === 'INVARIANT <claim>').meaning; + assert.match(inv, /except in FOR EACH/); + assert.deepEqual(out('DECLARE T : INTEGER <- 0\nFOR EACH X IN [1, 2, 3]\n INVARIANT X > 0 AND T < 4\n T <- T + X\nNEXT X\nOUTPUT T'), ['6']); + assert.equal(runOf('DECLARE T : INTEGER <- 0\nFOR EACH X IN [1, 2, 3]\n INVARIANT T < 4\n T <- T + X\nNEXT X\nOUTPUT T').error?.kind, 'invariant'); + assert.equal(runOf('FOR I <- 0 TO 3\n INVARIANT I < 3\nNEXT I').error?.kind, 'invariant'); + // matho: FLOOR and CEIL do not make their operand INTEGER; MOD does + const lowered = (tex) => lowerMatho(parseMatho(tex), 'pseudo').source; + assert.match(lowered('y = \\lfloor n \\rfloor'), /N : FLOAT/); + assert.match(lowered('r = a \\bmod b'), /A : INTEGER/); +}); diff --git a/tests/runview.test.mjs b/tests/runview.test.mjs new file mode 100644 index 0000000..85e14ee --- /dev/null +++ b/tests/runview.test.mjs @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The Run view (core/runview.ts): the run top to bottom, one row per step, the variables the student follows as +// columns with their value after each step, and the lifecycle of the value each names. Checked against the +// interpreters' own traces for every exercise, in pseudo and in Mojo. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { runColumns, runRows, defaultFollow, valuesAt, rulerCells, gridCells } from '../dist/common/runview.js'; +import { Workspace } from '../dist/common/workspace.js'; +import { PROBLEMS, problemById, workForProblem } from '../dist/common/problems.js'; + + +const traceOf = (p, lang) => { + const st = workForProblem(p); st.lang = lang; + const s = new Workspace(st); + return s.trace(); +}; +const RUNS = []; +for (const p of PROBLEMS) for (const lang of ['pseudo', 'mojo']) { + if (!(p.sources?.[lang] ?? '').trim()) continue; + const tr = traceOf(p, lang); + if (tr && tr.count) RUNS.push({ id: `${p.id} (${lang})`, tr }); +} + +test('every run with a trace has its variables as columns, and a suggestion to follow', () => { + assert.ok(RUNS.length >= 25, `the exercises give runs to check (${RUNS.length})`); + for (const { id, tr } of RUNS) { + const cols = runColumns(tr); + const keys = new Set(cols.map((c) => c.key)); + assert.equal(keys.size, cols.length, `${id}: one column per routine and name`); + const follow = defaultFollow(cols); + assert.ok(follow.length <= 5 && follow.every((k) => keys.has(k)), `${id}: the suggestion is at most five of the run's variables`); + if (cols.some((c) => c.writes && c.name !== 'self' && c.name !== '_')) assert.ok(follow.length, `${id}: something is suggested when something changes`); + assert.ok(!follow.some((k) => k.endsWith('.self') || k.endsWith('._')), `${id}: never self or _`); + assert.equal(new Set(follow.map((k) => k.split('.')[0])).size <= 1, true, `${id}: the suggestion is one routine's variables`); + } +}); + +test('each row is one step, and each cell is the variable\'s value after that step', () => { + for (const { id, tr } of RUNS) { + const cols = runColumns(tr); + const follow = cols.map((c) => c.key).slice(0, 12); + const rows = runRows(tr, follow); + assert.equal(rows.length, tr.count, `${id}: a row per step`); + rows.forEach((r, i) => { + assert.equal(r.step, i); + assert.equal(r.line, tr.steps[i].line ?? null); + const after = valuesAt(tr, follow, i + 1); + for (const v of after) { + const c = r.cells[v.key]; + assert.equal(c.text, v.inScope ? v.text : null, `${id} step ${i + 1} ${v.key}: the cell is the value after the step`); + if (c.wrote) assert.ok((tr.steps[i].writes ?? []).some((w) => w.name === v.key.split('.').slice(1).join('.')), `${id} step ${i + 1}: ${v.key} marked written only when the step writes it`); + } + }); + // a window of the run is the same rows + const mid = Math.floor(tr.count / 2); + assert.deepEqual(runRows(tr, follow, mid, mid + 3).map((r) => r.step), [mid, mid + 1, mid + 2].filter((x) => x < tr.count)); + // only the steps that change something: a subset, and every step that writes is kept + const few = runRows(tr, follow, 0, tr.count, { onlyWrites: true }); + assert.ok(few.length <= rows.length); + for (let i = 0; i < tr.count; i++) if ((tr.steps[i].writes ?? []).length) assert.ok(few.some((r) => r.step === i), `${id}: step ${i + 1} writes, so it stays`); + // what each step printed adds up to the run's output + assert.deepEqual(rows.flatMap((r) => r.printed), tr.output.slice(0, tr.states[tr.count]?.outputLength ?? tr.output.length)); + } +}); + +test('the lifecycle of the value a variable names is in that variable\'s column', () => { + const tr = traceOf(problemById('demo'), 'mojo'); + const events = tr.steps.flatMap((s) => (s.life ?? []).filter((e) => e.kind !== 'named')); + assert.ok(events.length > 0, 'the demo makes, moves and destroys values'); + const cols = runColumns(tr); + const all = cols.map((c) => c.key); + const rows = runRows(tr, all); + const marks = rows.flatMap((r) => Object.values(r.cells).flatMap((c) => c.life)); + const others = rows.flatMap((r) => r.otherLife); + assert.ok(marks.length > 0, 'some events land on a named variable'); + // every event is shown once for each place it belongs: a move is a value moved in and a value moved out + const expected = events.reduce((n, e) => n + (e.kind === 'moved' && e.from != null ? 2 : 1), 0); + const named = new Map(); + for (const s of tr.steps) for (const e of s.life ?? []) if (e.kind === 'named') named.set(e.id, (named.get(e.id) ?? 0) + 1); + assert.ok(marks.length + others.length >= expected, `every event is shown (${marks.length} in columns, ${others.length} beside the step, ${expected} events)`); + for (const r of rows) for (const c of Object.values(r.cells)) for (const m of c.life) assert.match(m.kind, /^(made|copied|moved-in|moved-out|destroyed)$/); + // following no variable puts every event beside its step + const bare = runRows(tr, []); + assert.equal(bare.reduce((n, r) => n + r.otherLife.length, 0), expected); +}); + +test('strings and lists get a ruler, tables a grid', () => { + assert.deepEqual(rulerCells({ value: 'GAT', type: 'STRING' }), ['G', 'A', 'T']); + assert.equal(rulerCells({ value: 'G', type: 'CHAR' }), null, 'one character is not a ruler'); + assert.deepEqual(rulerCells({ text: '"GAT"', value: { k: 'str', v: 'GAT' } }), ['G', 'A', 'T'], 'a Mojo String'); + assert.deepEqual(rulerCells({ text: '[1, 2]', value: { k: 'list', items: [{ k: 'int', v: 1n }, { k: 'int', v: 2n }] } }), ['1', '2'], 'a Mojo List[Int]'); + assert.equal(rulerCells({ value: 'x'.repeat(81) }), null, 'too long to draw'); + const ed = traceOf(problemById('edit-distance'), 'pseudo'); + const d = runColumns(ed).find((c) => c.name === 'D'); + assert.ok(d, 'edit-distance has its table D'); + const [v] = valuesAt(ed, [d.key], ed.count - 1); + assert.ok(v.grid && v.grid.length > 1 && v.grid[0].length > 1, 'D is drawn as a grid'); + assert.ok(v.grid.flat().some((x) => /^\d+$/.test(x)), 'with its numbers in the cells'); + assert.equal(gridCells({ value: [[1, 2], 3] }), null, 'not a table'); +}); diff --git a/tests/services.test.mjs b/tests/services.test.mjs new file mode 100644 index 0000000..b54594a --- /dev/null +++ b/tests/services.test.mjs @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Runs against the compiled output: `npm test` builds first. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { toAscii, toMath, parseSpec, weave, extractSpec } from '../dist/services/speco.js'; +import { studyOf, runPseudo, provePseudo } from '../dist/services/pseudo.js'; +import { readLingo, runLingo } from '../dist/services/lingo.js'; +import { parseAsm, runAsmProgram, parseSetup } from '../dist/services/asm.js'; +import { Workspace } from '../dist/common/workspace.js'; +import { PROBLEMS, problemById, workForProblem } from '../dist/common/problems.js'; + + +test('every catalog problem parses, runs, and (where it has pseudocode) proves without throwing', () => { + for (const p of PROBLEMS) { + const s = new Workspace(workForProblem(p)); + s.allDiagnostics(); + const tr = s.trace(); + const empty = !Object.values(p.sources).some((src) => src && src.trim()); + if (p.id !== 'five-bugs' && !empty) assert.ok(tr, `${p.id} produced no trace: ${s.runError}`); + if (p.sources.pseudo || p.sources.lingo) { s.proof(); s.tests(); } + } +}); + +test('the known-good examples give the lesson answers', () => { + const by = Object.fromEntries(PROBLEMS.map((p) => [p.id, p])); + const last = (id, lang) => { const st = workForProblem(by[id]); if (lang) st.lang = lang; const s = new Workspace(st); return s.trace().output.at(-1); }; + assert.equal(last('count-base', 'pseudo'), 'COUNT_BASE returned 3'); + assert.equal(last('count-base', 'lingo'), 'COUNT_BASES returned 3'); + assert.equal(last('match', 'pseudo'), 'MATCH returned [1]'); + assert.equal(last('revcomp', 'pseudo'), 'REVERSE_COMPLEMENT returned "TGTAATC"'); + assert.match(last('asm-first-instructions'), /^rax = 6/); +}); + +test('the off-by-one bug is caught by the run, the tests, and the proof', () => { + const st = workForProblem(PROBLEMS.find((p) => p.id === 'match-off-by-one')); st.lang = 'pseudo'; + const s = new Workspace(st); + assert.equal(s.trace().error.kind, 'index'); + assert.ok(s.tests().some((t) => t.failures.length > 0)); + assert.ok(s.proof().report.counts.broken >= 1); +}); + +test('speco: math notation lowers to the pseudocode spelling and back', () => { + assert.equal(toAscii('result = |{ k ∈ [0, |T|) : T[k] = B }|'), 'RESULT == LENGTH({ k IN [0, LENGTH(T)) : T[k] == B })'); + assert.equal(toAscii('∀ K ∈ [0, M) : A[K] ≤ B ∧ ¬ (X ≠ Y)'), 'FORALL K IN [0, M) : A[K] <= B AND NOT (X != Y)'); + assert.equal(toMath('RESULT == LENGTH({ K IN [0, LENGTH(T)) : T[K] == B })'), 'result = |{ K ∈ [0, |T|) : T[K] = B }|'); + assert.equal(toAscii(toMath('FORALL K IN [0, J) : T[I + K] == P[K]')), 'FORALL K IN [0, J) : T[I + K] == P[K]'); +}); + +test('speco: a spec written separately weaves in, proves, and does not duplicate what is already there', () => { + const pseudo = PROBLEMS.find((p) => p.id === 'count-base').sources.pseudo; + const spec = parseSpec(`SPEC COUNT_BASE(B, T)\n ENSURE result = |{ K ∈ [0, |T|) : T[K] = B }|\n LOOP FOR I\n INVARIANT Total = |{ K ∈ [0, I) : T[K] = B }|`); + assert.equal(spec.diagnostics.length, 0); + const w = weave(pseudo, spec); + assert.equal(w.diagnostics.length, 0); + assert.match(w.source, /FUNCTION COUNT_BASE[^\n]*\n\s+ENSURE RESULT == LENGTH/); + assert.match(w.source, /FOR I <- 0 TO N\n\s+INVARIANT Total == LENGTH/); + const st = studyOf(w.source); + assert.ok(st.ok, st.diagnostics.map((d) => d.message).join('; ')); + const pr = provePseudo(st); + assert.equal(pr.counts.open + pr.counts.broken, 0, JSON.stringify(pr.counts)); + assert.ok(pr.counts.proved >= 3); + // weaving the same spec again adds nothing + const again = weave(w.source, spec); + assert.equal(again.source, w.source); + // and extracting it back gives a spec that round-trips + const ex = extractSpec(w.source); + assert.match(ex, /^SPEC COUNT_BASE\(B, T\)/); + assert.equal(weave(pseudo, parseSpec(ex)).source, w.source); +}); + +test('speco: a spec for a routine that does not exist is reported, not silently dropped', () => { + const w = weave('FUNCTION F() RETURNS INTEGER\n RETURN 1\nENDFUNCTION', parseSpec('SPEC G()\n ENSURE RESULT == 1')); + assert.equal(w.diagnostics.length, 1); + assert.equal(w.diagnostics[0].id, 'no-routine'); +}); + +test('lingo: English steps lower, run, and report English line numbers', () => { + const r = readLingo('To count the G bases in T:\n Start with Count at 0.\n For each letter b of T:\n If b is \'G\', add 1 to Count.\n The answer is Count.\n'); + assert.ok(r.ok, r.diagnostics.map((d) => d.message).join('; ')); + const tr = runLingo(r, { call: 'COUNT_BASE("GGA")' }); // a call aimed at another routine is re-aimed + assert.equal(tr.output.at(-1), 'COUNT_G_BASES returned 2'); + assert.ok(tr.steps.every((s) => s.line == null || s.line <= 5), 'steps carry English lines'); +}); + +test('lingo: an ambiguous range asks a question with two rewrites', () => { + const r = readLingo('To count in T:\n Start with Count at 0.\n For each i from 0 to the length of T:\n Add 1 to Count.\n The answer is Count.\n'); + const q = r.diagnostics.find((d) => d.id === 'range-ends'); + assert.ok(q && q.question && q.question.options.length === 2); + assert.match(q.question.options[0].text, /up to but not including/); +}); + +test('asm: setup lines, assembly errors, and a run', () => { + const setup = parseSetup('rdi = 0x1000\nrsi = 7\ndl = \'A\'\nmemory 0x1000 "GATTACA"\nflags CF'); + assert.equal(setup.registers.rdi, 0x1000n); assert.equal(setup.registers.dl, 65n); assert.equal(setup.memory[0].text, 'GATTACA'); assert.equal(setup.flags.CF, true); + const bad = parseAsm(' mov rax, [rbx], [rcx]\n'); + assert.equal(bad.ok, false); assert.equal(bad.diagnostics[0].line, 1); + const good = parseAsm(' mov eax, 40\n add eax, 2\n ret\n'); + const tr = runAsmProgram(good, {}); + assert.equal(tr.count, 3); + assert.match(tr.output.at(-1), /rax = 42/); +}); diff --git a/tests/snapshots/practices.json b/tests/snapshots/practices.json new file mode 100644 index 0000000..1109ae5 --- /dev/null +++ b/tests/snapshots/practices.json @@ -0,0 +1,68 @@ +{ + "count-base": [ + "6-8 declare-then-assign", + "7-9 declare-then-assign", + "11-13 codepoint-loop", + "15 augmented-assign" + ], + "count-base-proved": [ + "6-8 declare-then-assign", + "7-9 declare-then-assign", + "11-12 codepoint-loop", + "13 augmented-assign" + ], + "match": [ + "6-10 declare-then-assign", + "7-11 declare-then-assign" + ], + "match-proved": [ + "8-12 declare-then-assign", + "9-13 declare-then-assign" + ], + "match-off-by-one": [ + "6-10 declare-then-assign", + "7-11 declare-then-assign" + ], + "match-reset": [ + "5-9 declare-then-assign", + "6-10 declare-then-assign", + "7-11 declare-then-assign" + ], + "first-unknown": [ + "7-8 declare-then-assign", + "11 augmented-assign" + ], + "gc-fraction": [ + "6-7 declare-then-assign", + "8-9 codepoint-loop", + "11 augmented-assign" + ], + "hamming": [ + "7-8 declare-then-assign", + "12 augmented-assign" + ], + "best-position": [ + "8-9 declare-then-assign" + ], + "tally": [ + "7-8 codepoint-loop" + ], + "kmers": [], + "revcomp": [ + "6-8 declare-then-assign", + "20 augmented-assign" + ], + "binary-search": [ + "6-9 declare-then-assign", + "7-10 declare-then-assign" + ], + "edit-distance": [ + "5-7 declare-then-assign", + "6-8 declare-then-assign" + ], + "translate-codons": [ + "11-18 declare-then-assign", + "27 augmented-assign", + "29 augmented-assign" + ] +} diff --git a/tests/snapshots/sync-mojo/best-position.mojo b/tests/snapshots/sync-mojo/best-position.mojo new file mode 100644 index 0000000..48aafe9 --- /dev/null +++ b/tests/snapshots/sync-mojo/best-position.mojo @@ -0,0 +1,19 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# The position of the highest quality score; the first one, if several tie. +def best_position(q: List[Int]) -> Int: + # REQUIRE LENGTH(Q) >= 1 + # ENSURE 0 <= RESULT AND RESULT < LENGTH(Q) + # ENSURE FORALL K IN [0, LENGTH(Q)) : Q[K] <= Q[RESULT] + var best: Int + best = 0 + # INVARIANT 0 <= Best AND Best < I + # INVARIANT FORALL K IN [0, I) : Q[K] <= Q[Best] + for i in range(1, len(q)): + if q[i] > q[best]: + best = i + return best + + +def main(): + print(best_position([12, 30, 25, 30, 8])) diff --git a/tests/snapshots/sync-mojo/binary-search.mojo b/tests/snapshots/sync-mojo/binary-search.mojo new file mode 100644 index 0000000..9c08193 --- /dev/null +++ b/tests/snapshots/sync-mojo/binary-search.mojo @@ -0,0 +1,25 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# Is X in the sorted list L? Halve the range that could hold it each time. +def contains_sorted(l: List[Int], x: Int) -> Bool: + # REQUIRE FORALL K IN [0, LENGTH(L) - 1) : L[K] <= L[K + 1] + var lo: Int + var hi: Int + var mid: Int + lo = 0 + hi = len(l) + # INVARIANT 0 <= Lo AND Lo <= Hi AND Hi <= LENGTH(L) + # DECREASES Hi - Lo + while lo < hi: + mid = (lo + hi) // 2 + if l[mid] == x: + return True + elif l[mid] < x: + lo = mid + 1 + else: + hi = mid + return False + + +def main(): + print(contains_sorted([2, 5, 8, 13, 21], 13)) diff --git a/tests/snapshots/sync-mojo/count-base-proved.mojo b/tests/snapshots/sync-mojo/count-base-proved.mojo new file mode 100644 index 0000000..79d4db3 --- /dev/null +++ b/tests/snapshots/sync-mojo/count-base-proved.mojo @@ -0,0 +1,18 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# How many positions of T hold the base B, with its specification. +def count_base(b: String, t: String) -> Int: + # ENSURE RESULT == LENGTH({ K IN [0, LENGTH(T)) : T[K] == B }) + var n: Int + var total: Int + n = t.count_codepoints() + total = 0 + # INVARIANT Total == LENGTH({ K IN [0, I) : T[K] == B }) + for i in range(n): + if String(t[codepoint=i]) == b: + total = total + 1 + return total + + +def main(): + print(count_base("A", "GATTACA")) diff --git a/tests/snapshots/sync-mojo/count-base.mojo b/tests/snapshots/sync-mojo/count-base.mojo new file mode 100644 index 0000000..4f1f426 --- /dev/null +++ b/tests/snapshots/sync-mojo/count-base.mojo @@ -0,0 +1,21 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# How many positions of T hold the base B. +def count_base(b: String, t: String) -> Int: + # Set up + var n: Int + var total: Int + n = t.count_codepoints() + total = 0 + # Visit each position + for i in range(n): + # Test this position + if String(t[codepoint=i]) == b: + # Update the running total + total = total + 1 + # Report + return total + + +def main(): + print(count_base("A", "GATTACA")) diff --git a/tests/snapshots/sync-mojo/edit-distance.mojo b/tests/snapshots/sync-mojo/edit-distance.mojo new file mode 100644 index 0000000..66d007f --- /dev/null +++ b/tests/snapshots/sync-mojo/edit-distance.mojo @@ -0,0 +1,31 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# The fewest single-letter edits (substitute, insert, delete) that turn A into B. +def edit_distance(a: String, b: String) -> Int: + var n: Int + var m: Int + n = a.count_codepoints() + m = b.count_codepoints() + var d = List[List[Int]]() + for _ in range(n + 1): + var _row1 = List[Int]() + for _ in range(m + 1): + _row1.append(0) + d.append(_row1^) + for i in range(n + 1): + d[i][0] = i + for j in range(m + 1): + d[0][j] = j + for i in range(1, n + 1): + for j in range(1, m + 1): + var cost: Int + if String(a[codepoint=i - 1]) == String(b[codepoint=j - 1]): + cost = 0 + else: + cost = 1 + d[i][j] = min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + cost) + return d[n][m] + + +def main(): + print(edit_distance("GATTACA", "GCATGCA")) diff --git a/tests/snapshots/sync-mojo/first-unknown.mojo b/tests/snapshots/sync-mojo/first-unknown.mojo new file mode 100644 index 0000000..762baa9 --- /dev/null +++ b/tests/snapshots/sync-mojo/first-unknown.mojo @@ -0,0 +1,16 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# The position of the first N in T, or LENGTH(T) when there is none. +def first_unknown(t: String) -> Int: + # ENSURE 0 <= RESULT AND RESULT <= LENGTH(T) + # ENSURE FORALL K IN [0, RESULT) : T[K] != 'N' + var i: Int + i = 0 + # INVARIANT FORALL K IN [0, I) : T[K] != 'N' + while i < t.count_codepoints() and String(t[codepoint=i]) != "N": + i = i + 1 + return i + + +def main(): + print(first_unknown("GATNACA")) diff --git a/tests/snapshots/sync-mojo/gc-fraction.mojo b/tests/snapshots/sync-mojo/gc-fraction.mojo new file mode 100644 index 0000000..7a5d942 --- /dev/null +++ b/tests/snapshots/sync-mojo/gc-fraction.mojo @@ -0,0 +1,16 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# The fraction of T that is G or C. +def gc_fraction(t: String) -> Float64: + # REQUIRE LENGTH(T) > 0 + var gc: Int + gc = 0 + for _b_pos1 in range(t.count_codepoints()): + var b = String(t[codepoint=_b_pos1]) + if b == "G" or b == "C": + gc = gc + 1 + return Float64(gc) / Float64(t.count_codepoints()) + + +def main(): + print(gc_fraction("GATTACA")) diff --git a/tests/snapshots/sync-mojo/hamming.mojo b/tests/snapshots/sync-mojo/hamming.mojo new file mode 100644 index 0000000..0564364 --- /dev/null +++ b/tests/snapshots/sync-mojo/hamming.mojo @@ -0,0 +1,17 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# How many positions differ between two reads of the same length. +def hamming(a: String, b: String) -> Int: + # REQUIRE LENGTH(A) == LENGTH(B) + # ENSURE RESULT == LENGTH({ K IN [0, LENGTH(A)) : A[K] != B[K] }) + var differences: Int + differences = 0 + # INVARIANT Differences == LENGTH({ K IN [0, I) : A[K] != B[K] }) + for i in range(a.count_codepoints()): + if String(a[codepoint=i]) != String(b[codepoint=i]): + differences = differences + 1 + return differences + + +def main(): + print(hamming("GATTACA", "GACTATA")) diff --git a/tests/snapshots/sync-mojo/kmers.mojo b/tests/snapshots/sync-mojo/kmers.mojo new file mode 100644 index 0000000..e0d379f --- /dev/null +++ b/tests/snapshots/sync-mojo/kmers.mojo @@ -0,0 +1,21 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# How many times each word of K letters occurs in T. +def kmer_counts(t: String, k: Int) raises -> Dict[String, Int]: + # REQUIRE K >= 1 + var counts = Dict[String, Int]() + var word: String + for i in range(t.count_codepoints() - k + 1): + var _piece2 = String("") + for _at1 in range(i, i + k): + _piece2 += String(t[codepoint=_at1]) + word = _piece2 + if word in counts: + counts[word] = counts[word] + 1 + else: + counts[word] = 1 + return counts.copy() + + +def main() raises: + print(kmer_counts("GATTACAGATT", 3)) diff --git a/tests/snapshots/sync-mojo/match-off-by-one.mojo b/tests/snapshots/sync-mojo/match-off-by-one.mojo new file mode 100644 index 0000000..0f60d16 --- /dev/null +++ b/tests/snapshots/sync-mojo/match-off-by-one.mojo @@ -0,0 +1,27 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# One bound is wrong. Every position where pattern P occurs exactly in text T. +def match_(p: String, t: String) -> List[Int]: + # Set up + var n: Int + var m: Int + var matched: Bool + var answer = List[Int]() + n = t.count_codepoints() + m = p.count_codepoints() + # Visit each legal start + for i in range(n): + # Test: does the pattern match here? + matched = True + for j in range(m): + if String(t[codepoint=i + j]) != String(p[codepoint=j]): + matched = False + break + # Update + if matched: + answer.append(i) + return answer.copy() + + +def main(): + print(match_("ATT", "GATTACA")) diff --git a/tests/snapshots/sync-mojo/match-proved.mojo b/tests/snapshots/sync-mojo/match-proved.mojo new file mode 100644 index 0000000..e19b637 --- /dev/null +++ b/tests/snapshots/sync-mojo/match-proved.mojo @@ -0,0 +1,28 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# MATCH, with the specification from "Specifying the problem", +# and the two invariants that prove it. +def match_(p: String, t: String) -> List[Int]: + # REQUIRE LENGTH(P) >= 1 + # ENSURE RESULT == SORTED({ S IN [0, LENGTH(T) - LENGTH(P) + 1) : FORALL K IN [0, LENGTH(P)) : T[S + K] == P[K] }) + var n: Int + var m: Int + var matched: Bool + var answer = List[Int]() + n = t.count_codepoints() + m = p.count_codepoints() + # INVARIANT Answer == SORTED({ S IN [0, I) : FORALL K IN [0, M) : T[S + K] == P[K] }) + for i in range(n - m + 1): + matched = True + # INVARIANT Matched == (FORALL K IN [0, J) : T[I + K] == P[K]) + for j in range(m): + if String(t[codepoint=i + j]) != String(p[codepoint=j]): + matched = False + break + if matched: + answer.append(i) + return answer.copy() + + +def main(): + print(match_("ATT", "GATTACA")) diff --git a/tests/snapshots/sync-mojo/match-reset.mojo b/tests/snapshots/sync-mojo/match-reset.mojo new file mode 100644 index 0000000..94e4dfb --- /dev/null +++ b/tests/snapshots/sync-mojo/match-reset.mojo @@ -0,0 +1,23 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# Matched is reset in the wrong place. +def match_(p: String, t: String) -> List[Int]: + var n: Int + var m: Int + var matched: Bool + var answer = List[Int]() + n = t.count_codepoints() + m = p.count_codepoints() + matched = True + for i in range(n - m + 1): + for j in range(m): + if String(t[codepoint=i + j]) != String(p[codepoint=j]): + matched = False + break + if matched: + answer.append(i) + return answer.copy() + + +def main(): + print(match_("GC", "GATGCA")) diff --git a/tests/snapshots/sync-mojo/match.mojo b/tests/snapshots/sync-mojo/match.mojo new file mode 100644 index 0000000..0be5eab --- /dev/null +++ b/tests/snapshots/sync-mojo/match.mojo @@ -0,0 +1,27 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# Every position where pattern P occurs exactly in text T. +def match_(p: String, t: String) -> List[Int]: + # Set up + var n: Int + var m: Int + var matched: Bool + var answer = List[Int]() + n = t.count_codepoints() + m = p.count_codepoints() + # Visit each legal start + for i in range(n - m + 1): + # Test: does the pattern match here? + matched = True + for j in range(m): + if String(t[codepoint=i + j]) != String(p[codepoint=j]): + matched = False + break + # Update + if matched: + answer.append(i) + return answer.copy() + + +def main(): + print(match_("ATT", "GATTACA")) diff --git a/tests/snapshots/sync-mojo/revcomp.mojo b/tests/snapshots/sync-mojo/revcomp.mojo new file mode 100644 index 0000000..f750e15 --- /dev/null +++ b/tests/snapshots/sync-mojo/revcomp.mojo @@ -0,0 +1,25 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# The reverse complement of a strand: read backwards, each base swapped for its partner. +def reverse_complement(t: String) -> String: + # ENSURE LENGTH(RESULT) == LENGTH(T) + var answer: String + var partner: String + answer = "" + for i in range(t.count_codepoints() - 1, -1, -1): + if String(t[codepoint=i]) == "A": + partner = "T" + elif String(t[codepoint=i]) == "T": + partner = "A" + elif String(t[codepoint=i]) == "G": + partner = "C" + elif String(t[codepoint=i]) == "C": + partner = "G" + else: + partner = "N" + answer = answer + partner + return answer + + +def main(): + print(reverse_complement("GATTACA")) diff --git a/tests/snapshots/sync-mojo/tally.mojo b/tests/snapshots/sync-mojo/tally.mojo new file mode 100644 index 0000000..1910d59 --- /dev/null +++ b/tests/snapshots/sync-mojo/tally.mojo @@ -0,0 +1,17 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. + +# How many times each base appears in T. +def tally_bases(t: String) raises -> Dict[String, Int]: + var tally = Dict[String, Int]() + var b: String + for i in range(t.count_codepoints()): + b = String(t[codepoint=i]) + if b in tally: + tally[b] = tally[b] + 1 + else: + tally[b] = 1 # first sighting of this base + return tally.copy() + + +def main() raises: + print(tally_bases("GATTACA")) diff --git a/tests/snapshots/sync-mojo/translate-codons.mojo b/tests/snapshots/sync-mojo/translate-codons.mojo new file mode 100644 index 0000000..327649d --- /dev/null +++ b/tests/snapshots/sync-mojo/translate-codons.mojo @@ -0,0 +1,34 @@ +# Generated by Programming+ from pseudo. Not reviewed for Mojo best practices. +from std.collections import Set + +# Translate a reading frame into amino acids, stopping at the first stop codon. +def stops() -> Set[String]: + return Set[String]("TAA", "TAG", "TGA") + + +def translate(t: String) raises -> String: + var table = Dict[String, String]() + var protein: String + var codon: String + table["ATG"] = "M" + table["GCC"] = "A" + table["AAA"] = "K" + table["TTT"] = "F" + table["GGC"] = "G" + protein = "" + for i in range(0, t.count_codepoints() // 3 * 3, 3): + var _piece2 = String("") + for _at1 in range(i, i + 3): + _piece2 += String(t[codepoint=_at1]) + codon = _piece2 + if codon in stops(): + break + elif codon in table: + protein = protein + table[codon] + else: + protein = protein + "X" + return protein + + +def main() raises: + print(translate("ATGGCCAAATAGTTT")) diff --git a/tests/spdx.test.mjs b/tests/spdx.test.mjs new file mode 100644 index 0000000..3653f64 --- /dev/null +++ b/tests/spdx.test.mjs @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Every source file written here says who holds its copyright and under which license, in two SPDX lines at its +// top, so a file copied out of the repository still says where it came from. Left without them, on purpose: +// generated files (their tools rewrite them), and the files that came from the course (src/engines/ and +// src/curriculum/), which THIRD_PARTY.md puts under the same copyright and license. A new file fails here until +// it has the lines; this test goes with them if the owner drops the headers. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +const ROOT = new URL('..', import.meta.url).pathname; +const LEFT_OUT = [/^src\/engines\//, /^src\/curriculum\//, /^src\/mojo\/generated\//, /^src\/library\/generated\.ts$/]; +const HOLDER = 'SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC'; +const LICENSE = 'SPDX-License-Identifier: Apache-2.0'; + +test('every hand-written source, script and test starts with its SPDX lines', () => { + const files = []; + const walk = (dir) => { + for (const e of fs.readdirSync(path.join(ROOT, dir), { withFileTypes: true })) { + const f = `${dir}/${e.name}`; + if (e.isDirectory()) { if (!['node_modules', '__pycache__'].includes(e.name)) walk(f); } + else if (/\.(ts|js|mjs|py|sh)$/.test(e.name) && !LEFT_OUT.some((r) => r.test(f))) files.push(f); + } + }; + for (const d of ['src', 'scripts', 'tests']) walk(d); + assert.ok(files.length > 140, `${files.length} files read`); + const missing = []; + for (const f of files) { + const lines = fs.readFileSync(path.join(ROOT, f), 'utf8').split('\n'); + const at = lines[0].startsWith('#!') ? 1 : 0; + const mark = /\.(py|sh)$/.test(f) ? '# ' : '// '; + if (lines[at] !== mark + HOLDER || lines[at + 1] !== mark + LICENSE) missing.push(f); + } + assert.deepEqual(missing, []); + assert.match(fs.readFileSync(path.join(ROOT, 'LICENSE.md'), 'utf8'), /Apache License/); +}); diff --git a/tests/sync-coverage.test.mjs b/tests/sync-coverage.test.mjs new file mode 100644 index 0000000..bef72c5 --- /dev/null +++ b/tests/sync-coverage.test.mjs @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// What each translator can and cannot say, over every exercise, every corpus program, and the single-construct +// fixtures. The test writes docs/sync-coverage.md (deterministically, so a rerun changes nothing) and holds each +// translator to a floor that can only rise. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { programs, EQUATIONS } from './helpers/sync-programs.mjs'; +import { studyOf } from '../dist/services/pseudo.js'; +import { checkMojo } from '../dist/mojo/check.js'; +import { pseudoToMojo } from '../dist/sync/pseudo-to-mojo.js'; +import { pseudoToLingo } from '../dist/sync/pseudo-to-lingo.js'; +import { pseudoToMatho } from '../dist/sync/pseudo-to-matho.js'; +import { parseMatho } from '../dist/services/matho.js'; + +globalThis.localStorage = { getItem() { return null; }, setItem() { }, removeItem() { } }; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const docPath = path.join(here, '..', 'docs', 'sync-coverage.md'); + +/** Floors: programs each translator says completely. They only rise. */ +const FLOOR = { mojo: 76, lingo: 31, matho: 3 }; + +const reason = (m) => m + .replace(/^English has no sentence for /, '').replace(/^matho has no form for /, '') + .replace(/, so the (lingo|matho) pane keeps its last good text\.?$/, '').replace(/ \(line \d+\)/g, '') + .replace(/^not translated: pseudo line \d+/, 'a line with no faithful Mojo spelling'); + +function tally(rows) { + const by = new Map(); + for (const [id, why] of rows) { const e = by.get(why) ?? []; if (!e.includes(id)) e.push(id); by.set(why, e); } + return [...by.entries()].sort((a, b) => b[1].length - a[1].length || a[0].localeCompare(b[0])); +} +const list = (ids) => (ids.length > 4 ? `${ids.slice(0, 4).join(', ')}, and ${ids.length - 4} more` : ids.join(', ')); + +function measure() { + const all = programs().map((p) => ({ ...p, parses: studyOf(p.pseudo).ok })); + const ok = all.filter((p) => p.parses); + const rows = { mojo: { full: 0, gaps: [], checker: 0, no: [] }, lingo: { full: 0, no: [] }, matho: { full: 0, no: [] } }; + for (const p of ok) { + const m = pseudoToMojo(p.pseudo, { call: p.call }); + if ('unsupported' in m) rows.mojo.no.push([p.id, reason(m.unsupported[0].message)]); + else { + const gaps = m.notes.filter((n) => n.id === 'translate-gap'); + const clean = !checkMojo(m.source).diagnostics.some((d) => d.severity === 'error'); + if (gaps.length) for (const g of gaps) rows.mojo.gaps.push([p.id, reason(g.message)]); + else if (clean) rows.mojo.full++; + else rows.mojo.checker++; + } + const l = pseudoToLingo(p.pseudo); + if ('unsupported' in l) for (const d of l.unsupported) rows.lingo.no.push([p.id, reason(d.message)]); else rows.lingo.full++; + const t = pseudoToMatho(p.pseudo); + if ('unsupported' in t) rows.matho.no.push([p.id, reason(t.unsupported[0].message)]); else rows.matho.full++; + } + const eqs = Object.entries(EQUATIONS).map(([id, [src]]) => { + const r = pseudoToMatho(parseMatho(src).program.pseudo); + return [id, 'unsupported' in r ? reason(r.unsupported[0].message) : null]; + }); + return { all, ok, rows, eqs }; +} + +function report(x) { + const { all, ok, rows, eqs } = x; + const ex = all.filter((p) => p.id.startsWith('ex:')).length; + const fx = all.filter((p) => p.id.startsWith('fixture:')).length; + const out = []; + out.push('# Translation coverage', '', + '<!-- Written by tests/sync-coverage.test.mjs. Do not edit; rerun `npm test` to refresh. -->', '', + `Each translator in \`src/sync/\` is run over every exercise with pseudo (${ex}), every program in \`library/pseudo/programs/\` (${all.length - ex - fx}), and ${fx} one-construct fixtures (\`tests/helpers/sync-programs.mjs\`). ${all.length - ok.length} of the ${all.length} have pseudo that does not read, by design (they teach the notation's error messages), and are left out below. That leaves **${ok.length} programs**.`, '', + '| From pseudo to | Says it completely | Says it with a gap or a checker error | Cannot say it |', '| --- | ---: | ---: | ---: |', + `| Mojo | ${rows.mojo.full} | ${new Set(rows.mojo.gaps.map((g) => g[0])).size + rows.mojo.checker} | ${rows.mojo.no.length} |`, + `| lingo | ${rows.lingo.full} | 0 | ${ok.length - rows.lingo.full} |`, + `| matho | ${rows.matho.full} | 0 | ${ok.length - rows.matho.full} |`, '', + 'Mojo is a program-for-program translation that always produces something: a line with no faithful spelling becomes a visible `raise Error("not translated: pseudo line N")` and a warning. Lingo and matho must read back as the same program, so what they cannot say exactly they refuse, with the construct and its line, and the pane keeps its last good text.', ''); + const section = (title, rowsIn, intro) => { + out.push(`## ${title}`, '', intro, ''); + if (!rowsIn.length) { out.push('Nothing.', ''); return; } + out.push('| What it has no form for | Programs | Which |', '| --- | ---: | --- |'); + for (const [why, ids] of tally(rowsIn)) out.push(`| ${why.replace(/\|/g, '/')} | ${ids.length} | ${list(ids)} |`); + out.push(''); + }; + section('Mojo: lines that become a visible gap', rows.mojo.gaps, 'These programs translate, with the listed lines left as a `raise Error`.'); + section('Mojo: programs the translator refuses', rows.mojo.no, 'Pseudo the course itself rejects is not translated.'); + section('lingo: what English has no sentence for', rows.lingo.no, 'English is read by `readSteps`, whose grammar has no sentence for these.'); + section('matho: what is not an equation', rows.matho.no, 'An equation lowers to one `FUNCTION` that returns a value; a program with output, text, a loop that does more than add up, or steps outside a function is not one.'); + out.push('## matho: the equation sets', '', + `Each set in \`tests/matho.test.mjs\` is lowered to pseudo by the matho reader, written back as equations by \`pseudoToMatho\`, and checked: ${eqs.filter((e) => !e[1]).length} of ${eqs.length} round-trip.`, '', + '| Equation set | Round-trips | Why not |', '| --- | :---: | --- |'); + for (const [id, why] of eqs) out.push(`| ${id} | ${why ? 'no' : 'yes'} | ${why ?? ''} |`); + out.push(''); + return out.join('\n'); +} + +test('the coverage report is written, and every translator is at or above its floor', () => { + const x = measure(); + assert.ok(x.rows.mojo.full >= FLOOR.mojo, `Mojo says ${x.rows.mojo.full} programs completely, below ${FLOOR.mojo}`); + assert.ok(x.rows.lingo.full >= FLOOR.lingo, `lingo says ${x.rows.lingo.full} programs, below ${FLOOR.lingo}`); + assert.ok(x.rows.matho.full >= FLOOR.matho, `matho says ${x.rows.matho.full} programs, below ${FLOOR.matho}`); + const text = report(x); + fs.mkdirSync(path.dirname(docPath), { recursive: true }); + if (!fs.existsSync(docPath) || fs.readFileSync(docPath, 'utf8') !== text) fs.writeFileSync(docPath, text); + assert.equal(report(measure()), text, 'the report is deterministic'); +}); diff --git a/tests/sync-hub.test.mjs b/tests/sync-hub.test.mjs new file mode 100644 index 0000000..8e10760 --- /dev/null +++ b/tests/sync-hub.test.mjs @@ -0,0 +1,244 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Stage 8: the sync hub (restructure plan, section 2.3), without any UI. Pseudo is the hub; the pane that is +// edited is the authority and is never rewritten; an authority with errors changes nothing; a language that +// cannot say the program goes stale rather than blank; contracts ride along; opening rewrites nothing; and the +// whole cascade is fast enough that typing stays instant. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { SyncHub, LINKED, namesFromMatho } from '../dist/sync/hub.js'; +import { pseudoToLingo } from '../dist/sync/pseudo-to-lingo.js'; +import { checkMojo } from '../dist/mojo/check.js'; +import { studyOf } from '../dist/services/pseudo.js'; +import { EQUATIONS, FIXTURES } from './helpers/sync-programs.mjs'; +import { Workspace } from '../dist/common/workspace.js'; +import { PROBLEMS, problemById, workForProblem } from '../dist/common/problems.js'; + + +const sync = (r) => ({ authority: r.authority, stale: r.stale, names: r.names }); +const next = (r, lang, text) => new SyncHub(r.sources, sync(r)).edit(lang, text); + +const COUNT = `FUNCTION COUNT_BASE(B : CHAR, T : STRING) RETURNS INTEGER + DECLARE Total : INTEGER + Total <- 0 + FOR EACH C IN T + IF C == B THEN + Total <- Total + 1 + ENDIF + NEXT C + RETURN Total +ENDFUNCTION`; +const SIMPLE = `FUNCTION SUMMED(N : INTEGER, X : LIST OF FLOAT) RETURNS FLOAT + DECLARE S : FLOAT + S <- 0.0 + FOR I <- 0 TO N + S <- S + X[I] + NEXT I + RETURN S +ENDFUNCTION`; +const WHILE_PSEUDO = FIXTURES['fixture:div-mod'][0]; + +test('a lingo edit updates pseudo, matho and Mojo, and does not rewrite the lingo', () => { + const mean = new SyncHub({}).edit('matho', EQUATIONS['mean'][0]); + assert.ok(mean.sources.lingo.includes('Start with Total'), 'the equations also became English'); + const lingo = mean.sources.lingo.replace('The answer is 1 / N * Total.', 'The answer is 2 / N * Total.'); + const r = next(mean, 'lingo', lingo); + assert.equal(r.authority, 'lingo'); + assert.equal(r.sources.lingo, lingo, 'the edited pane is exactly what was typed'); + assert.match(r.sources.pseudo, /RETURN 2 \/ N \* Total/); + assert.match(r.sources.matho, /\\frac\{2\}\{n\}|2/, 'matho follows'); + assert.match(r.sources.matho, /\\sum/); + assert.match(r.mojo.source, /Float64\(2\) \/ Float64\(n\) \* total/); + assert.deepEqual(r.stale, {}); + assert.ok(r.changed.includes('pseudo') && r.changed.includes('matho') && !r.changed.includes('lingo')); +}); + +test('a pseudo edit updates lingo', () => { + const r0 = new SyncHub({}).edit('pseudo', SIMPLE); + assert.match(r0.sources.lingo, /Start with S at 0\.0/); + const r = next(r0, 'pseudo', SIMPLE.replace('S <- 0.0', 'S <- 2.0')); + assert.match(r.sources.lingo, /Start with S at 2\.0/); + assert.equal(r.sources.pseudo, SIMPLE.replace('S <- 0.0', 'S <- 2.0')); + assert.equal(r.authority, 'pseudo'); +}); + +test('an edit in X never rewrites X, for every linked language', () => { + const base = new SyncHub({}).edit('pseudo', SIMPLE); + for (const lang of LINKED) { + const text = base.sources[lang]; + if (!text) continue; + const r = new SyncHub(base.sources, sync(base)).edit(lang, text + '\n'); + assert.equal(r.sources[lang], text + '\n', `${lang} is kept as typed, trailing blank line included`); + } +}); + +test('an error in the authority changes nothing: the others keep their last good text and say why', () => { + const ok = new SyncHub({}).edit('pseudo', SIMPLE); + const before = { ...ok.sources }; + const r = next(ok, 'lingo', 'Gibberish that is no sentence of lingo at all.'); + assert.equal(r.sources.lingo, 'Gibberish that is no sentence of lingo at all.'); + for (const l of ['pseudo', 'matho', 'speco']) assert.equal(r.sources[l] ?? '', before[l] ?? '', `${l} kept its text`); + for (const l of ['pseudo', 'matho', 'speco']) assert.match(r.stale[l], /lingo line 1 does not read yet/); + assert.equal(r.stale.lingo, undefined, 'the pane being typed in is not stale'); + // a pseudo with errors does the same + const bad = next(ok, 'pseudo', 'FUNCTION BROKEN(\nRETURN'); + assert.match(bad.stale.lingo, /pseudo line \d+ does not read yet/); + assert.equal(bad.sources.lingo, before.lingo); + // and a good edit afterwards clears every mark + const fixed = next(bad, 'pseudo', SIMPLE); + assert.deepEqual(fixed.stale, {}); +}); + +test('matho goes stale, not blank, on a WHILE; a later expressible program brings it back', () => { + const r0 = new SyncHub({}).edit('matho', EQUATIONS['mean'][0]); + const matho = r0.sources.matho; + const r = next(r0, 'pseudo', WHILE_PSEUDO); + assert.equal(r.sources.matho, matho, 'matho keeps its last good text'); + assert.match(r.stale.matho, /matho has no form for a WHILE loop \(line \d+\)/); + assert.equal(r.stale.lingo, undefined, 'lingo can say a WHILE'); + assert.match(r.sources.lingo, /\S/); + const back = next(r, 'pseudo', SIMPLE); + assert.equal(back.stale.matho, undefined); + assert.notEqual(back.sources.matho, matho); +}); + +test('contracts survive an edit in English or in equations', () => { + const p = problemById('count-base-proved'); + const st = workForProblem(p); + const filled = new SyncHub(st.sources).fill(); + const spec = filled.sources.speco; + assert.match(spec, /ENSURE/); + const r = next({ sources: filled.sources, authority: 'speco', stale: {}, names: {} }, 'lingo', filled.sources.lingo.replace('Add 1 to Total.', 'Add 2 to Total.')); + assert.equal(r.sources.speco, spec, 'the speco is what it was'); + assert.match(r.sources.pseudo, /Total <- Total \+ 2/, 'the edit took'); + assert.match(r.sources.pseudo, /ENSURE RESULT == LENGTH/, 'the contract is woven into the pseudo the English lowered to'); + assert.match(r.sources.pseudo, /INVARIANT/); + assert.match(r.mojo.source, /# ENSURE/, 'the translator carries the contract as a comment'); + // a claim about a routine the English no longer has is kept in the speco, not dropped + const renamed = filled.sources.lingo.replace('To count base for B and T:', 'To tally base for B and T:'); + const r2 = next({ sources: filled.sources, authority: 'speco', stale: {}, names: {} }, 'lingo', renamed); + assert.match(r2.sources.speco, /kept:/); + assert.match(r2.sources.speco, /ENSURE/); +}); + +test('an edit in speco puts its contracts into the pseudo, then lingo and matho follow', () => { + const r0 = new SyncHub({}).edit('pseudo', SIMPLE); + const spec = 'SPEC SUMMED(N, X)\n REQUIRE N >= 0\n'; + const r = next(r0, 'speco', spec); + assert.equal(r.sources.speco, spec); + assert.match(r.sources.pseudo, /REQUIRE N >= 0/); + assert.equal(r.authority, 'speco'); + // replacing it replaces: the old REQUIRE is gone + const r2 = next(r, 'speco', 'SPEC SUMMED(N, X)\n REQUIRE N >= 1\n'); + assert.match(r2.sources.pseudo, /REQUIRE N >= 1/); + assert.doesNotMatch(r2.sources.pseudo, /REQUIRE N >= 0/); + // a speco with a syntax error changes nothing + const bad = next(r, 'speco', 'SPEC SUMMED(N, X\n REQUIRE ??\n'); + assert.equal(bad.sources.pseudo, r.sources.pseudo); + assert.ok(bad.stale.pseudo); +}); + +test('the student\'s spelling of equation names survives a trip through pseudo', () => { + const r0 = new SyncHub({}).edit('matho', '\\overline{x} = \\frac{1}{n} \\sum_{i=0}^{n-1} x_i'); + assert.deepEqual(r0.names, { X_BAR: '\\overline{x}' }); + const r = next(r0, 'pseudo', r0.sources.pseudo.replace('RETURN 1 / N * Total', 'RETURN 2 / N * Total')); + assert.match(r.sources.matho, /^\\overline\{x\} = /m); + assert.deepEqual(namesFromMatho('f(x) = x + 1'), {}, 'a function written with its inputs is spelled by the generator'); +}); + +test('loading an authored exercise changes nothing until the first edit', () => { + for (const p of PROBLEMS) { + const st = workForProblem(p); + const before = JSON.stringify(st.sources); + const filled = new SyncHub(st.sources).fill(); + for (const l of [...LINKED, 'mojo', 'asm']) if ((st.sources[l] ?? '').trim()) assert.equal(filled.sources[l], st.sources[l], `${p.id}: authored ${l} is shown as written`); + assert.equal(JSON.stringify(st.sources), before, `${p.id}: the state's own sources are untouched`); + } + // through the session: reading never writes, and nothing is saved or marked as edited + const p = problemById('count-base'); + const s = new Workspace(workForProblem(p)); + const raw = JSON.stringify(s.state.sources); + assert.match(s.source('lingo'), /\S/, 'a missing pane is shown, generated from the pseudo'); + s.parsedLingo(); s.derivedMojo(); s.lineMaps(); + assert.equal(JSON.stringify(s.state.sources), raw); + assert.equal(s.state.sync, undefined); + assert.equal(s.linkState('lingo').kind, 'authored'); +}); + +test('takeAuthority makes a stale pane the authority and regenerates the others from it', () => { + const s = new Workspace(workForProblem(problemById('count-base'))); + s.edit('matho', 'x = '); // does not read: everything else is stale + assert.equal(s.linkState('pseudo').kind, 'stale'); + s.edit('pseudo', COUNT); + assert.equal(s.state.sync.authority, 'pseudo'); + assert.ok(s.linkState('matho').kind === 'stale' || s.linkState('matho').kind === 'sync'); + s.takeAuthority('lingo'); + assert.equal(s.state.sync.authority, 'lingo'); +}); + +test('line maps compose through the hub', () => { + const r = new SyncHub({}).edit('pseudo', SIMPLE); + // pseudo line 3 is `S <- 0.0`; its lingo and Mojo lines come back through the maps + const lingoLines = SyncHub.relate(r.maps, 'pseudo', 3, 'lingo'); + assert.ok(lingoLines.length, 'a lingo line matches'); + assert.match(r.sources.lingo.split('\n')[lingoLines[0] - 1], /Start with S at 0\.0/); + const mojoLines = SyncHub.relate(r.maps, 'pseudo', 3, 'mojo'); + assert.match(r.mojo.source.split('\n')[mojoLines[0] - 1], /s = 0\.0/); + // from Mojo back to lingo, through the pseudo + const back = SyncHub.relate(r.maps, 'mojo', mojoLines[0], 'lingo'); + assert.deepEqual(back, lingoLines); + // an authority's own lowering is used the other way: lingo line -> pseudo line + const l = new SyncHub({}).edit('lingo', pseudoToLingo(SIMPLE).source); + const lingoLine = l.sources.lingo.split('\n').findIndex((t) => /Start with S at/.test(t)) + 1; + const ps = SyncHub.relate(l.maps, 'lingo', lingoLine, 'pseudo'); + assert.ok(ps.some((n) => /S <- 0\.0/.test(l.sources.pseudo.split('\n')[n - 1])), 'the lingo line maps to the assignment'); + assert.ok(SyncHub.relate(l.maps, 'lingo', lingoLine, 'mojo').length); +}); + +test('line maps stay right when contracts are woven into the pseudo an English edit produced', () => { + const filled = new SyncHub(workForProblem(problemById('count-base-proved')).sources).fill(); + const r = new SyncHub(filled.sources).edit('lingo', filled.sources.lingo); + const lingoLine = r.sources.lingo.split('\n').findIndex((t) => /Add 1 to Total/.test(t)) + 1; + const ps = SyncHub.relate(r.maps, 'lingo', lingoLine, 'pseudo'); + assert.ok(ps.length); + assert.match(r.sources.pseudo.split('\n')[ps[0] - 1], /Total <- Total \+ 1/); +}); + +test('every exercise: an edit in any linked pane leaves the Mojo in step with the pseudo and checking clean', () => { + let n = 0; + for (const p of PROBLEMS) { + const st = workForProblem(p); + if (!(st.sources.pseudo ?? '').trim() || !st.call) continue; + if (!studyOf(st.sources.pseudo).ok) continue; + const r = new SyncHub(st.sources, undefined, { call: st.call }).edit('pseudo', st.sources.pseudo); + assert.equal(r.sources.pseudo, st.sources.pseudo); + assert.match(r.mojo.source, /Generated by Programming\+/, p.id); + n++; + } + assert.ok(n >= 12, `${n} exercises went through the hub`); +}); + +test('timing: a full cascade (lingo edit -> pseudo, matho, speco, Mojo) on the largest exercise stays under 50 ms', () => { + let big = null; + for (const p of PROBLEMS) { + const st = workForProblem(p); + const t = st.sources.pseudo ?? ''; + if (t.trim() && studyOf(t).ok && (!big || t.length > big.pseudo.length)) big = { id: p.id, pseudo: t, st }; + } + assert.ok(big); + const filled = new SyncHub(big.st.sources, undefined, { call: big.st.call }).fill(); + // a lingo that reads: whatever the English generator wrote, or, when it has no English, the smallest program + const lingo = filled.sources.lingo || pseudoToLingo(SIMPLE).source; + const start = filled.sources.lingo ? filled.sources : { lingo }; + const hub = () => new SyncHub(start, undefined, { call: big.st.call }); + hub().edit('lingo', lingo); // warm up the engines + const times = []; + for (let i = 0; i < 7; i++) { + const t0 = performance.now(); + hub().edit('lingo', lingo + (i % 2 ? '' : '\n')); + times.push(performance.now() - t0); + } + times.sort((a, b) => a - b); + const median = times[3]; + assert.ok(median < 50, `${big.id}: the median cascade took ${median.toFixed(1)} ms (limit 50)`); +}); diff --git a/tests/sync-lingo.test.mjs b/tests/sync-lingo.test.mjs new file mode 100644 index 0000000..5e6aae6 --- /dev/null +++ b/tests/sync-lingo.test.mjs @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Stage 5: pseudo -> lingo. What is written must read (readSteps accepts it), must read as the same +// routines, must behave like the original once lowered back to pseudo, and writing it again from that +// pseudo must give the same lingo (a fixed point, comments aside). A program with no English form is +// reported as unsupported with a reason; the translator never writes lingo the reader would reject. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { programs, stripComments } from './helpers/sync-programs.mjs'; +import { studyOf, runPseudo } from '../dist/services/pseudo.js'; +import { readLingo, lowerLingo } from '../dist/services/lingo.js'; +import { pseudoToLingo } from '../dist/sync/pseudo-to-lingo.js'; + +globalThis.localStorage = { getItem() { return null; }, setItem() { }, removeItem() { } }; + +const SUPPORTED_AT_LEAST = 31; // raise as English learns more forms; counted below + +const run = (pseudo, call, input) => { + const st = studyOf(pseudo); + assert.ok(st.ok, 'pseudo should study'); + const t = runPseudo(st, { call, input, maxSteps: 200000 }); + return { out: t.output.map((l) => l.replace(/^\w+ returned /, 'returned ')), err: t.error ? t.error.kind : null }; +}; + +function judge(p) { + if (!studyOf(p.pseudo).ok) return { verdict: 'no-pseudo' }; + const r = pseudoToLingo(p.pseudo); + if ('unsupported' in r) return { verdict: 'unsupported', r }; + const parsed = readLingo(r.source); + if (!parsed.ok) return { verdict: 'unread', detail: parsed.diagnostics.map((d) => d.message).join(' | '), r }; + const low = lowerLingo(parsed, 'pseudo'); + if (!low) return { verdict: 'unlowered', r }; + return { verdict: 'ok', r, low }; +} + +const verdicts = new Map(); +test('every program: the lingo reads, lowers to pseudo that behaves like the original, and is a fixed point; or the program is reported unsupported', () => { + const problems = []; + let supported = 0; + for (const p of programs()) { + const j = judge(p); + verdicts.set(p.id, j); + if (j.verdict === 'no-pseudo' || j.verdict === 'unsupported') continue; + if (j.verdict !== 'ok') { problems.push(`${p.id}: ${j.verdict}: ${j.detail ?? ''}`); continue; } + supported++; + // same behavior: the call (or the input-driven main) gives the same output and the same kind of stop + const want = run(p.pseudo, p.call, p.input); + let got; + try { got = run(j.low.source, p.call, p.input); } catch (e) { problems.push(`${p.id}: lowered pseudo does not study: ${e.message}`); continue; } + if (JSON.stringify(want) !== JSON.stringify(got)) problems.push(`${p.id}: behavior differs: ${JSON.stringify(want).slice(0, 120)} vs ${JSON.stringify(got).slice(0, 120)}`); + // fixed point + const again = pseudoToLingo(j.low.source); + if ('unsupported' in again) { problems.push(`${p.id}: the lowered pseudo is no longer supported`); continue; } + if (stripComments(again.source) !== stripComments(j.r.source)) problems.push(`${p.id}: not a fixed point`); + } + assert.deepEqual(problems, []); + assert.ok(supported >= SUPPORTED_AT_LEAST, `${supported} programs have English, fewer than ${SUPPORTED_AT_LEAST}`); +}); + +test('a program with no English form is unsupported with a located reason, not a crash', () => { + let n = 0; + for (const p of programs()) { + const j = verdicts.get(p.id) ?? judge(p); + if (j.verdict !== 'unsupported') continue; + n++; + assert.ok(j.r.unsupported.length > 0, p.id); + for (const d of j.r.unsupported) { + assert.ok(Number.isInteger(d.line) && d.line >= 1, `${p.id}: line`); + assert.ok(d.message.length > 0, p.id); + } + } + assert.ok(n > 0); +}); + +test('the line map has one entry per output line and points inside the pseudo', () => { + for (const p of programs()) { + const j = verdicts.get(p.id) ?? judge(p); + if (j.verdict !== 'ok') continue; + const lines = j.r.source.replace(/\n$/, '').split('\n'); + assert.equal(j.r.map.length, lines.length, p.id); + const n = p.pseudo.split('\n').length; + for (const m of j.r.map) assert.ok(m === null || (Number.isInteger(m) && m >= 1 && m <= n), `${p.id}: ${m}`); + } +}); + +test('the English is plain: a counting loop and a sum read the way a student would say them', () => { + const r = pseudoToLingo(`FUNCTION SUM_TO(N : INTEGER) RETURNS INTEGER + DECLARE Total : INTEGER + Total <- 0 + FOR I <- 1 TO N + Total <- Total + I + NEXT I + RETURN Total +ENDFUNCTION +`); + assert.ok(!('unsupported' in r), JSON.stringify(r)); + assert.ok(readLingo(r.source).ok, r.source); + assert.ok(/Total/.test(r.source)); +}); + +test('pseudo that does not parse is unsupported, and records have no English form', () => { + const bad = pseudoToLingo('FUNCTION F(\n'); + assert.ok('unsupported' in bad && bad.unsupported.length > 0); + const rec = pseudoToLingo(`RECORD P + X : INTEGER +ENDRECORD +FUNCTION F(A : INTEGER) RETURNS INTEGER + DECLARE R : P + R.X <- A + RETURN R.X +ENDFUNCTION +`); + assert.ok('unsupported' in rec, 'records are left unsupported by design'); +}); diff --git a/tests/sync-matho.test.mjs b/tests/sync-matho.test.mjs new file mode 100644 index 0000000..755f688 --- /dev/null +++ b/tests/sync-matho.test.mjs @@ -0,0 +1,236 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Stage 6: pseudo -> matho. For the programs that are equations, the generated LaTeX must read, lower back to +// pseudo that behaves like the original, and be a fixed point; every other program is refused with the construct +// and its line. The equation sets are the ones tests/matho.test.mjs checks by hand-worked values. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { programs, EQUATIONS } from './helpers/sync-programs.mjs'; +import { studyOf, runPseudo } from '../dist/services/pseudo.js'; +import { parseMatho } from '../dist/services/matho.js'; +import { pseudoToMatho } from '../dist/sync/pseudo-to-matho.js'; + +globalThis.localStorage = { getItem() { return null; }, setItem() { }, removeItem() { } }; + +/** + * Equations the reader lowers but this generator does not write back, and why. An equation leaves this list + * when it round-trips; the test fails if a listed equation does, so the list can only shrink. + */ +const KNOWN_GAPS = { + 'matrix-row': 'the row index i is an input that only subscripts, and the reader recognizes that only for a left side written y_i, with the function name carrying the subscript', + 'choose': 'the reader names the second factorial counter K2 (K is the input), and a loop counter has to be a single letter in matho', +}; + +const SUPPORTED_AT_LEAST = 24; // raise as matho learns more forms; counted below + +const num = (line) => { const m = /returned (.*)$/.exec(line ?? ''); return m ? m[1] : null; }; +function run(pseudo, call, input = []) { + const st = studyOf(pseudo); + assert.ok(st.ok, `pseudo should study: ${st.diagnostics.filter((d) => d.severity === 'error').map((d) => d.message).join(' | ')}`); + const t = runPseudo(st, { call, input, maxSteps: 200000 }); + return { value: t.error ? `error:${t.error.kind}` : num(t.output[t.output.length - 1]), out: t.error ? [] : t.output.slice(0, -1) }; +} +const same = (a, b) => a.value === b.value || (!Number.isNaN(Number(a.value)) && !Number.isNaN(Number(b.value)) && Math.abs(Number(a.value) - Number(b.value)) <= 1e-9 * Math.max(1, Math.abs(Number(a.value)))) && JSON.stringify(a.out) === JSON.stringify(b.out); +const lowerPseudo = (latex) => { const r = parseMatho(latex); assert.ok(r.ok, `the generated equations should read: ${latex}\n${r.diagnostics.map((d) => d.message).join(' | ')}`); return r.program.pseudo; }; + +const results = new Map(); +test('every equation set: the LaTeX reads, lowers to pseudo that behaves like the original, and is a fixed point; or it is a listed gap', () => { + const problems = []; + let ok = 0; + for (const [id, [src, call]] of Object.entries(EQUATIONS)) { + const first = parseMatho(src); + assert.ok(first.ok, `${id}: the fixture should lower`); + const pseudo = first.program.pseudo; + const m = pseudoToMatho(pseudo); + results.set(id, m); + const known = KNOWN_GAPS[id]; + if ('unsupported' in m) { + if (!known) problems.push(`${id}: ${m.unsupported.map((d) => d.message).join(' | ')}`); + continue; + } + if (known) { problems.push(`${id}: now round-trips; take it off KNOWN_GAPS (${known})`); continue; } + ok++; + // reads, and same behavior: the original equations' pseudo and the pseudo of the written equations + let again; + try { again = lowerPseudo(m.source); } catch (e) { problems.push(`${id}: ${e.message}`); continue; } + const a = run(pseudo, call); const b = run(again, call); + if (!same(a, b)) problems.push(`${id}: behavior differs: ${JSON.stringify(a)} vs ${JSON.stringify(b)}`); + // fixed point + const m2 = pseudoToMatho(again); + if ('unsupported' in m2) { problems.push(`${id}: the lowered pseudo is no longer supported: ${m2.unsupported[0].message}`); continue; } + if (m2.source !== m.source) problems.push(`${id}: not a fixed point:\n${m.source}\nvs\n${m2.source}`); + } + assert.deepEqual(problems, []); + assert.ok(ok >= SUPPORTED_AT_LEAST, `${ok} equation sets round-trip, fewer than ${SUPPORTED_AT_LEAST}`); +}); + +test('every exercise and corpus program: accepted and checked the same way, or refused with a reason and a line', () => { + let accepted = 0; + for (const p of programs()) { + if (!studyOf(p.pseudo).ok) continue; + const m = pseudoToMatho(p.pseudo); + if ('unsupported' in m) { + assert.ok(m.unsupported.length > 0, p.id); + for (const d of m.unsupported) { + assert.ok(Number.isInteger(d.line) && d.line >= 1 && d.line <= p.pseudo.split('\n').length, `${p.id}: line ${d.line}`); + assert.ok(d.message.length > 20, p.id); + } + continue; + } + accepted++; + const again = lowerPseudo(m.source); + if (p.call) assert.ok(same(run(p.pseudo, p.call), run(again, p.call)), `${p.id}: behavior differs`); + const m2 = pseudoToMatho(again); + assert.ok(!('unsupported' in m2) && m2.source === m.source, `${p.id}: not a fixed point`); + } + assert.ok(accepted >= 3, `${accepted} exercise, corpus, and fixture programs are equations, fewer than 3`); +}); + +test('a program that is not an equation is refused with the construct and its line', () => { + const ask = (pseudo) => { const r = pseudoToMatho(pseudo); assert.ok('unsupported' in r, pseudo); return r.unsupported.map((d) => d.message).join(' | '); }; + assert.match(ask(`FUNCTION COUNT_DOWN(N : INTEGER) RETURNS INTEGER + DECLARE K : INTEGER + K <- N + WHILE K > 0 + K <- K - 1 + ENDWHILE + RETURN K +ENDFUNCTION +`), /matho has no form for a WHILE loop \(line 4\)/); + assert.match(ask(`FUNCTION F(N : INTEGER) RETURNS INTEGER + OUTPUT N + RETURN N +ENDFUNCTION +`), /OUTPUT.*\(line 2\)/); + assert.match(ask(`FUNCTION SHOUT(T : STRING) RETURNS STRING + RETURN T +ENDFUNCTION +`), /(answers STRING|not a number).*\(line 1\)/); + assert.match(ask(`FUNCTION F(N : INTEGER) RETURNS INTEGER + IF N > 0 THEN + IF N > 1 THEN + RETURN 2 + ELSE + RETURN 1 + ENDIF + ELSE + RETURN 0 + ENDIF +ENDFUNCTION +`), /nested IF.*\(line 3\)/); + assert.match(ask('OUTPUT 1 + 2\n'), /steps outside a function/); + assert.match(ask('FUNCTION F(\n'), /./); +}); + +test('names: SIGMA is \\sigma, X_BAR is \\bar{x}, a single letter is lower case, and a hint restores the student\'s own spelling', () => { + const pseudo = `FUNCTION X_BAR(N : INTEGER, X : LIST OF FLOAT) RETURNS FLOAT + DECLARE Total : FLOAT + Total <- 0.0 + FOR I <- 0 TO N + Total <- Total + X[I] + NEXT I + RETURN 1 / N * Total +ENDFUNCTION + +FUNCTION SIGMA(V : FLOAT) RETURNS FLOAT + RETURN V * V +ENDFUNCTION +`; + const plain = pseudoToMatho(pseudo); + assert.ok(!('unsupported' in plain), JSON.stringify(plain)); + assert.match(plain.source, /^\\bar\{x\} = /m); + assert.match(plain.source, /^\\sigma = /m); + assert.match(plain.source, /x_\{i\}/); + const hinted = pseudoToMatho(pseudo, { names: { X_BAR: '\\overline{x}' } }); + assert.ok(!('unsupported' in hinted), JSON.stringify(hinted)); + assert.match(hinted.source, /^\\overline\{x\} = /m); + assert.equal(parseMatho(hinted.source).program.defs[0].name, 'X_BAR'); +}); + +test('a hint that does not lower back to the function is not trusted', () => { + const r = pseudoToMatho(`FUNCTION SQUARE(V : FLOAT) RETURNS FLOAT + RETURN V * V +ENDFUNCTION +`, { names: { SQUARE: 'q' } }); + assert.ok('unsupported' in r); + assert.match(r.unsupported[0].message, /SQUARE comes back as/); +}); + +test('the line map has one entry per output line and points inside the pseudo', () => { + for (const [id, [src]] of Object.entries(EQUATIONS)) { + const m = results.get(id) ?? pseudoToMatho(parseMatho(src).program.pseudo); + if ('unsupported' in m) continue; + const lines = m.source.replace(/\n$/, '').split('\n'); + assert.equal(m.map.length, lines.length, id); + const n = parseMatho(src).program.pseudo.split('\n').length; + for (const x of m.map) assert.ok(x === null || (Number.isInteger(x) && x >= 1 && x <= n), `${id}: ${x}`); + } +}); + +test('a sum from 1 reads x_i for the pseudo\'s X[I - 1], and a sum from 0 reads x_i for X[I]', () => { + const from1 = pseudoToMatho(parseMatho(EQUATIONS['sum-from-one'][0]).program.pseudo); + assert.match(from1.source, /\\sum_\{i=1\}\^\{n\} x_\{i\}/); + const from0 = pseudoToMatho(parseMatho(EQUATIONS['mean'][0]).program.pseudo); + assert.match(from0.source, /\\sum_\{i=0\}\^\{n-1\} x_\{i\}/); + // L[I] in a loop the reader would shift is not an equation it can say + const r = pseudoToMatho(`FUNCTION S(N : INTEGER, L : LIST OF FLOAT) RETURNS FLOAT + DECLARE Total : FLOAT + Total <- 0.0 + FOR I <- 1 TO N + 1 + Total <- Total + L[I] + NEXT I + RETURN Total +ENDFUNCTION +`); + assert.ok('unsupported' in r); + assert.match(r.unsupported[0].message, /starts at 1/); +}); + +/** Pseudo a student wrote by hand (not lowered from matho) that is still an equation: the same three checks. */ +const HAND = { + 'whole-number-types': [`FUNCTION AREA(W : INTEGER, H : INTEGER) RETURNS INTEGER + RETURN W * H +ENDFUNCTION +`, 'AREA(3, 4)'], + 'elseif-chain': [`FUNCTION SIGN(X : FLOAT) RETURNS INTEGER + IF X < 0 THEN + RETURN -1 + ELSEIF X == 0 THEN + RETURN 0 + ENDIF + RETURN 1 +ENDFUNCTION +`, 'SIGN(-2.5)'], + 'sum-of-squares-to-n': [`FUNCTION SQUARES(N : INTEGER) RETURNS INTEGER + DECLARE Total : INTEGER + Total <- 0 + FOR K <- 1 TO N + 1 + Total <- Total + K * K + NEXT K + RETURN Total +ENDFUNCTION +`, 'SQUARES(4)'], + 'mean-of-a-list': [`FUNCTION MEAN(L : LIST OF FLOAT) RETURNS FLOAT + DECLARE Total : FLOAT + Total <- 0.0 + FOR I <- 0 TO LENGTH(L) + Total <- Total + L[I] + NEXT I + RETURN Total / LENGTH(L) +ENDFUNCTION +`, 'MEAN([1.0, 2.0, 6.0])'], +}; + +test('pseudo written by hand that is an equation: accepted when matho can say it, and held to the same three checks', () => { + const said = []; + for (const [id, [pseudo, call]] of Object.entries(HAND)) { + const m = pseudoToMatho(pseudo); + if ('unsupported' in m) { assert.equal(id, 'mean-of-a-list', `${id}: ${m.unsupported[0].message}`); continue; } + said.push(id); + const again = lowerPseudo(m.source); + assert.ok(same(run(pseudo, call), run(again, call)), `${id}: ${JSON.stringify(run(pseudo, call))} vs ${JSON.stringify(run(again, call))}`); + const m2 = pseudoToMatho(again); + assert.ok(!('unsupported' in m2) && m2.source === m.source, `${id}: not a fixed point`); + } + assert.deepEqual(said, ['whole-number-types', 'elseif-chain', 'sum-of-squares-to-n']); +}); diff --git a/tests/sync-mojo.test.mjs b/tests/sync-mojo.test.mjs new file mode 100644 index 0000000..9be21dc --- /dev/null +++ b/tests/sync-mojo.test.mjs @@ -0,0 +1,526 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Stage 4: pseudo -> Mojo. The generated Mojo must be accepted by the checker, run in the interpreter, and +// print what the pseudo prints; its style is pinned by a snapshot per exercise; every standard-library name +// it writes is in the harvested index. Programs it cannot yet hold to that are listed below, each with its reason. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { EXERCISES } from '../dist/exercises/index.js'; +import { studyOf, runPseudo } from '../dist/services/pseudo.js'; +import { checkMojo } from '../dist/mojo/check.js'; +import { runMojo } from '../dist/mojo/interp.js'; +import { pseudoToMojo, GENERATED_HEADER, USED_STDLIB, USED_PRELUDE } from '../dist/sync/pseudo-to-mojo.js'; +import { formatAsMojo, sameOutput, outputDistance } from '../dist/sync/compare.js'; +import { MOJO_STD_EXPORTS } from '../dist/mojo/generated/stdlib.js'; +import { MOJO_STD_STRUCTS } from '../dist/mojo/generated/types.js'; + +globalThis.localStorage = { getItem() { return null; }, setItem() { }, removeItem() { } }; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const corpusDir = path.join(here, '..', 'library', 'pseudo', 'programs'); +const snapDir = path.join(here, 'snapshots', 'sync-mojo'); + +/** + * Programs that do not (yet) get the full check, and why. A program leaves this list when it passes; + * the test fails if a listed program passes, so the list can only shrink. + * declined: the Mojo interpreter declines the construct (it declines what it has not been verified + * against godbolt on); the translation is still checked, and the run is not. + * differs: the notation and Mojo disagree by design, and a literal translation keeps Mojo's behavior. + * checker: the checker's error is the right answer for what the pseudo says. + */ +const KNOWN_GAPS = { + '23-dice.pseudo': ['declined', 'random_si64 draws from Mojo\'s own generator, whose numbers the interpreter cannot know'], + '42-numerics.pseudo': ['declined', 'sin is Mojo\'s own routine (or the C library\'s), not reproduced digit for digit'], + '46-projectile.pseudo': ['declined', 'cos is Mojo\'s own routine (or the C library\'s), not reproduced digit for digit'], + '47-binomial-and-ph.pseudo': ['declined', 'log10 is Mojo\'s own routine, not reproduced digit for digit'], + '33-stops-div.pseudo': ['differs', 'dividing by zero stops the notation; a Float64 division in Mojo gives nan'], + '44-parallel-race.pseudo': ['differs', 'the notation detects a data race in PARALLEL FOR; Mojo runs the passes in order and prints a result'], + '45-parallel-neighbor-race.pseudo': ['differs', 'the notation detects a data race in PARALLEL FOR; Mojo runs the passes in order and prints a result'], + '50-float32.pseudo': ['gap', 'EXACT has no Mojo spelling: Mojo prints the shortest decimal that rounds back, never the exact value'], + '51-ieee.pseudo': ['declined', 'log is Mojo\'s own routine, not reproduced digit for digit'], + '52-elementary.pseudo': ['declined', 'cbrt and the other transcendental functions are Mojo\'s own routines, which the interpreter declines on purpose (docs/trust.md); hypot it runs'], + '35-stops-unset.pseudo': ['checker', 'the pseudo reads a variable that was never given a value; the checker reports exactly that, as the compiler would'], + '36-stops-counter.pseudo': ['checker', 'the pseudo reads a FOR counter after its loop, where it has no value; the checker reports exactly that'], +}; + +/** Programs whose pseudo does not parse, by design. The translator must say so, not crash. */ +const NO_PSEUDO = new Set(['ex:five-bugs', '37-static-errors.pseudo', '41-reserved-name.pseudo']); + +const AGREE_AT_LEAST = 57; // raise as the interpreter learns more; counted below + +function programs() { + const out = []; + for (const e of EXERCISES) if (e.sources.pseudo) out.push({ id: `ex:${e.id}`, name: e.id, pseudo: e.sources.pseudo, call: e.call ?? null, input: [] }); + for (const f of fs.readdirSync(corpusDir).filter((x) => x.endsWith('.pseudo')).sort()) { + const src = fs.readFileSync(path.join(corpusDir, f), 'utf8'); + out.push({ id: f, name: f, pseudo: src, call: null, input: [...src.matchAll(/^\/\/ INPUT: (.*)$/gm)].map((m) => m[1]) }); + } + return out; +} + +/** How one program fares: what the translator, the checker, the interpreter, and the pseudo run say. */ +function judge(p) { + const st = studyOf(p.pseudo); + if (!st.ok) return { verdict: 'no-pseudo', result: pseudoToMojo(p.pseudo, { call: p.call }) }; + const tr = runPseudo(st, { call: p.call, input: p.input, maxSteps: 200000 }); + const result = pseudoToMojo(p.pseudo, { call: p.call }); + assert.ok(!('unsupported' in result), `${p.id}: ${JSON.stringify(result.unsupported)}`); + const gaps = result.notes.filter((n) => n.id === 'translate-gap'); + const checked = checkMojo(result.source); + const errors = checked.diagnostics.filter((d) => d.severity === 'error'); + if (gaps.length) return { verdict: 'gap', detail: gaps.map((g) => `line ${g.line}: ${g.message}`).join('; '), result }; + if (errors.length) return { verdict: 'checker', detail: errors.map((d) => `L${d.line} ${d.message}`).join(' | '), result }; + const run = runMojo(checked.module, result.source, { input: p.input, maxSteps: 400000 }); + if (run.refused) return { verdict: 'declined', detail: run.refused, result }; + const stops = run.exitCode !== 0 || run.limited; + if (tr.error) return stops ? { verdict: 'agree', result } : { verdict: 'differs', detail: `pseudo stops (${tr.error.kind}); Mojo ran to the end and printed ${JSON.stringify(run.stdout.slice(0, 80))}`, result }; + if (stops) return { verdict: 'differs', detail: `Mojo stopped (exit ${run.exitCode}); pseudo printed ${JSON.stringify(tr.output.slice(0, 2))}`, result }; + const want = p.call && tr.raw.result !== undefined + ? [...tr.raw.output, formatAsMojo(tr.raw.result)].join('\n') + : tr.raw.output.join('\n'); + return sameOutput(want, run.stdout) + ? { verdict: 'agree', result } + : { verdict: 'differs', detail: `pseudo printed ${JSON.stringify(want.slice(0, 100))}, Mojo printed ${JSON.stringify(run.stdout.slice(0, 100))}`, result }; +} + +const verdicts = new Map(); +test('every exercise and corpus program: the generated Mojo checks, runs, and agrees with the pseudo, or is a listed gap', () => { + const unexpected = []; + let agree = 0; + for (const p of programs()) { + const j = judge(p); + verdicts.set(p.id, j); + if (j.verdict === 'no-pseudo') { + if (!NO_PSEUDO.has(p.id)) unexpected.push(`${p.id}: the pseudo does not parse, and the program is not listed in NO_PSEUDO`); + continue; + } + if (NO_PSEUDO.has(p.id)) { unexpected.push(`${p.id}: listed in NO_PSEUDO but its pseudo parses`); continue; } + const known = KNOWN_GAPS[p.id]; + if (j.verdict === 'agree') { + agree++; + if (known) unexpected.push(`${p.id}: now agrees; take it off KNOWN_GAPS (${known[1]})`); + continue; + } + if (!known) { unexpected.push(`${p.id}: ${j.verdict}: ${j.detail}`); continue; } + if (known[0] !== j.verdict) unexpected.push(`${p.id}: listed as ${known[0]} but is ${j.verdict}: ${j.detail}`); + } + assert.deepEqual(unexpected, []); + assert.ok(agree >= AGREE_AT_LEAST, `${agree} programs agree, fewer than ${AGREE_AT_LEAST}`); +}); + +test('the pseudo that does not parse gets a reason, not a crash', () => { + for (const id of NO_PSEUDO) { + const p = programs().find((x) => x.id === id); + const r = pseudoToMojo(p.pseudo, { call: p.call }); + assert.ok('unsupported' in r, id); + assert.ok(r.unsupported.length > 0 && r.unsupported.every((d) => d.severity === 'error'), id); + } +}); + +test('every exercise with pseudo has its generated Mojo pinned by a snapshot (UPDATE_SNAPSHOTS=1 rewrites them for review)', () => { + fs.mkdirSync(snapDir, { recursive: true }); + const wanted = new Set(); + for (const p of programs().filter((x) => x.id.startsWith('ex:'))) { + const r = pseudoToMojo(p.pseudo, { call: p.call }); + if ('unsupported' in r) continue; // five-bugs: its pseudo has mistakes by design + const file = path.join(snapDir, `${p.name}.mojo`); + wanted.add(`${p.name}.mojo`); + if (process.env.UPDATE_SNAPSHOTS) fs.writeFileSync(file, r.source); + assert.ok(fs.existsSync(file), `no snapshot for ${p.name}; run with UPDATE_SNAPSHOTS=1 and review the new file`); + assert.equal(r.source, fs.readFileSync(file, 'utf8'), `the generated Mojo for ${p.name} changed; if the change is wanted, run with UPDATE_SNAPSHOTS=1 and review the diff`); + } + for (const f of fs.readdirSync(snapDir)) assert.ok(wanted.has(f), `snapshot ${f} belongs to no exercise`); +}); + +test('the translation is literal: the declare-then-assign, index-loop, and total = total + 1 habits are kept', () => { + const ex = EXERCISES.find((e) => e.id === 'count-base'); + const r = pseudoToMojo(ex.sources.pseudo, { call: ex.call }); + for (const line of [' var n: Int', ' var total: Int', ' n = t.count_codepoints()', ' total = 0', ' for i in range(n):', ' if String(t[codepoint=i]) == b:', ' total = total + 1', ' return total']) { + assert.ok(r.source.split('\n').includes(line), `missing: ${line}`); + } + assert.ok(!r.source.includes('+='), 'the translator never rewrites total = total + 1'); +}); + +test('the translator does not import or mention the best-practices helper', () => { + const text = fs.readFileSync(path.join(here, '..', 'src', 'sync', 'pseudo-to-mojo.ts'), 'utf8'); + assert.ok(!/practices/i.test(text), 'src/sync/pseudo-to-mojo.ts refers to the helper'); +}); + +test('every generated source starts with the generated-code header and has a line map that fits', () => { + for (const p of programs()) { + const r = pseudoToMojo(p.pseudo, { call: p.call }); + if ('unsupported' in r) continue; + const lines = r.source.replace(/\n$/, '').split('\n'); + assert.equal(lines[0], GENERATED_HEADER, p.id); + assert.equal(r.map.length, lines.length, `${p.id}: one map entry per line`); + assert.equal(r.map[0], null, p.id); + const pseudoLines = p.pseudo.split('\n').length; + for (const m of r.map) assert.ok(m === null || (Number.isInteger(m) && m >= 1 && m <= pseudoLines), `${p.id}: map entry ${m}`); + } +}); + +test('the map links Mojo lines to the pseudo lines they came from', () => { + const ex = EXERCISES.find((e) => e.id === 'count-base'); + const r = pseudoToMojo(ex.sources.pseudo, { call: ex.call }); + const lines = r.source.split('\n'); + const pseudo = ex.sources.pseudo.split('\n'); + const at = lines.findIndex((l) => l.includes('total = total + 1')); + assert.ok(pseudo[r.map[at] - 1].includes('Total <- Total + 1'), pseudo[r.map[at] - 1]); +}); + +test('grounding: every standard-library name the translator writes is in the harvested index, with the module it imports from', () => { + const structs = MOJO_STD_STRUCTS; + const exported = (n) => MOJO_STD_EXPORTS[n] ?? (structs[n] ? 'std' : undefined); + for (const [name, mod] of Object.entries(USED_STDLIB)) assert.equal(MOJO_STD_EXPORTS[name], mod, `${name} should come from ${mod}`); + // names the index leaves out are named here, with why: Float64 is a SIMD alias, abs is exported from two modules + const NOT_INDEXED = { Float64: 'an alias of SIMD', abs: 'exported from two modules, so the index leaves it out', Int: 'a builtin type' }; + for (const name of USED_PRELUDE) assert.ok(exported(name) || NOT_INDEXED[name], `${name} is not in the harvested index`); +}); + +test('every import in a generated source is one the translator knows, and no import is unused', () => { + for (const p of programs()) { + const r = pseudoToMojo(p.pseudo, { call: p.call }); + if ('unsupported' in r) continue; + const body = r.source.split('\n').filter((l) => !l.startsWith('from ')).join('\n'); + for (const m of r.source.matchAll(/^from (\S+) import (.+)$/gm)) { + for (const name of m[2].split(', ')) { + assert.equal(USED_STDLIB[name], m[1], `${p.id}: ${m[0]}`); + assert.ok(new RegExp(`\\b${name}\\b`).test(body), `${p.id}: ${name} is imported and never used`); + } + } + } +}); + +test('main is written from the call: literals follow the parameter types (TRUE, character literals, floats, lists)', () => { + const pseudo = `FUNCTION F(B : BOOL, C : CHAR, X : FLOAT, L : LIST OF INTEGER) RETURNS BOOL + RETURN B AND C == 'A' AND X > 0.5 AND LENGTH(L) > 1 +ENDFUNCTION +`; + const r = pseudoToMojo(pseudo, { call: "F(TRUE, 'A', 1, [3, 4])" }); + // a list argument is written in brackets and takes the parameter's type; List[Int](3, 4) is not a constructor (pixi run mojo-local) + assert.ok(r.source.includes(' print(f(True, "A", 1.0, [3, 4]))'), r.source); + assert.equal(checkMojo(r.source).diagnostics.filter((d) => d.severity === 'error').length, 0); + const run = runMojo(checkMojo(r.source).module, r.source, {}); + assert.equal(run.stdout, 'True\n'); +}); + +/** + * Spellings the pinned compiler asked for. Each was Mojo the checker and the interpreter accepted and the compiler + * rejected, or Mojo that printed something else than the pseudo; `pixi run mojo-local` and `pixi run mojo-fuzz` + * found them and hold them. Here the text itself is pinned, so a change to it is seen without a compiler. + */ +const mojoOf = (pseudo, call = null) => { const r = pseudoToMojo(pseudo, { call }); assert.ok(!('unsupported' in r), JSON.stringify(r.unsupported)); return r; }; +const has = (r, ...lines) => { for (const l of lines) assert.ok(r.source.split('\n').some((x) => x.trim() === l), `missing: ${l}\n${r.source}`); }; +const gaps = (r) => r.notes.filter((n) => n.id === 'translate-gap'); + +test('a list in brackets takes its type from where it lands; where nothing gives it one, it is named first', () => { + const r = mojoOf('DECLARE L : LIST OF FLOAT <- [1.5, 2]\nDECLARE G : LIST OF LIST OF INTEGER\nDECLARE N : INTEGER <- 3\nG <- [[1, 2], [N]]\nL <- [N, 0.5]\nOUTPUT LENGTH([4, 5, 6]), " ", SUM_OF([N, N])\nWHILE LENGTH([1, 2]) > N\n N <- N + 1\nENDWHILE\nOUTPUT L, G'); + has(r, 'var l: List[Float64] = [1.5, 2]', 'g = [[1, 2], [n]]', 'l = [Float64(n), 0.5]', 'var _list1: List[Int] = [4, 5, 6]', 'var _list2: List[Int] = [n, n]'); + assert.ok(!/List\[\w+\]\([^)]/.test(r.source), `List[T](a, b) is not a constructor:\n${r.source}`); + // a list of literals cannot change between two checks of a loop condition, so it may be named ahead of the loop + assert.deepEqual(gaps(r), []); + assert.equal(checkMojo(r.source).diagnostics.filter((d) => d.severity === 'error').length, 0); + assert.equal(runMojo(checkMojo(r.source).module, r.source, {}).stdout, '3 6\n[3.0, 0.5][[1, 2], [3]]\n'); +}); + +test('numbers of both kinds meet the way Mojo lets them: a literal on the left is written in the other side\'s type', () => { + const r = mojoOf('DECLARE X : FLOAT <- 95.5\nDECLARE S : FLOAT32 <- 0.1\nDECLARE N : INTEGER <- 3\nOUTPUT 90 <= X < 101, " ", 1 == X, " ", -1 < X, " ", 2 ^ X, " ", (-2) ^ 8, " ", (X ^ 2) ^ 3, " ", X ^ 2 ^ 3\nOUTPUT 0.1 == S, " ", N < X <= 100, " ", N IN [0, 5), " ", MIN(100, 1e10), " ", MAX(N, X)\nCASE OF X\n 0 TO 50: OUTPUT "low"\n OTHERWISE: OUTPUT "high"\nENDCASE'); + const text = r.source; + for (const piece of ['90.0 <= x < 101.0', '1.0 == x', '-1.0 < x', '2.0 ** x', '(-2) ** 8', '(x ** 2) ** 3', 'x ** Float64(2 ** 3)', 'Float32(0.1) == s', 'Float64(n) < x <= 100.0', '0 <= n < 5', 'min(100.0, 10000000000.0)', 'max(Float64(n), x)', 'if 0.0 <= x < 50.0:']) { + assert.ok(text.includes(piece), `missing: ${piece}\n${text}`); + } +}); + +test('ROUND spells out its halves, INT of a number known at compile time goes through Float64, and -0.0 keeps its sign', () => { + const r = mojoOf('DECLARE X : FLOAT <- 2.5\nOUTPUT ROUND(X), " ", ROUND(7), " ", INT(0.1), " ", INT(PI()), " ", INT(X), " ", COPYSIGN(2.0, -0.0)'); + has(r, 'var _x1 = x', 'var _rounded2 = Int(_x1)', 'if _x1 - Float64(_rounded2) >= 0.5:', '_rounded2 += 1', 'elif _x1 - Float64(_rounded2) <= -0.5:', '_rounded2 -= 1'); + for (const piece of ['Int(Float64(0.1))', 'Int(Float64(pi))', 'Int(x)', 'copysign(Float64(2.0), Float64(-0.0))']) assert.ok(r.source.includes(piece), `missing: ${piece}\n${r.source}`); + assert.ok(!r.source.includes('round('), 'Mojo\'s round() takes a half to the even neighbor'); + assert.ok(r.notes.some((n) => n.id === 'translate-round')); +}); + +test('MIN and MAX of a list are loops, a BOOL made into text is TRUE or FALSE, and brackets survive where Mojo binds tighter', () => { + const r = mojoOf('DECLARE L : LIST OF INTEGER <- [4, 1, 9]\nDECLARE S : STRING <- "ab"\nDECLARE P : BOOL <- TRUE\nOUTPUT MIN(L), " ", MAX(L), " ", TO_STRING(P), " ", LENGTH(S & "c"), " ", BIT_NOT(5 MOD 3), " ", UPPER(S & S), " ", REPLACE("a,b" & "c", ",", S)'); + has(r, 'var _least2 = l[0]', 'if _x1 < _least2:', 'var _most4 = l[0]', 'if _x3 > _most4:'); + for (const piece of ['String("TRUE" if p else "FALSE")', '(s + "c").count_codepoints()', '~(5 % 3)', '(s + s).upper()', 'String("a,b" + "c").replace(",", s)']) assert.ok(r.source.includes(piece), `missing: ${piece}\n${r.source}`); + // ** binds tighter than ~ as it does than a minus sign + assert.ok(mojoOf('DECLARE N : INTEGER <- 5\nOUTPUT BIT_NOT(N) ^ 3').source.includes('(~n) ** 3')); +}); + +test('a variable handed to a BYREF parameter before it has a value starts at zero; one that has a value is left as declared', () => { + const r = mojoOf('PROCEDURE Half(N : INTEGER, BYREF Out : INTEGER)\n Out <- N DIV 2\nENDPROCEDURE\nDECLARE A : INTEGER\nDECLARE B : INTEGER\nB <- 1\nCALL Half(10, A)\nCALL Half(4, B)\nOUTPUT A, " ", B'); + has(r, 'var a: Int = 0', 'var b: Int', 'b = 1', 'half(10, a)'); + assert.ok(r.notes.some((n) => n.id === 'translate-out-argument')); + assert.equal(runMojo(checkMojo(r.source).module, r.source, {}).stdout, '5 2\n'); +}); + +test('a record or a list put into a list is a copy of its own', () => { + const r = mojoOf('TYPE Point\n DECLARE X : INTEGER\nENDTYPE\nDECLARE P : Point\nDECLARE Ps : LIST OF Point\nDECLARE Row : LIST OF INTEGER <- [1]\nDECLARE Grid : LIST OF LIST OF INTEGER\nP.X <- 3\nAPPEND(Ps, P)\nAPPEND(Grid, Row)\nINSERT(Grid, 0, Row)\nOUTPUT LENGTH(Ps), LENGTH(Grid)'); + has(r, 'ps.append(p.copy())', 'grid.append(row.copy())', 'grid.insert(0, row.copy())'); +}); + +test('what the notation computes only sometimes is never written ahead of the line: the right of AND and OR, the end of a chain, a CASE label', () => { + const guard = mojoOf('DECLARE L : LIST OF FLOAT\nDECLARE N : INTEGER <- 0\nIF N > 0 AND MEAN_OF(L) > 2.0 THEN\n OUTPUT "big"\nENDIF\nOUTPUT "done"'); + assert.equal(gaps(guard).length, 1, 'MEAN_OF is a loop; ahead of the IF it would run where the notation skips it'); + assert.equal(gaps(guard)[0].line, 3); + const left = mojoOf('DECLARE L : LIST OF FLOAT <- [1.0, 5.0]\nIF MEAN_OF(L) > 2.0 AND LENGTH(L) > 1 THEN\n OUTPUT "big"\nENDIF'); + assert.deepEqual(gaps(left), [], 'the left side is always computed, so its loop may be written first'); + const chain = mojoOf('DECLARE L : LIST OF FLOAT\nDECLARE X : FLOAT <- 1.0\nOUTPUT 2.0 <= X <= MEAN_OF(L)'); + assert.equal(gaps(chain).length, 1); + const range = mojoOf('DECLARE X : FLOAT <- 0.3\nOUTPUT X IN [0, 12)'); + assert.equal(gaps(range).length, 1, 'a range holds whole numbers: 0.3 IN [0, 12) is FALSE, and 0.0 <= x < 12.0 would say True'); +}); + +test('outputDistance tells a rounding difference from another answer', () => { + assert.equal(outputDistance('3 TRUE [1, 2]', '3.0 True [1, 2]'), 0); + assert.equal(outputDistance('infinity -infinity not a number', 'inf -inf nan'), 0); + const far = outputDistance('x 0.34641016151377535', 'x 0.3464101615137754'); + assert.ok(far > 0 && far < 1e-15, String(far)); + assert.equal(outputDistance('1.0 2.0', '1.0'), Infinity, 'another count of numbers'); + assert.equal(outputDistance('yes 1.0', 'no 1.0'), Infinity, 'other text'); + assert.ok(outputDistance('-256', '256') > 1); + assert.equal(outputDistance('0.0', '1e-300'), 1, 'nothing is close to zero relatively: the caller decides what near zero means'); +}); + +test('main follows the pseudo\'s own lines, and a procedure call is not printed', () => { + const r = pseudoToMojo('PROCEDURE SHOW(N : INTEGER)\n OUTPUT N\nENDPROCEDURE\n', { call: 'SHOW(4)' }); + assert.ok(r.source.includes('def main():\n show(4)\n'), r.source); + const none = pseudoToMojo('FUNCTION F(N : INTEGER) RETURNS INTEGER\n RETURN N\nENDFUNCTION\n'); + assert.ok(!none.source.includes('def main'), 'with no call and no lines of its own there is nothing to run'); +}); + +test('raises is inferred through dictionary reads and through callers, and nothing else raises', () => { + const pseudo = `FUNCTION GET(M : HASHMAP OF STRING : INTEGER, K : STRING) RETURNS INTEGER + RETURN M[K] +ENDFUNCTION +FUNCTION TWICE(M : HASHMAP OF STRING : INTEGER, K : STRING) RETURNS INTEGER + RETURN GET(M, K) * 2 +ENDFUNCTION +FUNCTION PLAIN(N : INTEGER) RETURNS INTEGER + RETURN N + 1 +ENDFUNCTION +`; + const r = pseudoToMojo(pseudo, { call: null }); + assert.ok(r.source.includes('def get(m: Dict[String, Int], k: String) raises -> Int:'), r.source); + assert.ok(r.source.includes('def twice(m: Dict[String, Int], k: String) raises -> Int:'), r.source); + assert.ok(r.source.includes('def plain(n: Int) -> Int:'), r.source); + const withMain = pseudoToMojo(pseudo + 'OUTPUT PLAIN(1)\n'); + assert.ok(withMain.source.includes('def main():'), 'main does not raise when nothing it calls does'); + const raising = pseudoToMojo(pseudo + 'DECLARE M : HASHMAP OF STRING : INTEGER\nM["a"] <- 1\nOUTPUT TWICE(M, "a")\n'); + assert.ok(raising.source.includes('def main() raises:'), raising.source); +}); + +test('a set needs its import, and the import comes from the harvested module', () => { + const r = pseudoToMojo('TYPE Bases = SET OF CHAR\nDEFINE Purines (\'A\', \'G\') : Bases\nOUTPUT \'A\' IN Purines\n'); + assert.ok(r.source.includes('from std.collections import Set'), r.source); + assert.equal(checkMojo(r.source).diagnostics.filter((d) => d.severity === 'error').length, 0); +}); + +test('what has no faithful Mojo becomes a visible raise and a warning, never a guess', () => { + const r = pseudoToMojo('DECLARE N : FLOAT\nN <- 1.0\nN <- RANDN()\nOUTPUT N\n'); + const gap = r.notes.find((n) => n.id === 'translate-gap'); + assert.ok(gap && gap.severity === 'warning' && gap.line === 3, JSON.stringify(r.notes ?? r)); + assert.ok(r.source.includes('raise Error("not translated: pseudo line 3")'), r.source); + assert.ok(r.source.includes('def main() raises:'), 'the routine that raises says so'); + assert.equal(checkMojo(r.source).diagnostics.filter((d) => d.severity === 'error').length, 0, 'the result still checks'); +}); + +test('float literals are not folded the way Mojo folds them: 0.1 + 0.2 stays a Float64 sum', () => { + const r = pseudoToMojo('OUTPUT 0.1 + 0.2 == 0.3\n'); + assert.ok(r.source.includes('Float64(0.1) + 0.2 == 0.3'), r.source); + const run = runMojo(checkMojo(r.source).module, r.source, {}); + assert.equal(run.stdout, 'FALSE\n'); +}); + +test('quantifiers become loops written before the line that uses them, and agree with the pseudo', () => { + const src = fs.readFileSync(path.join(corpusDir, '27-quantifiers.pseudo'), 'utf8'); + const r = pseudoToMojo(src); + assert.ok(/var _all\d+ = True/.test(r.source) && /var _any\d+ = False/.test(r.source) && /var _sum\d+ = 0/.test(r.source), r.source); + assert.equal(verdicts.get('27-quantifiers.pseudo')?.verdict, 'agree'); +}); + +test('a quantifier inside a loop condition is a gap, not a wrong answer', () => { + const r = pseudoToMojo('DECLARE L : LIST OF INTEGER\nL <- [1, 2]\nWHILE EXISTS K IN [0, LENGTH(L)) : L[K] > 5\n L[0] <- 9\nENDWHILE\n'); + assert.ok(r.notes.some((n) => n.id === 'translate-gap'), JSON.stringify(r.notes)); +}); + +/** + * Spellings the pinned compiler asked for in whole programs: `pixi run mojo-fuzz --statements` builds generated + * programs of statements (loops, IF and CASE, collections, routines with BYREF, records) and found each of these. + */ +const pseudoOut = (src) => { const st = studyOf(src); const tr = runPseudo(st, { maxSteps: 200000 }); assert.equal(tr.error, null, JSON.stringify(tr.error)); return tr.output.join('\n'); }; +const mojoOut = (r) => { const ck = checkMojo(r.source); assert.deepEqual(ck.diagnostics.filter((d) => d.severity === 'error').map((d) => d.message), []); const run = runMojo(ck.module, r.source, { maxSteps: 400000 }); assert.ok(!run.refused, run.refused); return run.stdout.replace(/\n$/, ''); }; + +test('two pseudo names that make one Mojo name are told apart', () => { + const src = 'DECLARE RunTotal : INTEGER <- 1\nDECLARE RUN_TOTAL : INTEGER <- 2\nOUTPUT RunTotal, " ", RUN_TOTAL'; + const r = mojoOf(src); + has(r, 'var run_total: Int = 1', 'var run_total_2: Int = 2', 'print(run_total, " ", run_total_2, sep="")'); + assert.equal(mojoOut(r), pseudoOut(src)); +}); + +test('two constants that make one Mojo name are told apart, as two variables are', () => { + // the statement fuzzer, seed 61: the compiler said "invalid redefinition of 'WORD_LIST2'" + const src = 'CONSTANT WordList2 = 2\nCONSTANT WORD_LIST2 = 6\nOUTPUT WordList2, " ", WORD_LIST2, " ", WORD_LIST2 + WordList2'; + const r = mojoOf(src); + has(r, 'comptime WORD_LIST2: Int = 2', 'comptime WORD_LIST2_2: Int = 6', 'print(WORD_LIST2, " ", WORD_LIST2_2, " ", WORD_LIST2_2 + WORD_LIST2, sep="")'); + assert.equal(mojoOut(r), pseudoOut(src)); +}); + +test('a value the translation already named is handed to a call as it is, beside a BYREF argument', () => { + // the statement fuzzer, seed 7302: `var _arg3: List[String] = _parts2` was rejected ("cannot be implicitly copied") + const src = 'TYPE Point\n DECLARE Label : STRING\nENDTYPE\nPROCEDURE Show(BYREF P : Point, Parts : LIST OF STRING, Order : LIST OF INTEGER)\n OUTPUT LENGTH(Parts), " ", P.Label, " ", Order\nENDPROCEDURE\nFUNCTION Build(P : Point, Ns : LIST OF INTEGER) RETURNS INTEGER\n CALL Show(P, SPLIT(P.Label, ","), SORTED(Ns))\n RETURN 0\nENDFUNCTION\nDECLARE Q : Point\nQ.Label <- "a,b,c"\nOUTPUT Build(Q, [3, 1, 2])'; + const r = mojoOf(src); + assert.doesNotMatch(r.source, /var _arg\d+: List\[String\] = _parts\d+$/m); + assert.equal(mojoOut(r), pseudoOut(src)); +}); + +test('a variable with no value that is read only inside an IF or a loop starts at zero, as one some path sets does', () => { + // the statement fuzzer, seed 1 (programs 382, 529, 2671): the notation runs when the path is not taken; Mojo said + // "use of uninitialized value" + const src = 'DECLARE N : INTEGER\nDECLARE B : BOOL <- FALSE\nIF B THEN\n OUTPUT N\nENDIF\nFOR I <- 0 TO 0\n DECLARE M : FLOAT\n IF B THEN\n OUTPUT M\n ENDIF\nNEXT I\nOUTPUT "end"'; + const r = mojoOf(src); + has(r, 'var n: Int = 0', 'var m: Float64 = 0.0'); + assert.ok(r.notes.some((n) => n.id === 'translate-maybe-unset')); + assert.equal(mojoOut(r), pseudoOut(src)); + // a read every run reaches is left for the compiler to reject, as the notation stops there + has(mojoOf('DECLARE K : INTEGER\nOUTPUT K'), 'var k: Int'); + assert.doesNotMatch(mojoOf('DECLARE K : INTEGER\nOUTPUT K').source, /var k: Int = 0/); +}); + +test('an argument that calls something is worked out once when a loop written for the built-in mentions it', () => { + // the statement fuzzer, seed 202: SLICE(F(X), 0, 2) ran F once per letter, and its OUTPUT showed each time + const F = 'FUNCTION Word(N : INTEGER) RETURNS STRING\n OUTPUT "word ", N\n RETURN "x,y,z"\nENDFUNCTION\nFUNCTION Nums(N : INTEGER) RETURNS LIST OF INTEGER\n OUTPUT "nums ", N\n RETURN [N, N + 1, N + 2]\nENDFUNCTION\nFUNCTION Pick(N : INTEGER) RETURNS INTEGER\n OUTPUT "pick ", N\n RETURN N\nENDFUNCTION\n'; + for (const body of ['OUTPUT SLICE(Word(1), 0, 3)', 'DECLARE S : STRING\nS <- SLICE(Word(1), 0, MIN(2, LENGTH(Word(2))))\nOUTPUT S', 'OUTPUT INDEX_OF(Nums(4), Pick(6))', 'OUTPUT DOT(Nums(1), Nums(2))', + // a list in brackets is named by the translation already (ci-fuzz, seed 1: `var _value6 = _list2` does not copy) + 'DECLARE LI : LIST OF INTEGER\nLI <- [3, 1, 2]\nOUTPUT INDEX_OF([FLOOR(3.25), MAX((-1), 1000)], MAX(MIN(LI), 3))', 'OUTPUT DOT([ABS(-1), 2], [Pick(3), 4])']) { + const r = mojoOf(F + body); + assert.equal(mojoOut(r), pseudoOut(F + body), body); + } +}); + +test('a name declared inside a block and used after it is declared before the block', () => { + const src = 'DECLARE N : INTEGER <- 3\nIF N > 0 THEN\n DECLARE Inner : INTEGER <- 5\nENDIF\nOUTPUT Inner\nFOR I <- 0 TO 2\n DECLARE Row : LIST OF INTEGER\n APPEND(Row, I)\n DECLARE T : INTEGER <- I\nNEXT I\nOUTPUT T, " ", Row'; + const r = mojoOf(src); + has(r, 'var inner: Int = 0', 'inner = 5', 'var t: Int = 0', 'var row: List[Int] = List[Int]()', 'row = List[Int]()', 't = i'); + assert.ok(r.source.indexOf('var inner: Int = 0') < r.source.indexOf('if n > 0:'), r.source); + assert.ok(r.notes.some((n) => n.id === 'translate-lifted')); + assert.equal(mojoOut(r), pseudoOut(src)); +}); + +test('a scalar some path leaves without a value starts at zero; one every path assigns, or none does, is left as declared', () => { + const src = 'DECLARE N : INTEGER <- 3\nDECLARE W : INTEGER\nIF N > 1 THEN\n W <- 1\nENDIF\nOUTPUT W\nDECLARE V : INTEGER\nFOR I <- 0 TO N\n V <- I\nNEXT I\nOUTPUT V\nDECLARE Z : INTEGER\nIF N > 1 THEN\n Z <- 1\nELSE\n Z <- 2\nENDIF\nOUTPUT Z'; + const r = mojoOf(src); + has(r, 'var w: Int = 0', 'var v: Int = 0', 'var z: Int'); + assert.ok(r.notes.some((n) => n.id === 'translate-maybe-unset')); + assert.equal(mojoOut(r), pseudoOut(src)); + // read before anything can give it a value: every run of the notation stops there, and the compiler rejects it too + has(mojoOf('DECLARE T : INTEGER\nOUTPUT T + 1'), 'var t: Int'); +}); + +test('CONTINUE in a REPEAT tests the UNTIL condition before the next pass', () => { + const src = 'DECLARE K : INTEGER <- 0\nREPEAT\n K <- K + 1\n IF K < 3 THEN\n CONTINUE\n ENDIF\n OUTPUT "k ", K\nUNTIL K >= 2\nOUTPUT K'; + const r = mojoOf(src); + assert.ok(/ {12}if k >= 2:\n {16}break\n {12}continue\n/.test(r.source), r.source); + assert.equal(pseudoOut(src), '2'); + assert.equal(mojoOut(r), '2'); +}); + +test('DELETE of a member or key that is not there changes nothing, and does not raise', () => { + const r = mojoOf('DECLARE S : SET OF INTEGER <- {3, 1}\nDELETE(S, 7)\nDECLARE M : HASHMAP OF STRING : FLOAT <- {"k": 1}\nDELETE(M, "zz")\nDECLARE G : HASHMAP OF INTEGER : LIST OF INTEGER\nDELETE(G, 2)\nOUTPUT LENGTH(S), LENGTH(M)'); + has(r, 's.discard(7)', '_ = m.pop("zz", 0.0)', '_ = g.pop(2, List[Int]())', 'var m: Dict[String, Float64] = {"k": 1}'); + assert.ok(r.source.includes('def main():'), 'nothing here raises'); +}); + +test('a map literal lands in the map\'s types, and a SLICE of a list is a new list', () => { + const r = mojoOf('DECLARE H : HASHMAP OF CHAR : LIST OF INTEGER <- {\'d\': [3]}\nDECLARE L : LIST OF INTEGER <- [1, 2, 3]\nDECLARE M : LIST OF INTEGER\nM <- SLICE(L, 0, 2)\nOUTPUT M, LENGTH(H)'); + has(r, 'var h: Dict[String, List[Int]] = {"d": [3]}', 'm = List[Int](l[0:2])'); +}); + +test('FOR EACH walks the text, list or map as it was when the loop began', () => { + const src = 'DECLARE S : STRING <- "abc"\nFOR EACH Ch IN S\n S <- "z" & S\n OUTPUT Ch\nNEXT Ch\nDECLARE H : HASHMAP OF STRING : INTEGER <- {"a": 1}\nFOR EACH K IN H\n H[K] <- H[K] * 2\nNEXT K\nOUTPUT S, H'; + const r = mojoOf(src); + has(r, 'var _walk2 = String(s)', 'var ch = String(_walk2[codepoint=_ch_pos1])', 'var _walk3 = h.copy()', 'for k in _walk3.keys():'); + assert.equal(pseudoOut(src), 'a\nb\nc\nzzzabc{"a": 2}'); +}); + +test('notes say what differs, and only where it does: / and DIV by zero, IEEE, whole and fractional powers', () => { + const ids = (src) => mojoOf(src).notes.map((n) => n.id); + assert.ok(ids('DECLARE X : FLOAT <- 1.0\nOUTPUT X / 3').includes('translate-divide')); + assert.ok(!ids('USE IEEE\nDECLARE X : FLOAT <- 1.0\nOUTPUT X / 0.0').includes('translate-divide'), 'under USE IEEE the notation gives infinity too'); + // probes/int-div-zero and int-mod-zero: Mojo's // and % by zero give 0 + assert.ok(ids('DECLARE N : INTEGER <- 7\nDECLARE Z : INTEGER <- 0\nOUTPUT N DIV Z').includes('translate-divide-int')); + assert.ok(ids('DECLARE N : INTEGER <- 7\nDECLARE Z : INTEGER <- 0\nOUTPUT N MOD (Z + 1)').includes('translate-divide-int')); + assert.ok(!ids('DECLARE N : INTEGER <- 7\nOUTPUT N DIV 2, N MOD (-3)').includes('translate-divide-int'), 'a nonzero literal divisor cannot be zero'); + const whole = mojoOf('DECLARE X : FLOAT <- 1.1\nDECLARE N : INTEGER <- 20\nOUTPUT X ^ N, X ^ 3'); + assert.deepEqual(whole.notes.filter((n) => /power/.test(n.id)).map((n) => n.id), ['translate-power-whole']); + assert.match(whole.notes.find((n) => n.id === 'translate-power-whole').message, /repeated multiplication/); + const frac = mojoOf('DECLARE X : FLOAT <- 2.0\nOUTPUT X ^ 0.5'); + assert.match(frac.notes.find((n) => n.id === 'translate-power').message, /13 significant digits/); + assert.ok(!/1e10/.test(frac.notes.find((n) => n.id === 'translate-power').message)); + const unknown = mojoOf('DECLARE X : FLOAT <- 2.0\nDECLARE Y : FLOAT <- 0.5\nOUTPUT POW(X, Y)'); + assert.match(unknown.notes.find((n) => n.id === 'translate-power').message, /whole power.*fractional power/); +}); + +test('a FLOAT made into text is printed, never used as text: Mojo spells some FLOATs otherwise', () => { + // the notation writes 6.02e23, Mojo 6.02e+23; they switch to an exponent at other sizes, and where two shortest + // spellings tie they pick differently (11340668257756.562 and ...563), so the text itself is not the notation's + const used = mojoOf('DECLARE X : FLOAT <- 6.02e23\nOUTPUT 100 * LENGTH(TO_STRING(X))\nDECLARE S : STRING\nS <- "x" & TO_STRING(X)\nOUTPUT S'); + assert.deepEqual(gaps(used).map((g) => g.line), [2, 4]); + assert.match(gaps(used)[0].message, /used as text/); + // printed, it is the same value: OUTPUT compares by value + const shown = mojoOf('DECLARE X : FLOAT <- 2.5\nOUTPUT "x = " & TO_STRING(X) & " " & TO_STRING(X + 1.0), " ", TO_STRING(X)'); + assert.deepEqual(gaps(shown), []); + has(shown, 'print((("x = " + String(x)) + " ") + String(x + 1.0), " ", String(x), sep="")'); + assert.equal(mojoOut(shown), 'x = 2.5 3.5 2.5'); + // a list of text made into text has quotes in the notation and none in Mojo + assert.equal(gaps(mojoOf('DECLARE L : LIST OF STRING <- ["a"]\nOUTPUT LENGTH(TO_STRING(L))')).length, 1); + assert.deepEqual(gaps(mojoOf('DECLARE L : LIST OF INTEGER <- [1]\nOUTPUT LENGTH(TO_STRING(L))')), []); +}); + +test('SUM_OF, MAX, MEAN_OF, SORTED and IN of a two-dimensional ARRAY look at every cell, not at the rows', () => { + const src = 'DECLARE A : ARRAY[2, 3] OF INTEGER\nFOR I <- 0 TO 2\n FOR J <- 0 TO 3\n A[I, J] <- I * 10 + J\n NEXT J\nNEXT I\nOUTPUT SUM_OF(A)\nOUTPUT MAX(A)\nOUTPUT MEAN_OF(A)\nOUTPUT 12 IN A\nOUTPUT SORTED(A)'; + const r = mojoOf(src); + has(r, 'var _cells4 = List[Int]()', 'for _row2 in a:', 'for _x3 in _row2:', '_cells4.append(_x3)', 'for _x5 in _cells4:'); + assert.equal(mojoOut(r), pseudoOut(src)); +}); + +test('a constant has its type, so Mojo does not fold it as a literal; Mojo\'s own words are not names', () => { + const src = 'CONSTANT Rate = 0.1\nCONSTANT Word = "GC"\nCONSTANT Lim = 4\nOUTPUT Rate + 0.2 == 0.3, " ", INT(Rate), " ", REPLACE(Word, "G", "x"), " ", Lim\nDECLARE Nonlocal : INTEGER <- 2\nOUTPUT Nonlocal'; + const r = mojoOf(src); + has(r, 'comptime RATE: Float64 = 0.1', 'comptime WORD: String = "GC"', 'comptime LIM: Int = 4', 'var nonlocal_: Int = 2'); + assert.equal(pseudoOut(src), 'FALSE 0 xC 4\n2'); + assert.ok(mojoOf('TYPE Self\n DECLARE X : INTEGER\nENDTYPE\nDECLARE P : Self\nP.X <- 1\nOUTPUT P.X').source.includes('struct Self_(Copyable, Movable):')); + assert.ok(mojoOf('TYPE Dict\n DECLARE X : INTEGER\nENDTYPE\nDECLARE P : Dict\nP.X <- 1\nOUTPUT P.X').source.includes('var p = Dict_(0)')); +}); + +test('a value made from the collection it is stored into is named first, with its type', () => { + const r = mojoOf('DECLARE L : LIST OF INTEGER <- [1, 2, 3]\nL <- SLICE(L, 0, 2)\nDECLARE G : LIST OF LIST OF INTEGER <- [[1], [2]]\nG[0] <- [G[1][0], 5]\nDECLARE S : LIST OF STRING <- ["a", "b"]\nS[0] <- JOIN(S, "-")\nOUTPUT L, G, S'); + has(r, 'var _value1: List[Int] = List[Int](l[0:2])', 'l = _value1^', 'g[0] = [g[1][0], 5]', 'var _value2: String = String("-").join(s)', 's[0] = _value2^'); +}); + +test('a variable handed BYREF and also read by another argument of the call is copied for that argument first', () => { + const src = 'PROCEDURE Grow(BYREF L : LIST OF INTEGER, Old : LIST OF INTEGER, N : INTEGER)\n APPEND(L, N)\n OUTPUT LENGTH(L), " ", LENGTH(Old)\nENDPROCEDURE\nDECLARE Xs : LIST OF INTEGER <- [1]\nCALL Grow(Xs, Xs, LENGTH(Xs))'; + const r = mojoOf(src); + has(r, 'var _arg1: List[Int] = xs.copy()', 'var _arg2: Int = len(xs)', 'grow(xs, _arg1^, _arg2)'); + has(mojoOf('PROCEDURE Grow(BYREF L : LIST OF INTEGER, Old : LIST OF INTEGER)\n APPEND(L, LENGTH(Old))\nENDPROCEDURE\nDECLARE Xs : LIST OF INTEGER <- [1]\nCALL Grow(Xs, [Xs[0], 2])'), 'var _arg1: List[Int] = [xs[0], 2]'); + assert.equal(mojoOut(r), pseudoOut(src)); +}); + +test('FILL works its value out once', () => { + const src = 'FUNCTION Noisy(N : INTEGER) RETURNS INTEGER\n OUTPUT "called"\n RETURN N * 2\nENDFUNCTION\nDECLARE Xs : LIST OF INTEGER <- FILL(3, Noisy(4))\nDECLARE G : LIST OF LIST OF INTEGER <- FILL(2, [1, 2])\nG[0][0] <- 9\nOUTPUT Xs, " ", G'; + const r = mojoOf(src); + has(r, 'var _item1: Int = noisy(4)', '_fill2.append(_item1)'); + assert.equal(pseudoOut(src), 'called\n[8, 8, 8] [[9, 2], [1, 2]]'); + assert.equal(mojoOut(r), pseudoOut(src)); +}); + +test('a CASE subject is worked out once, before any arm', () => { + const src = 'FUNCTION Noisy(N : INTEGER) RETURNS INTEGER\n OUTPUT "called"\n RETURN N\nENDFUNCTION\nCASE OF Noisy(3)\n 1 : OUTPUT "one"\n 2, 3 : OUTPUT "two or three"\n OTHERWISE : OUTPUT "other"\nENDCASE\nCASE OF Noisy(5)\nENDCASE'; + const r = mojoOf(src); + has(r, 'var _subject1: Int = noisy(3)', 'if _subject1 == 1:', 'elif _subject1 == 2 or _subject1 == 3:', 'var _subject2: Int = noisy(5)'); + assert.equal(pseudoOut(src), 'called\ntwo or three\ncalled'); + assert.equal(mojoOut(r), pseudoOut(src)); +}); diff --git a/tests/sync-speco.test.mjs b/tests/sync-speco.test.mjs new file mode 100644 index 0000000..adc02e2 --- /dev/null +++ b/tests/sync-speco.test.mjs @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Stage 7: speco as a lens on pseudo's contracts. extractSpec reads the contracts out of pseudo; weave in +// replace mode writes a speco into pseudo in place of what is there; reconcileSpec keeps the claims that have +// lost their routine or loop. The laws below make sync settle after one pass and never lose a contract. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { programs } from './helpers/sync-programs.mjs'; +import { studyOf } from '../dist/services/pseudo.js'; +import { parseSpec, weave, extractSpec, reconcileSpec } from '../dist/services/speco.js'; + +globalThis.localStorage = { getItem() { return null; }, setItem() { }, removeItem() { } }; + +const CONTRACT = /^\s*(REQUIRE|ENSURE|INVARIANT|DECREASES)\b/; +const withContracts = () => programs().filter((p) => p.pseudo.split('\n').some((l) => CONTRACT.test(l)) && studyOf(p.pseudo).ok); +const stripContracts = (src) => src.split('\n').filter((l) => !CONTRACT.test(l)).join('\n'); + +const COUNT = `FUNCTION COUNT_BASE(B : CHAR, T : STRING) RETURNS INTEGER + DECLARE Total : INTEGER + Total <- 0 + FOR I <- 0 TO LENGTH(T) + IF T[I] == B THEN + Total <- Total + 1 + ENDIF + NEXT I + RETURN Total +ENDFUNCTION +`; +const SPEC = 'SPEC COUNT_BASE(B, T)\n ENSURE result = |{ K ∈ [0, |T|) : T[K] = B }|\n LOOP FOR I\n INVARIANT Total = |{ K ∈ [0, I) : T[K] = B }|'; + +test('there are programs with contracts to hold the laws to', () => { + const ids = withContracts().map((p) => p.id); + assert.ok(ids.includes('ex:count-base-proved') && ids.includes('ex:match-proved'), ids.join(', ')); + assert.ok(ids.length >= 10, `${ids.length} programs with contracts`); +}); + +test('replace mode: what the speco says replaces what the routine and the selected loops said, and insert mode stays as it was', () => { + const old = weave(COUNT, parseSpec('SPEC COUNT_BASE(B, T)\n REQUIRE LENGTH(T) >= 0\n ENSURE RESULT >= 0\n LOOP FOR I\n INVARIANT Total >= 0\n DECREASES LENGTH(T) - I')).source; + assert.match(old, /REQUIRE LENGTH\(T\) >= 0/); + const w = weave(old, parseSpec(SPEC), { replace: true }); + assert.deepEqual(w.diagnostics, []); + assert.ok(!/REQUIRE|DECREASES|RESULT >= 0|Total >= 0/.test(w.source), w.source); + assert.match(w.source, /FUNCTION COUNT_BASE[^\n]*\n\s+ENSURE RESULT == LENGTH\(\{ K IN \[0, LENGTH\(T\)\) : T\[K\] == B \}\)\n/); + assert.match(w.source, /FOR I <- 0 TO LENGTH\(T\)\n\s+INVARIANT Total == LENGTH/); + assert.ok(studyOf(w.source).ok); + // insert mode keeps what is there and adds the new claims + const kept = weave(old, parseSpec(SPEC)); + assert.match(kept.source, /REQUIRE LENGTH\(T\) >= 0/); + assert.match(kept.source, /DECREASES LENGTH\(T\) - I/); + assert.match(kept.source, /ENSURE RESULT == LENGTH/); +}); + +test('replace mode: routines and loops the spec does not name are left alone, and a unit with no claims clears its routine', () => { + const two = `${COUNT} +FUNCTION OTHER(N : INTEGER) RETURNS INTEGER + REQUIRE N >= 0 + RETURN N +ENDFUNCTION +`; + const w = weave(two, parseSpec('SPEC COUNT_BASE(B, T)\n ENSURE RESULT >= 0'), { replace: true }); + assert.match(w.source, /REQUIRE N >= 0/); + const cleared = weave(w.source, parseSpec('SPEC COUNT_BASE(B, T)\n REQUIRE TRUE'), { replace: true }); + assert.ok(!/ENSURE RESULT >= 0/.test(cleared.source)); + assert.match(cleared.source, /REQUIRE TRUE/); +}); + +test('replace mode is idempotent, and its map names the speco line each written line came from', () => { + const spec = parseSpec(SPEC); + const once = weave(COUNT, spec, { replace: true }); + const twice = weave(once.source, spec, { replace: true }); + assert.equal(twice.source, once.source); + assert.equal(once.map.length, 2); + const lines = once.source.split('\n'); + for (const m of once.map) assert.ok(/ENSURE|INVARIANT/.test(lines[m.to - 1]) && m.from >= 1); +}); + +test('lens law: extract(weave(P, extract(P))) === extract(P), for every program with contracts', () => { + const bad = []; + for (const p of withContracts()) { + const spec = extractSpec(p.pseudo); + const w = weave(p.pseudo, parseSpec(spec), { replace: true }); + if (w.diagnostics.length) { bad.push(`${p.id}: ${w.diagnostics[0].message}`); continue; } + if (!studyOf(w.source).ok) { bad.push(`${p.id}: the woven pseudo does not read`); continue; } + if (extractSpec(w.source) !== spec) bad.push(`${p.id}: extract(weave(P, extract(P))) differs`); + const again = weave(w.source, parseSpec(spec), { replace: true }); + if (again.source !== w.source) bad.push(`${p.id}: weaving twice changes the pseudo`); + } + assert.deepEqual(bad, []); +}); + +test('preservation: a spec woven into pseudo with no contracts, then extracted, comes back, for every program with contracts', () => { + const bad = []; + for (const p of withContracts()) { + const spec = extractSpec(p.pseudo); + const bare = stripContracts(p.pseudo); + const w = weave(bare, parseSpec(spec), { replace: true }); + if (w.diagnostics.length) { bad.push(`${p.id}: ${w.diagnostics[0].message}`); continue; } + if (extractSpec(w.source) !== spec) bad.push(`${p.id}: the spec did not come back`); + // and the contracts it restored are the ones the program had, up to spelling + const norm = (t) => t.split('\n').filter((l) => CONTRACT.test(l)).map((l) => l.trim().replace(/\s+/g, '')).join('|'); + if (norm(w.source) !== norm(p.pseudo)) bad.push(`${p.id}: the contracts written back differ from the original's`); + } + assert.deepEqual(bad, []); +}); + +test('orphans: a spec whose routine is gone is kept, says why, and is reported on its line', () => { + const prev = `${extractSpec(weave(COUNT, parseSpec(SPEC)).source)}\n\nSPEC GONE(N)\n REQUIRE N >= 0\n`; + const r = reconcileSpec(COUNT.replace('COUNT_BASE', 'COUNT_BASE'), prev); + assert.match(r.source, /\/\/ kept: the pseudocode has no FUNCTION or PROCEDURE called GONE\nSPEC GONE\(N\)\n REQUIRE N ≥ 0/); + assert.equal(r.diagnostics.length, 1); + assert.equal(r.diagnostics[0].id, 'spec-orphan'); + assert.equal(r.source.split('\n')[r.diagnostics[0].line - 1], 'SPEC GONE(N)'); + // the pseudo's own contracts (none here) are the spec's: COUNT_BASE has none, so its claims were deleted there + assert.ok(!/SPEC COUNT_BASE/.test(r.source)); + // a fixed point + assert.equal(reconcileSpec(COUNT, r.source).source, r.source); +}); + +test('orphans: a spec for a loop that is gone stays in its unit, and a routine that comes back takes its claims again', () => { + const woven = weave(COUNT, parseSpec(SPEC)).source; + const prev = extractSpec(woven); + // the FOR loop becomes a WHILE loop: the invariant has no loop to be about + const whileVersion = woven.replace(/FOR I <- 0 TO LENGTH\(T\)\n\s+INVARIANT[^\n]*\n/, 'WHILE FALSE\n').replace(' NEXT I', ' ENDWHILE'); + const r = reconcileSpec(whileVersion, prev); + assert.match(r.source, /ENSURE result/); + assert.match(r.source, /\/\/ kept: COUNT_BASE has no loop matching "LOOP FOR I"\n\s+LOOP FOR I\n\s+INVARIANT Total/); + assert.equal(r.diagnostics.length, 1); + assert.equal(r.source.split('\n')[r.diagnostics[0].line - 1].trim(), 'LOOP FOR I'); + assert.equal(reconcileSpec(whileVersion, r.source).source, r.source); + // the FOR loop returns, with no invariant yet: the kept claim is its again + const back = reconcileSpec(stripContracts(woven), r.source); + assert.ok(!/kept:/.test(back.source), back.source); + assert.match(back.source, /LOOP FOR I\n\s+INVARIANT Total/); + assert.equal(back.diagnostics.length, 0); + // a routine that was away and is back, with no contracts yet, takes its kept spec again + const gone = reconcileSpec('FUNCTION OTHER(N : INTEGER) RETURNS INTEGER\n RETURN N\nENDFUNCTION\n', prev); + assert.match(gone.source, /kept: the pseudocode has no FUNCTION or PROCEDURE called COUNT_BASE/); + const returned = reconcileSpec(stripContracts(woven), gone.source); + assert.ok(!/kept:/.test(returned.source)); + assert.match(returned.source, /^SPEC COUNT_BASE\(B, T\)\n\s+ENSURE result/); + // but a contract deleted in the pseudo, from a spec that was not waiting, is gone from the spec too + assert.ok(!/SPEC COUNT_BASE/.test(reconcileSpec(stripContracts(woven), prev).source)); +}); + +test('a loop the spec names that the routine does not have is reported on the LOOP line of the speco', () => { + const w = weave(COUNT, parseSpec('SPEC COUNT_BASE(B, T)\n ENSURE RESULT >= 0\n\n LOOP WHILE\n INVARIANT Total >= 0')); + const d = w.diagnostics.find((x) => x.id === 'no-loop'); + assert.ok(d); + assert.equal(d.line, 4); + // the spec's own text is not touched by weaving: it is the caller's to keep + assert.match(w.source, /ENSURE RESULT >= 0/); +}); diff --git a/tests/tasks.test.mjs b/tests/tasks.test.mjs new file mode 100644 index 0000000..a0598b5 --- /dev/null +++ b/tests/tasks.test.mjs @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The commands people run live in pixi.toml; the README's table lists them. These tests keep the +// two in step, and keep every `pixi run <task>` mentioned anywhere pointing at a task that exists. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +const ROOT = new URL('..', import.meta.url).pathname; +const read = (p) => fs.readFileSync(path.join(ROOT, p), 'utf8'); + +/** Task names in pixi.toml's [tasks] table: `name = {` at the start of a line, until the next table. */ +function pixiTasks() { + const text = read('pixi.toml'); + const body = text.slice(text.indexOf('\n[tasks]') + 1).split(/\n\[(?!tasks\])/)[0]; + return [...body.matchAll(/^([a-z][a-z0-9-]*)\s*=\s*\{/gm)].map((m) => m[1]); +} + +/** Task names in the README's Commands table: the first cell, a backticked name with optional arguments. */ +function readmeTasks() { + const readme = read('README.md'); + const section = readme.slice(readme.indexOf('## Commands'), readme.indexOf('\n## ', readme.indexOf('## Commands') + 1)); + return [...section.matchAll(/^\| `([a-z][a-z0-9-]*)(?: [^`]*)?` \|/gm)].map((m) => m[1]); +} + +test('the README lists every pixi task, once, and nothing else', () => { + const tasks = pixiTasks(); + assert.ok(tasks.length > 20, `only ${tasks.length} tasks read from pixi.toml`); + const listed = readmeTasks(); + assert.deepEqual([...listed].sort(), [...tasks].sort()); + assert.equal(new Set(listed).size, listed.length, 'a task is listed twice'); +}); + +test('every pixi task has a description', () => { + const text = read('pixi.toml'); + for (const t of pixiTasks()) { + const line = text.split('\n').find((l) => l.startsWith(`${t} = {`)); + assert.match(line, /description = "[^"]+"/, `${t} has no description`); + } +}); + +test('every `pixi run <task>` in the docs, scripts and sources names a real task', () => { + const tasks = new Set(pixiTasks()); + const files = []; + const walk = (dir) => { + for (const e of fs.readdirSync(path.join(ROOT, dir), { withFileTypes: true })) { + const p = path.join(dir, e.name); + if (e.isDirectory()) { if (!['node_modules', 'dist', '.git', '.pixi', '.tools', 'corpus', 'golden', 'package', 'work'].includes(e.name)) walk(p); } // work/ is git-ignored notes + else if (/\.(md|mjs|ts|py|lua|yml|toml)$/.test(e.name)) files.push(p); + } + }; + walk('.'); + const unknown = []; + for (const f of files) { + for (const m of read(f).matchAll(/pixi run ([a-z][a-z0-9-]*)/g)) if (!tasks.has(m[1])) unknown.push(`${f}: pixi run ${m[1]}`); + } + assert.deepEqual(unknown, []); +}); + +test('package.json keeps only what CI and a machine without pixi need', () => { + const pkg = JSON.parse(read('package.json')); + assert.deepEqual(Object.keys(pkg.scripts).sort(), ['build', 'check', 'test', 'watch']); +}); + +test('every document the README points to exists', () => { + const readme = read('README.md'); + for (const m of readme.matchAll(/\]\(((?:docs|mojo|scripts)\/[^)#]+)\)/g)) assert.ok(fs.existsSync(path.join(ROOT, m[1])), `${m[1]} is linked from the README but missing`); +}); + +test('package.json and pixi.toml agree: the same version, and the Node pixi pins is one engines.node allows', () => { + // package.json is what a machine without pixi reads (and CI did, until pixi came to the runner); the two can drift + const pkg = JSON.parse(read('package.json')); + const toml = read('pixi.toml'); + assert.equal(/^version = "([^"]+)"/m.exec(toml)?.[1], pkg.version, 'the version in pixi.toml [workspace] and in package.json'); + const parts = (s) => s.split('.').map(Number).concat([0, 0]).slice(0, 3); + const atLeast = (a, b) => { for (let i = 0; i < 3; i++) if (a[i] !== b[i]) return a[i] > b[i]; return true; }; + const floor = /^>=\s*(\d+(?:\.\d+){0,2})$/.exec(pkg.engines?.node ?? ''); + assert.ok(floor, `engines.node is "${pkg.engines?.node}"; this test reads ">=X.Y.Z"`); + const pinned = /^nodejs = ">=(\d+\.\d+\.\d+)/m.exec(toml); + assert.ok(pinned, 'pixi.toml pins nodejs with a lower bound'); + assert.ok(atLeast(parts(pinned[1]), parts(floor[1])), `pixi installs Node ${pinned[1]}, below engines.node ${pkg.engines.node}`); + assert.ok(atLeast(parts(process.versions.node), parts(floor[1])), `this Node is ${process.versions.node}, below engines.node ${pkg.engines.node}`); +}); diff --git a/tests/third-party.test.mjs b/tests/third-party.test.mjs new file mode 100644 index 0000000..8638f24 --- /dev/null +++ b/tests/third-party.test.mjs @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// The third-party inventories cannot drift from what they describe. library/THIRD_PARTY.md is written by +// `pixi run mojo-emit` (scripts/mojo/mojocorpus/notices.py) from the pin, the lock and every harvested file; +// THIRD_PARTY.md here is written by hand and held to the locks and pins it names. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const ROOT = new URL('..', import.meta.url).pathname; +const read = (p) => fs.readFileSync(path.join(ROOT, p), 'utf8'); + +function python(code) { + const r = spawnSync('python3', ['-c', `import sys, json; sys.path.insert(0, 'scripts/mojo')\nfrom mojocorpus import notices\nfrom mojocorpus.common import load_pin\npin = load_pin()\n${code}`], { encoding: 'utf8', cwd: ROOT }); + assert.equal(r.status, 0, r.stderr.split('\n').slice(-20).join('\n')); + return JSON.parse(r.stdout); +} +const noPython = spawnSync('python3', ['--version']).status !== 0 ? 'python3 is not installed' : false; + +test('library/THIRD_PARTY.md is what mojo-emit writes from the pin, the lock and the corpus', { skip: noPython }, () => { + const want = python('print(json.dumps(notices.render(pin)))'); + assert.ok(read('library/THIRD_PARTY.md') === want, 'library/THIRD_PARTY.md is out of date: run `pixi run mojo-emit` and commit library/'); +}); + +test("every source's license was checked at the commit the lock holds, and the copies are the ones checked", { skip: noPython }, () => { + assert.deepEqual(python('print(json.dumps(notices.problems(pin)))'), []); +}); + +test('moving a source to another commit makes the inventory stale and its license unchecked', { skip: noPython }, () => { + const r = python(` +before = notices.render(pin) +real = notices.load_lock +def moved(): + lock = real() + lock["sources"]["mojo-quest"]["commit"] = "0" * 40 + return lock +notices.load_lock = moved +print(json.dumps({"changed": notices.render(pin) != before, "problems": notices.problems(pin)}))`); + assert.equal(r.changed, true); + assert.deepEqual(r.problems, ['mojo-quest: license checked at 8e884a709866, but the lock holds 000000000000']); +}); + +test('the inventory counts every harvested program and names every file without a header', { skip: noPython }, () => { + const inv = python('print(json.dumps(notices.inventory(pin)))'); + const mojo = []; + const walk = (d) => { for (const e of fs.readdirSync(path.join(ROOT, d), { withFileTypes: true })) { const p = path.join(d, e.name); if (e.isDirectory()) walk(p); else if (p.endsWith('.mojo')) mojo.push(p); } }; + walk('library/mojo/corpus'); + assert.equal(inv.programs.length, mojo.length, 'a .mojo file under the corpus is in no manifest'); + const table = read('library/THIRD_PARTY.md'); + for (const p of inv.programs.filter((p) => p.header === 'none')) assert.ok(table.includes(`\`${p.file}\``), `${p.file} has no header and is not listed`); + for (const p of inv.programs.filter((p) => p.header !== 'none' && !p.header.startsWith('Apache-2.0 WITH LLVM-exception'))) assert.ok(table.includes(p.file), p.file); +}); + +test('THIRD_PARTY.md names the versions and licenses the locks and the Dafny pin hold', () => { + const doc = read('THIRD_PARTY.md'); + const row = (name) => doc.split('\n').find((l) => l.startsWith(`| ${name} `)) ?? ''; + const lock = read('pixi.lock'); + const conda = (pkg) => { + const m = lock.match(new RegExp(`\\n- conda: \\S*/${pkg}-(\\d[^-/]*)-[^/\\s]*\\.conda\\n(?: .*\\n)*? license: (.+)`)); + assert.ok(m, `${pkg} is not in pixi.lock`); + return { version: m[1], license: m[2].trim() }; + }; + const mojo = conda('mojo'); + assert.ok(row('Mojo').includes(`| ${mojo.version} |`) && row('Mojo').includes(mojo.license), `Mojo ${mojo.version}, ${mojo.license}`); + for (const pkg of ['mojo-compiler', 'mojo-python', 'mblack']) { + assert.equal(conda(pkg).license, mojo.license, `${pkg} has another license than mojo`); + assert.ok(row('Mojo').includes(`\`${pkg}\``), `${pkg} is not named in the Mojo row`); + } + for (const [name, pkg] of [['Node.js', 'nodejs'], ['Python', 'python']]) { + const c = conda(pkg); + assert.ok(row(name).includes(`| ${c.version} |`) && row(name).includes(`| ${c.license} |`), `${name} ${c.version}, ${c.license}`); + } + const ts = JSON.parse(read('package-lock.json')).packages['node_modules/typescript']; + assert.ok(row('TypeScript').includes(`| ${ts.version} |`) && row('TypeScript').includes(`| ${ts.license} |`), `TypeScript ${ts.version}, ${ts.license}`); + const dafny = read('scripts/dafny.py').match(/^VERSION = "([^"]+)"/m)[1]; + assert.ok(row('Dafny').includes(`| ${dafny} |`), `Dafny ${dafny}`); + const npm = Object.keys(JSON.parse(read('package-lock.json')).packages).filter((k) => k); + assert.deepEqual(npm, ['node_modules/typescript'], 'a new npm package needs a row here (and AGENTS.md allows none)'); +}); + +test('THIRD_PARTY.md describes every generated file the app ships', () => { + const doc = read('THIRD_PARTY.md'); + for (const f of fs.readdirSync(path.join(ROOT, 'src/mojo/generated'))) assert.ok(doc.includes(`\`src/mojo/generated/${f}\``), `src/mojo/generated/${f} is not described`); +}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..26b9dea --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,47 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "lib": [ + "ES2022", + "DOM", + "DOM.Iterable" + ], + "rootDir": "src", + "outDir": "dist", + "allowJs": true, + "checkJs": false, + "strict": true, + "noImplicitAny": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "noImplicitOverride": true, + "allowUnusedLabels": false, + "noUncheckedSideEffectImports": true, + "skipLibCheck": true, + "sourceMap": true, + "declaration": true, + "types": [] + }, + "include": [ + "src/**/*.ts", + "src/engines/**/*.js" + ], + "exclude": [ + "src/engines/pseudo/dom.js", + "src/engines/pseudo/views.js", + "src/engines/pseudo/views-analysis.js", + "src/engines/pseudo/lab.js", + "src/engines/pseudo/editor.js", + "src/engines/pseudo/exercise.js", + "src/engines/pseudo/sharpen.js", + "src/engines/pseudo/plan.js", + "src/engines/pseudo/planner.js", + "src/engines/pseudo/highlight.js", + "src/engines/asm86/sandbox.js", + "src/engines/asm86/specs-hw.js", + "src/engines/shared/share.js", + "src/engines/shared/viz.js" + ] +}