# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Scienting # # Source Code: https://git.scient.ing/papers/paperspec # See the LICENSE.md file for full license terms. """A package's datasets, checked against the package (spec/15, paperspec.datasets).""" import json import pytest from conftest import FIXTURES from helpers_cli import ( CHECKS_ONLY, HALE, edit_package, key_for, read_package, rules_of, store, ) from paperspec import datasets, package DS = "datasets/ds-hale-bond-params/" def errors(root): return [ e for e in package.validate(root, store()).errors if e["rule"].startswith("doc.dataset") ] def edited(tmp_path, change): """A copy of the main example with its dataset's files changed by `change(files)`.""" files, absent = read_package(HALE) change(files) root = tmp_path / HALE.name package.write(root, files, absent) return root def lines(files, name): return [json.loads(x) for x in files[DS + name].decode().splitlines()] def put_lines(files, name, recs): files[DS + name] = package.jsonl(recs) def test_the_example_dataset_breaks_no_rule(): assert errors(HALE) == [] def test_a_csv_value_that_is_not_its_cell_is_refused(tmp_path): def change(files): files[DS + "data.csv"] = files[DS + "data.csv"].replace(b"310.0", b"310.00") [e] = errors(edited(tmp_path, change)) assert e["rule"] == "doc.dataset.value" assert "'310.00' but its anchor reads '310.0'" in e["message"] def test_a_numeric_value_without_an_anchor_is_refused(tmp_path): def change(files): put_lines(files, "anchors.jsonl", lines(files, "anchors.jsonl")[1:]) assert {e["rule"] for e in errors(edited(tmp_path, change))} == { "doc.dataset.unanchored" } def test_an_anchor_to_a_row_the_csv_lacks_is_refused(tmp_path): def change(files): recs = lines(files, "anchors.jsonl") recs[0] = dict(recs[0], row="no-such-row") put_lines(files, "anchors.jsonl", recs) rules = {e["rule"] for e in errors(edited(tmp_path, change))} assert "doc.dataset.anchor" in rules def test_an_anchor_that_does_not_resolve_as_current_is_refused(tmp_path): def change(files): recs = lines(files, "anchors.jsonl") recs[0]["anchor"] = dict( recs[0]["anchor"], node_hash="0" * 64, own_hash="0" * 64 ) put_lines(files, "anchors.jsonl", recs) found = errors(edited(tmp_path, change)) assert any( e["rule"] == "doc.dataset.anchor" and "resolves as" in e["message"] for e in found ) def test_an_anchor_off_its_schema_is_refused(tmp_path): def change(files): recs = lines(files, "anchors.jsonl") recs[0] = dict(recs[0], page=3) put_lines(files, "anchors.jsonl", recs) assert "doc.dataset.anchor" in {e["rule"] for e in errors(edited(tmp_path, change))} def test_a_folder_not_named_by_the_id_is_refused(tmp_path): def change(files): for p in [p for p in files if p.startswith(DS)]: files[p.replace("ds-hale-bond-params", "ds-other")] = files.pop(p) [e] = errors(edited(tmp_path, change)) assert (e["rule"], e["where"]) == ("doc.dataset.id", "datasets/ds-other") def test_a_repeated_primary_key_is_refused(tmp_path): def change(files): csv = files[DS + "data.csv"].decode().splitlines() files[DS + "data.csv"] = ("\n".join([*csv, csv[1]]) + "\n").encode() assert "doc.dataset.key" in {e["rule"] for e in errors(edited(tmp_path, change))} def test_a_descriptor_off_its_schema_is_refused(tmp_path): def change(files): desc = json.loads(files[DS + "datapackage.json"]) desc["format"] = "scienting-dataset/9" files[DS + "datapackage.json"] = package.line(desc).encode() assert {e["rule"] for e in errors(edited(tmp_path, change))} == { "doc.dataset.schema" } @pytest.mark.parametrize( ("change", "where"), [ (lambda f: f.pop(DS + "datapackage.json"), "datasets/ds-hale-bond-params"), (lambda f: f.update({DS + "datapackage.json": b"{"}), DS + "datapackage.json"), ( lambda f: f.update({DS + "anchors.jsonl": b'{ "a": 1}\n'}), DS + "anchors.jsonl", ), (lambda f: f.pop(DS + "data.csv"), "datasets/ds-hale-bond-params"), (lambda f: f.update({DS + "data.csv": b"\xff\n"}), DS + "data.csv"), ], ids=[ "no-descriptor", "descriptor-not-json", "anchors-not-canonical", "no-csv", "csv-not-utf8", ], ) def test_a_file_that_is_missing_or_unreadable_is_refused(tmp_path, change, where): found = errors(edited(tmp_path, change)) assert any(e["rule"] == "doc.dataset.file" and e["where"] == where for e in found) def test_a_dataset_whose_cell_a_correction_changed_fails_until_rebuilt(tmp_path): # Datasets read the text, so a corrected cell breaks the value rule; revision leaves it out. def change(nodes): for n in nodes: if n["id"] == "c20cna6": n["content"] = [{"kind": "text", "text": "310.5"}] return nodes root = edit_package(tmp_path, HALE, {"nodes.jsonl": change}) assert "doc.dataset.anchor" in {e["rule"] for e in errors(root)} def test_the_content_hash_is_the_one_a_view_pins(): folder = FIXTURES / "records" / "datasets" / "ds-hale-ctct-scan" files = {p.name: p.read_bytes() for p in folder.iterdir()} view = json.loads((FIXTURES / "records" / "view.vl.json").read_text()) assert datasets.content_hash(files) in json.dumps(view) def test_folders_are_the_dataset_ids_a_package_holds(): files, _ = read_package(HALE) assert datasets.folders(files) == ["ds-hale-bond-params"] assert datasets.folders(["datasets/x", "nodes.jsonl"]) == [] def test_a_checks_only_package_is_not_looked_at(): # It carries no datasets, and the policy rules report one that does. report = package.validate(CHECKS_ONLY, store(), key_for(CHECKS_ONLY)) assert not [r for r in rules_of(report) if r.startswith("doc.dataset")]