Files
paperspec/tests/test_hashing.py
T
alexmandClaude 4fb8b4f5a7 Add paperspec correct, check math forms and datasets, and give the tools public names
The work the other paper tools planned for paperspec (paperdesk PLAN M5.4,
paperform PLAN 4.2):

- paperspec correct and paperspec.revision apply proposed corrections to a
  package, folder or zip, and write it only once it validates.
- doc.math.form is checked: label and style commands always, the form's own
  spelling when papermath is installed or forms= is passed.
- paperspec.datasets and seven doc.dataset.* rules check a package's datasets.
- fixtures/invalid gains suspect-witness and math-form-label (33 in all).
- Canonical JSON reads numbers as doubles, so 1e20 round-trips.
- The registries and the JSON-LD context ship in the wheel.
- Public names for what paperdesk and paperform imported as private ones,
  with the old names kept as aliases.
- The text and lossy-projection functions are about three times faster
  through the gate, tested equal to the old code on every code point.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
2026-10-20 22:10:17 -04:00

758 lines
26 KiB
Python
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""The projections and the five content values (spec/03 §3.6, spec/05 §5.1 to §5.3)."""
import hashlib
import hmac
import json
import unicodedata
from typing import ClassVar
import pytest
from hypothesis import given, settings, strategies as st
from conftest import FIXTURES
from helpers_lib1 import (
VALUE_FIELDS,
bare,
build_doc,
document,
heading,
ids_of_kind,
math_run,
package_nodes,
paragraph,
read_cases,
reference_values,
section,
sentence,
sentence_doc,
simple_doc,
text_run,
vectors,
)
from paperspec import hashing, jcs
from paperspec.hashing import (
MARK_CLOSE,
MARK_OPEN,
OBJ,
PLAIN,
Hasher,
Index,
as_printed,
compute,
lossy_from_strict,
meta_hash,
meta_object,
own_content,
plain_runs,
prose_length,
)
from paperspec.model import Doc
from paperspec.text import is_ws, norm, sha256_hex
PROPERTIES = settings(derandomize=True, max_examples=100, database=None, deadline=None)
PACKAGES = sorted(
p.name
for p in (FIXTURES / "packages").iterdir()
if p.is_dir() and not p.name.endswith(".checks-only")
)
def _hash(s):
return hashlib.sha256(s.encode("utf-8")).hexdigest()
class TestHasher:
def test_plain_hasher_is_sha256_of_text_or_bytes(self):
assert PLAIN("abc") == hashlib.sha256(b"abc").hexdigest()
assert PLAIN(b"abc") == PLAIN("abc")
assert Hasher().key is None
def test_keyed_hasher_is_hmac_sha256(self):
keyed = Hasher(b"Jefe")
expected = "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843"
assert keyed("what do ya want for nothing?") == expected
assert keyed(b"what do ya want for nothing?") == expected
assert keyed("x") == hmac.new(b"Jefe", b"x", hashlib.sha256).hexdigest()
def test_keyed_and_plain_values_differ(self):
assert Hasher(b"k")("x") != PLAIN("x")
assert Hasher(b"k")("x") != Hasher(b"l")("x")
@pytest.mark.parametrize("case", vectors("projections"))
def test_projection_vector(case):
doc, nid = sentence_doc(case["runs"])
strict, complexes = hashing.strict(doc, nid)
lossy = lossy_from_strict(strict)
plain = hashing.plain(doc, nid)
assert plain == case["plain"]
assert norm(plain) == case["normalized"]
assert strict == case["strict"]
assert lossy == case["lossy"]
assert complexes == case["complex"]
assert hashing.lossy(doc, nid) == lossy
assert _hash(norm(plain)) == case["text_hash"]
assert _hash(strict) == case["match_hash"]
assert _hash(lossy) == case["align_key"]
printed = hashing.printed(doc, nid)
assert printed == case.get("printed", plain)
@pytest.mark.parametrize("case", vectors("projections"))
def test_projection_vector_values_are_those_the_index_keeps(case):
doc, nid = sentence_doc(case["runs"])
values = compute(doc)[nid]
assert values["text_hash"] == case["text_hash"]
assert values["match_hash"] == case["match_hash"]
assert values["align_key"] == case["align_key"]
@pytest.mark.parametrize("group", vectors("projection-groups"))
def test_projection_group(group):
by_id = {c["id"]: c for c in read_cases("projections")}
field = group.get("same") or group.get("differ")
values = {by_id[x][field] for x in group["cases"]}
if "same" in group:
assert len(values) == 1
else:
assert len(values) == len(group["cases"])
@pytest.mark.parametrize("case", vectors("offsets"))
def test_offsets_vector_plain_text(case):
doc, nid = sentence_doc(case["runs"])
assert hashing.plain(doc, nid) == case["plain"]
@pytest.mark.parametrize("case", vectors("hashes"))
def test_hashes_vector(case):
doc = Doc(case["doc"], case["nodes"])
values = compute(doc)
assert values == case["values"]
for nid, own in case["own_content"].items():
assert sha256_hex(jcs.dumpb(own)) == values[nid]["own_hash"]
@pytest.mark.parametrize("case", vectors("hashes"))
def test_hashes_vector_agree_with_the_slow_computation(case):
doc = Doc(case["doc"], case["nodes"])
assert reference_values(doc) == case["values"]
def test_a_math_form_enters_no_content_value():
case = next(c for c in read_cases("hashes") if c["id"] == "math-form")
def unformed(node):
content = node.get("content")
if isinstance(content, dict):
content = {k: v for k, v in content.items() if k != "form"}
elif isinstance(content, list):
content = [{k: v for k, v in r.items() if k != "form"} for r in content]
return dict(node, content=content) if "content" in node else node
assert sum(json.dumps(n).count('"form"') for n in case["nodes"]) == 2
stripped = Doc(case["doc"], [unformed(n) for n in case["nodes"]])
assert compute(stripped) == case["values"]
@pytest.mark.parametrize("package", PACKAGES)
def test_fixture_values_are_the_stored_values(package):
nodes = package_nodes(FIXTURES / "packages" / package)
values = compute(Doc(package, bare(nodes)))
for n in nodes:
assert {k: n[k] for k in VALUE_FIELDS} == values[n["id"]]
@pytest.mark.parametrize("package", PACKAGES)
def test_fixture_values_agree_with_the_slow_computation(package):
nodes = bare(package_nodes(FIXTURES / "packages" / package))
doc = Doc(package, nodes)
assert Index.build(doc).values == reference_values(doc)
def test_keyed_values_follow_the_hasher():
doc = Doc("p", bare(package_nodes(FIXTURES / "packages" / "tiny2026tiny-j4qm")))
keyed = Hasher(b"team secret")
values = compute(doc, keyed)
assert values == reference_values(doc, keyed)
plain_values = compute(doc)
assert all(values[n][f] != plain_values[n][f] for n in values for f in VALUE_FIELDS)
class TestProjections:
def test_plain_projection_marks_math_and_equation_runs(self):
doc = build_doc(
document(
section(
heading("H"),
paragraph(
sentence(
text_run("Energy "),
math_run("E"),
text_run(" obeys "),
{"kind": "equation", "child": 0},
text_run("."),
children=[
{
"kind": "equation",
"content": {"notation": "tex", "source": "E=mc^2"},
}
],
)
),
)
)
)
(sent,) = [
n for n in ids_of_kind(doc, "sentence") if "E=mc" in hashing.plain(doc, n)
]
assert hashing.plain(doc, sent) == "Energy $E$ obeys $$E=mc^2$$."
strict, complexes = hashing.strict(doc, sent)
assert strict == "Energy E obeys E=mc^{2}."
assert complexes == []
def test_a_complex_equation_leaves_an_object_mark_and_a_key(self):
runs = [text_run("see "), math_run("\\frac{a}{b}"), text_run(" now")]
doc, nid = sentence_doc(runs)
strict, complexes = hashing.strict(doc, nid)
assert strict == f"see {OBJ} now"
assert complexes == ["tex:\\frac{a}{b}"]
assert OBJ in hashing.lossy(doc, nid)
def test_cite_xref_and_fnmark_runs_are_marked_and_dropped_from_the_lossy_form(self):
runs = [
text_run("Known"),
{"kind": "cite", "refs": ["aaaaaaa"], "text": " [ 1 ] "},
text_run(" and"),
{"kind": "xref", "target": "bbbbbbb", "text": "Fig. 2"},
{"kind": "fnmark", "text": "3"},
]
doc, nid = sentence_doc(runs)
strict, _ = hashing.strict(doc, nid)
assert (
strict
== f"Known{MARK_OPEN}[1]{MARK_CLOSE} and{MARK_OPEN}Fig.2{MARK_CLOSE}{MARK_OPEN}3{MARK_CLOSE}"
)
assert hashing.lossy(doc, nid) == "knownand"
assert hashing.plain(doc, nid) == "Known [ 1 ] andFig. 23"
def test_a_link_run_is_plain_text(self):
doc, nid = sentence_doc(
[text_run("See "), {"kind": "link", "text": "the site"}]
)
assert hashing.plain(doc, nid) == "See the site"
assert hashing.strict(doc, nid)[0] == "See the site"
def test_styled_text_folds_as_a_script(self):
runs = [
text_run("H"),
text_run("2", style=["sub"]),
text_run("O at 10"),
text_run("-3", style=["sup"]),
text_run("\N{PRIME}", style=["sup"]),
]
doc, nid = sentence_doc(runs)
assert hashing.strict(doc, nid)[0] == "H_{2}O at 10^{\N{MINUS SIGN}3}\N{PRIME}"
def test_an_unknown_run_kind_is_refused_by_every_projection(self):
doc, nid = sentence_doc([{"kind": "image", "text": "x"}])
with pytest.raises(ValueError, match="run kind image"):
hashing.plain(doc, nid)
with pytest.raises(ValueError, match="run kind image"):
hashing.strict(doc, nid)
def test_a_container_joins_the_projections_of_its_text_leaves(self):
doc = simple_doc("First one.", "Second one.")
para = ids_of_kind(doc, "paragraph")[0]
assert hashing.plain(doc, para) == "First one. Second one."
assert hashing.strict(doc, para) == ("First one. Second one.", [])
assert hashing.printed(doc, para) == "First one. Second one."
assert hashing.plain(doc, doc.root) == "Head First one. Second one."
def test_verbatim_and_equation_nodes_project_their_source(self):
doc = build_doc(
document(
section(
heading("H"),
{"kind": "code", "content": {"text": "x = 1"}},
{
"kind": "equation",
"content": {"notation": "tex", "source": "a_1"},
"children": [
{
"kind": "equation-line",
"content": {"notation": "tex", "source": "a_1"},
}
],
},
)
)
)
(code,) = ids_of_kind(doc, "code")
(eq,) = ids_of_kind(doc, "equation")
(line,) = ids_of_kind(doc, "equation-line")
assert hashing.plain(doc, code) == "x = 1"
assert hashing.strict(doc, code) == ("x = 1", [])
assert hashing.printed(doc, code) == "x = 1"
assert hashing.plain(doc, eq) == "$$a_1$$"
assert hashing.strict(doc, eq) == ("a_{1}", [])
assert hashing.printed(doc, eq) == "$$a_1$$"
assert hashing.plain(doc, line) == "$$a_1$$"
values = compute(doc)
assert values[eq]["text_hash"] == values[line]["text_hash"]
section_id = ids_of_kind(doc, "section")[0]
assert hashing.plain(doc, section_id) == "H x = 1 $$a_1$$"
class TestAsPrinted:
def test_runs_without_printed_are_returned_as_they_are(self):
runs = [text_run("a"), math_run("x"), text_run("b", style=["em"])]
assert as_printed(runs) == runs
def test_printed_replaces_the_text_and_rejoins_the_split_run(self):
runs = [
text_run("The "),
text_run("cat", printed="kat"),
text_run(" sat"),
]
assert as_printed(runs) == [text_run("The kat sat")]
def test_the_input_runs_are_not_changed(self):
runs = [text_run("The "), text_run("cat", printed="kat")]
as_printed(runs)
assert runs[1] == text_run("cat", printed="kat")
def test_runs_of_other_styles_or_kinds_are_not_joined(self):
runs = [
text_run("a "),
text_run("b", style=["em"], printed="c"),
text_run(" d"),
math_run("x"),
text_run("e", printed="f"),
]
assert as_printed(runs) == [
text_run("a "),
text_run("c", style=["em"]),
text_run(" d"),
math_run("x"),
text_run("f"),
]
def test_only_the_run_after_a_printed_one_is_joined(self):
runs = [text_run("a"), text_run("b"), text_run("c", printed="d"), text_run("e")]
assert as_printed(runs) == [text_run("a"), text_run("bde")]
def test_the_strict_projection_reads_the_runs_as_printed(self):
runs = [text_run("H"), text_run("2", printed="3", style=["sub"]), text_run("O")]
doc, nid = sentence_doc(runs)
assert hashing.printed(doc, nid) == "H3O"
assert hashing.strict(doc, nid)[0] == "H_{3}O"
assert hashing.plain(doc, nid) == "H2O"
class TestPlainRuns:
def test_unknown_kinds_are_refused(self):
with pytest.raises(ValueError, match="run kind sticker"):
plain_runs([{"kind": "sticker"}])
@pytest.mark.parametrize(
("strict", "lossy"),
[
("H_{2}O", "h2o"),
("x^{2} + y_{ab}", "x2+yab"),
("{a}", "{a}"),
("a}", "a}"),
("_{a{b}c}", "a{b}c"),
("_ {a}", "_{a}"),
("a_", "a_"),
(f"see{MARK_OPEN}12{MARK_CLOSE}now", "seenow"),
(f"see{MARK_OPEN}12", "see"),
(f"see {OBJ} now", f"see{OBJ}now"),
("a-b\N{HYPHEN}c\N{EN DASH}d­e", "abcde"),
("a\N{MINUS SIGN}b", "a\N{MINUS SIGN}b"),
("Straße", "strasse"),
("find", "find"),
("\N{MICRO SIGN}m", "μm"),
("CafÉ Au\N{NO-BREAK SPACE}Lait", "caféaulait"),
("", ""),
],
)
def test_lossy_from_strict(strict, lossy):
assert lossy_from_strict(strict) == lossy
@given(st.text())
@PROPERTIES
def test_lossy_projection_has_no_white_space_dashes_or_annotation_marks(s):
lossy = lossy_from_strict(s)
assert not any(is_ws(c) or unicodedata.category(c) == "Pd" for c in lossy)
assert "­" not in lossy
def test_prose_length_does_not_count_object_marks():
assert prose_length("abc") == 3
assert prose_length(f"a{OBJ}b{OBJ}") == 2
assert prose_length("") == 0
class TestOwnContent:
def test_a_sentence_keeps_its_runs_but_not_the_form_of_a_math_run(self):
runs = [text_run("x "), dict(math_run("a"), form="inline"), text_run("!")]
doc, nid = sentence_doc(runs)
own = own_content(doc, nid)
assert own == {
"kind": "sentence",
"runs": [text_run("x "), math_run("a"), text_run("!")],
}
def test_a_section_names_the_text_hash_of_its_heading(self):
doc = build_doc(
document(
section(heading("Intro"), paragraph(sentence(text_run("t")))),
section(paragraph(sentence(text_run("u")))),
)
)
first, second = ids_of_kind(doc, "section")
head = ids_of_kind(doc, "heading")[0]
assert own_content(doc, first, lambda x: f"hash of {x}") == {
"kind": "section",
"label": None,
"level": 1,
"heading": f"hash of {head}",
}
assert own_content(doc, second, lambda x: "unused")["heading"] is None
def test_a_label_is_part_of_the_own_content_of_a_labelled_kind(self):
doc = build_doc(
document(
section(heading("H"), paragraph(sentence(text_run("t"))), label="2.1")
)
)
sec = ids_of_kind(doc, "section")[0]
assert own_content(doc, sec, lambda x: "h")["label"] == "2.1"
para = ids_of_kind(doc, "paragraph")[0]
assert "label" not in own_content(doc, para)
def test_a_table_carries_its_columns_and_a_cell_its_attributes_with_defaults(self):
nodes = [
{"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0},
{"id": "bbbbbbb", "kind": "table", "parent": "aaaaaaa", "ord": 0},
{
"id": "ccccccc",
"kind": "table",
"parent": "aaaaaaa",
"ord": 1,
"attrs": {"columns": [{"id": "k", "header_path": []}]},
},
{"id": "ddddddd", "kind": "cell", "parent": "bbbbbbb", "ord": 0},
{
"id": "eeeeeee",
"kind": "cell",
"parent": "bbbbbbb",
"ord": 1,
"attrs": {
"column": "k",
"rowspan": 2,
"colspan": 3,
"role": "row-header",
},
},
{
"id": "fffffff",
"kind": "theorem",
"parent": "aaaaaaa",
"ord": 2,
"attrs": {"variant": "lemma"},
},
{"id": "ggggggg", "kind": "theorem", "parent": "aaaaaaa", "ord": 3},
{
"id": "hhhhhhh",
"kind": "code",
"parent": "aaaaaaa",
"ord": 4,
"content": {"text": "x"},
},
{
"id": "iiiiiii",
"kind": "code",
"parent": "aaaaaaa",
"ord": 5,
"label": "L",
"content": {"text": "y", "language": "python"},
},
]
doc = Doc(
"t", [dict(n, content=[]) if n["kind"] == "cell" else n for n in nodes]
)
assert own_content(doc, "bbbbbbb")["columns"] == []
assert own_content(doc, "ccccccc")["columns"] == [
{"id": "k", "header_path": []}
]
assert own_content(doc, "ddddddd") == {
"kind": "cell",
"runs": [],
"column": None,
"rowspan": 1,
"colspan": 1,
"role": "data",
}
assert own_content(doc, "eeeeeee") == {
"kind": "cell",
"runs": [],
"column": "k",
"rowspan": 2,
"colspan": 3,
"role": "row-header",
}
assert own_content(doc, "fffffff") == {
"kind": "theorem",
"label": None,
"variant": "lemma",
}
assert own_content(doc, "ggggggg")["variant"] is None
assert own_content(doc, "hhhhhhh") == {
"kind": "code",
"label": None,
"language": None,
"text": "x",
}
assert own_content(doc, "iiiiiii") == {
"kind": "code",
"label": "L",
"language": "python",
"text": "y",
}
def test_an_equation_run_is_named_by_kind_alone(self):
doc = build_doc(
document(
section(
heading("H"),
paragraph(
sentence(
{"kind": "equation", "child": 0},
children=[
{
"kind": "equation",
"content": {"notation": "tex", "source": "z"},
}
],
)
),
)
)
)
sent = next(n for n in ids_of_kind(doc, "sentence") if doc.kids(n))
assert own_content(doc, sent)["runs"] == [{"kind": "equation"}]
def _two_sentence_doc(first, second):
return build_doc(
document(
section(
heading("Head"),
paragraph(sentence(text_run(first)), sentence(text_run(second))),
paragraph(sentence(text_run("Untouched."))),
)
)
)
def _with_text(doc, nid, text):
nodes = [dict(doc.nodes[x]) for x in doc.order]
for n in nodes:
if n["id"] == nid:
n["content"] = [text_run(text)]
return Doc(doc.key, nodes)
class TestIndex:
def test_build_gives_the_values_of_compute(self):
doc = _two_sentence_doc("One.", "Two.")
index = Index.build(doc)
assert index.values == compute(doc)
assert set(index.values) == set(doc.nodes)
def test_plain_is_the_plain_projection(self):
doc = _two_sentence_doc("One.", "Two.")
index = Index.build(doc)
para = ids_of_kind(doc, "paragraph")[0]
assert index.plain(para) == hashing.plain(doc, para) == "One. Two."
assert index.plain(doc.root) == hashing.plain(doc, doc.root)
def test_update_after_an_edit_equals_a_rebuild_and_leaves_the_old_index_alone(self):
doc = _two_sentence_doc("One.", "Two.")
old = Index.build(doc)
before = {k: dict(v) for k, v in old.values.items()}
target = ids_of_kind(doc, "sentence")[1]
after = _with_text(doc, target, "Changed.")
new = old.update(after)
assert new.values == Index.build(after).values == reference_values(after)
assert old.values == before
assert new.values[target]["text_hash"] != old.values[target]["text_hash"]
untouched = ids_of_kind(doc, "sentence")[-1]
assert new.values[untouched] == old.values[untouched]
assert new.values[doc.root]["node_hash"] != old.values[doc.root]["node_hash"]
assert new.plain(doc.root) == hashing.plain(after, after.root)
def test_an_edit_changes_exactly_the_ancestors_of_the_edited_leaf(self):
doc = _two_sentence_doc("One.", "Two.")
target = ids_of_kind(doc, "sentence")[1]
after = _with_text(doc, target, "Changed.")
old = Index.build(doc)
new = old.update(after).values
old = old.values
changed = {n for n in old if old[n]["node_hash"] != new[n]["node_hash"]}
assert changed == {target, *doc.ancestors(target)}
def test_update_forgets_deleted_nodes_and_learns_new_ones(self):
doc = _two_sentence_doc("One.", "Two.")
old = Index.build(doc)
gone = ids_of_kind(doc, "paragraph")[1]
removed = {gone, *doc.preorder(gone)}
smaller = Doc(doc.key, [doc.nodes[x] for x in doc.order if x not in removed])
assert set(old.update(smaller).values) == set(smaller.nodes)
assert old.update(smaller).values == reference_values(smaller)
assert Index.build(smaller).update(doc).values == old.values
def test_a_keyed_index_keeps_its_hasher_through_updates(self):
keyed = Hasher(b"team")
doc = _two_sentence_doc("One.", "Two.")
target = ids_of_kind(doc, "sentence")[0]
after = _with_text(doc, target, "Other.")
new = Index.build(doc, keyed).update(after)
assert new.hasher is keyed
assert new.values == reference_values(after, keyed)
def test_an_equation_line_adds_nothing_to_its_containers_text(self):
doc = build_doc(
document(
section(
heading("H"),
{
"kind": "equation",
"content": {"notation": "tex", "source": "q"},
"children": [
{
"kind": "equation-line",
"content": {"notation": "tex", "source": "q"},
}
],
},
)
)
)
eq = ids_of_kind(doc, "equation")[0]
index = Index.build(doc)
assert index.plain(eq) == "$$q$$"
assert index.values == reference_values(doc)
class TestMetaHash:
DOCUMENT: ClassVar[dict] = {
"work": {"same_as": ["doi:10.1/x"]},
"version": {"type": "version-of-record"},
"relations": [],
"meta": {"title": "T"},
"language": "en",
"license": {"spdx": "CC0-1.0"},
"policy": "full",
"key": "ignored",
}
def test_meta_object_takes_six_members_and_nulls_the_missing(self):
obj = meta_object(self.DOCUMENT)
assert list(obj) == [
"work",
"version",
"relations",
"meta",
"language",
"license",
]
assert meta_object({}) == dict.fromkeys(obj)
def test_meta_hash_is_the_hash_of_the_canonical_object(self):
expected = _hash(jcs.dumps(meta_object(self.DOCUMENT)))
assert meta_hash(self.DOCUMENT) == expected
keyed = Hasher(b"k")
assert meta_hash(self.DOCUMENT, keyed) == keyed(
jcs.dumpb(meta_object(self.DOCUMENT))
)
assert meta_hash(self.DOCUMENT, keyed) != expected
def test_members_outside_the_six_do_not_change_the_hash(self):
other = dict(self.DOCUMENT, policy="withheld", key="another")
assert meta_hash(other) == meta_hash(self.DOCUMENT)
assert meta_hash(dict(self.DOCUMENT, language="fr")) != meta_hash(self.DOCUMENT)
def _lossy_by_hand(s):
"""The lossy projection of a strict projection, character by character (spec/05 §5.5)."""
out, stack, i, n = [], [], 0, len(s)
while i < n:
ch = s[i]
if ch in "_^" and i + 1 < n and s[i + 1] == "{":
stack.append("s")
i += 2
continue
if ch == "{":
stack.append("t")
out.append(ch)
elif ch == "}":
if not stack or stack.pop() == "t":
out.append(ch)
elif ch == hashing.MARK_OPEN:
j = s.find(hashing.MARK_CLOSE, i)
i = n if j < 0 else j + 1
continue
else:
out.append(ch)
i += 1
t = unicodedata.normalize("NFKC", "".join(out).casefold())
return "".join(
c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "­"
)
LOSSY_ALPHABET = [
"_",
"^",
"{",
"}",
hashing.MARK_OPEN,
hashing.MARK_CLOSE,
hashing.OBJ,
"a",
"B",
"\u00df", # sharp s, which case folding expands
"\u00ad", # soft hyphen
"-",
"\u2013", # en dash
"\u2212", # minus sign
" ",
"\u00a0", # no-break space
"\u2009", # thin space
"\ufb01", # the fi ligature, which NFKC expands
"\u00b2", # superscript two
"1",
]
@given(st.text(alphabet=st.sampled_from(LOSSY_ALPHABET), max_size=40))
@settings(derandomize=True, max_examples=500, database=None, deadline=None)
def test_the_lossy_projection_matches_its_character_by_character_definition(s):
assert hashing.lossy_from_strict(s) == _lossy_by_hand(s)
def test_the_lossy_projection_drops_every_dash_and_white_space_character():
every = "".join(chr(c) for c in range(0x110000) if not 0xD800 <= c <= 0xDFFF)
assert hashing.lossy_from_strict(every) == _lossy_by_hand(every)