Initial commit

tools/oracle/fingerprint.py

@@ -0,0 +1,61 @@
+"""Page fingerprints, scienting-page/1, and download-stamp rules, scienting-stamps/1.
+
+Non-normative; see spec/07-geometry.md.
+"""
+
+import math
+import re
+import unicodedata
+
+from . import jcs
+from .notation import fold_plain_text
+from .text import is_ws, norm, sha256_hex
+
+
+def round_point(x):
+    """Round half up to a whole point."""
+    return int(math.floor(x + 0.5))
+
+
+def _in_margin(box, size, margin):
+    x0, y0, x1, y1 = box
+    w, h = size
+    return x1 <= margin or x0 >= w - margin or y1 <= margin or y0 >= h - margin
+
+
+def is_stamp(line, size, rules):
+    text = norm(line["text"])
+    for r in rules:
+        if r.get("where", "any") == "margin":
+            if "box" not in line or not _in_margin(line["box"], size, r.get("margin", 36)):
+                continue
+        if re.search(r["pattern"], text, re.IGNORECASE | re.ASCII):
+            return r["id"]
+    return None
+
+
+def fp_text(text):
+    s = unicodedata.normalize("NFKC", fold_plain_text(text))
+    return "".join(c for c in s if not is_ws(c) and unicodedata.category(c) not in ("Cc", "Cf"))
+
+
+def fingerprint(page, rules):
+    """`page`: {"size": [w, h], "lines": [{"text", "box"?}], "images": [sha256...]}.
+
+    Returns (fingerprint, removed stamp rule IDs in line order).
+    """
+    size = page["size"]
+    kept, removed = [], []
+    for line in page.get("lines", []):
+        rid = is_stamp(line, size, rules)
+        if rid:
+            removed.append(rid)
+        else:
+            kept.append(line["text"])
+    payload = {
+        "format": "scienting-page/1",
+        "size": [round_point(size[0]), round_point(size[1])],
+        "text": fp_text(" ".join(kept)),
+        "images": sorted(page.get("images", [])),
+    }
+    return sha256_hex(jcs.dumpb(payload)), removed