paperspec/tools/oracle/keys.py

"""Work and document keys: a Scholar-style citation key plus a short hash of the work's identity.

Non-normative; see spec/02-model.md §2.4.
A work key looks like `vaswani2017attention-k3f9`. The first part is what Google Scholar would call the work; the
characters after the hyphen come from a hash of the work's identity, so two works Scholar would name alike never
share a key. A document key is the work key with a suffix for anything other than the version of record.
"""

import hashlib
import re
import unicodedata

from .text import ALPHABET

SCHEME = "scienting-key/1"
SUFFIX_LENGTH = 4
MAX_SUFFIX = 51
IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex")
WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$")
DOCUMENT_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$")
STOP_WORDS = frozenset("a an the of on in for to and or at by with from as into onto about toward towards via "
                       "is are be its it this that these those".split())
NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"])
_YEAR = re.compile(r"^\d{4}$")
_DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE)


def fold(text):
    """Lower-case ASCII with accents removed: `Müller` gives `muller`."""
    raw = text if isinstance(text, str) else ""
    return unicodedata.normalize("NFKD", raw).encode("ascii", "ignore").decode().lower()


def title_words(title):
    """A title's words, folded, with apostrophes removed and other punctuation splitting."""
    t = re.sub(r"['’`]", "", fold(title))
    return re.findall(r"[a-z0-9]+", t)


def family_name(name):
    """A best guess at the family name in a display name: the part before a comma, else the last word that is
    not a suffix such as `Jr.` or `III`."""
    name = (name or "").strip()
    if "," in name:
        head, _, tail = name.partition(",")
        if tail.strip().rstrip(".").lower() not in NAME_SUFFIXES or " " not in head.strip():
            return head.strip()
        name = head
    parts = name.split()
    while len(parts) > 1 and parts[-1].rstrip(".").lower() in NAME_SUFFIXES:
        parts.pop()
    return parts[-1] if parts else ""


def first_family(meta):
    """The first author's family name from a CSL-JSON item, as the key spells it, or `anon`."""
    authors = meta.get("author") or []
    if not authors:
        return "anon"
    a = authors[0]
    name = a.get("family") or family_name(a.get("literal") or a.get("name") or "")
    fam = re.sub(r"[^a-z]", "", fold(name))
    return fam or "anon"


def year_of(meta):
    """The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string."""
    parts = (meta.get("issued") or {}).get("date-parts") or []
    y = parts[0][0] if parts and parts[0] else None
    s = str(y) if isinstance(y, int) and not isinstance(y, bool) else (y if isinstance(y, str) else "")
    return s if _YEAR.match(s) else ""


def scholar_part(meta):
    """Family name, year and the first title word that is not a stop word: `vaswani2017attention`."""
    word = next((w for w in title_words(meta.get("title")) if w not in STOP_WORDS), "")
    return first_family(meta) + year_of(meta) + word


def norm_doi(value):
    """A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4)."""
    s = _DOI_PREFIX.sub("", value.strip()).strip()
    return s.translate(str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"))


def identity(same_as, meta):
    """What the key's hash is computed over: the first identifier of IDENTIFIER_ORDER among `same_as`, else the
    folded metadata `meta:<title words>|<year>|<family>`."""
    by_kind = {}
    for s in same_as or []:
        kind, _, value = s.partition(":")
        by_kind.setdefault(kind, value)
    for kind in IDENTIFIER_ORDER:
        if kind in by_kind and by_kind[kind]:
            value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip()
            return "%s:%s" % (kind, value)
    return "meta:%s|%s|%s" % (" ".join(title_words(meta.get("title"))), year_of(meta), first_family(meta))


def suffix(ident, length=SUFFIX_LENGTH):
    """The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32."""
    if not 1 <= length <= MAX_SUFFIX:
        raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX)
    digest = int.from_bytes(hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big")
    bits = digest >> (256 - 5 * length)
    return "".join(ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length))


def work_key(same_as, meta, length=SUFFIX_LENGTH):
    return scholar_part(meta) + "-" + suffix(identity(same_as, meta), length)


def unique_work_key(same_as, meta, taken):
    """The work key, lengthened by two characters at a time while another identity already holds it."""
    for length in range(SUFFIX_LENGTH, MAX_SUFFIX + 1, 2):
        key = work_key(same_as, meta, length)
        if key not in taken:
            return key
    raise ValueError("no free key for " + identity(same_as, meta))


def version_suffix(version):
    """The document key's suffix for a version: nothing for the version of record, `.si` for Supporting
    Information, `.v2` for a preprint or draft with a sequence number, else the version type."""
    t = version.get("type")
    if t == "version-of-record":
        return ""
    if t == "supplement":
        return ".si"
    seq = version.get("seq")
    if t in ("preprint", "draft", "submitted", "accepted") and isinstance(seq, int):
        return ".v%d" % seq
    return "." + t


def document_key(wkey, version):
    return wkey + version_suffix(version)


def split_key(dkey):
    """A document key's work key and its suffix (with the leading dot, or empty)."""
    m = WORK_KEY_RE.match(dkey.split(".", 1)[0]) if dkey else None
    if not m or not DOCUMENT_KEY_RE.match(dkey):
        raise ValueError("%r is not a document key" % dkey)
    head, dot, tail = dkey.partition(".")
    return head, (dot + tail if dot else "")


def suffix_of(wkey):
    return wkey.rsplit("-", 1)[1]


def suffix_agrees(wkey, same_as, meta):
    """Whether a stored work key's hash part is what the identity gives: always checked when the identity is an
    identifier such as a DOI, which never changes; a metadata identity may have been corrected since, so it
    is not held to the key."""
    ident = identity(same_as, meta)
    if ident.startswith("meta:"):
        return True
    sfx = suffix_of(wkey)
    return sfx == suffix(ident, len(sfx))