"""Work and document keys: a Scholar-style citation key plus a short hash of the work's identity. Non-normative; see spec/02-model.md §2.4. A work key looks like `vaswani2017attention-k3f9`. The first part is what Google Scholar would call the work; the characters after the hyphen come from a hash of the work's identity, so two works Scholar would name alike never share a key. A document key is the work key with a suffix for anything other than the version of record. """ import hashlib import re import unicodedata from .text import ALPHABET SCHEME = "scienting-key/1" SUFFIX_LENGTH = 4 MAX_SUFFIX = 51 IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex") WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$") DOCUMENT_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$") STOP_WORDS = frozenset("a an the of on in for to and or at by with from as into onto about toward towards via " "is are be its it this that these those".split()) NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"]) _YEAR = re.compile(r"^\d{4}$") _DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE) def fold(text): """Lower-case ASCII with accents removed: `Müller` gives `muller`.""" raw = text if isinstance(text, str) else "" return unicodedata.normalize("NFKD", raw).encode("ascii", "ignore").decode().lower() def title_words(title): """A title's words, folded, with apostrophes removed and other punctuation splitting.""" t = re.sub(r"['’`]", "", fold(title)) return re.findall(r"[a-z0-9]+", t) def family_name(name): """A best guess at the family name in a display name: the part before a comma, else the last word that is not a suffix such as `Jr.` or `III`.""" name = (name or "").strip() if "," in name: head, _, tail = name.partition(",") if tail.strip().rstrip(".").lower() not in NAME_SUFFIXES or " " not in head.strip(): return head.strip() name = head parts = name.split() while len(parts) > 1 and parts[-1].rstrip(".").lower() in NAME_SUFFIXES: parts.pop() return parts[-1] if parts else "" def first_family(meta): """The first author's family name from a CSL-JSON item, as the key spells it, or `anon`.""" authors = meta.get("author") or [] if not authors: return "anon" a = authors[0] name = a.get("family") or family_name(a.get("literal") or a.get("name") or "") fam = re.sub(r"[^a-z]", "", fold(name)) return fam or "anon" def year_of(meta): """The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string.""" parts = (meta.get("issued") or {}).get("date-parts") or [] y = parts[0][0] if parts and parts[0] else None s = str(y) if isinstance(y, int) and not isinstance(y, bool) else (y if isinstance(y, str) else "") return s if _YEAR.match(s) else "" def scholar_part(meta): """Family name, year and the first title word that is not a stop word: `vaswani2017attention`.""" word = next((w for w in title_words(meta.get("title")) if w not in STOP_WORDS), "") return first_family(meta) + year_of(meta) + word def norm_doi(value): """A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4).""" s = _DOI_PREFIX.sub("", value.strip()).strip() return s.translate(str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")) def identity(same_as, meta): """What the key's hash is computed over: the first identifier of IDENTIFIER_ORDER among `same_as`, else the folded metadata `meta:|<year>|<family>`.""" by_kind = {} for s in same_as or []: kind, _, value = s.partition(":") by_kind.setdefault(kind, value) for kind in IDENTIFIER_ORDER: if kind in by_kind and by_kind[kind]: value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip() return "%s:%s" % (kind, value) return "meta:%s|%s|%s" % (" ".join(title_words(meta.get("title"))), year_of(meta), first_family(meta)) def suffix(ident, length=SUFFIX_LENGTH): """The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32.""" if not 1 <= length <= MAX_SUFFIX: raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX) digest = int.from_bytes(hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big") bits = digest >> (256 - 5 * length) return "".join(ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length)) def work_key(same_as, meta, length=SUFFIX_LENGTH): return scholar_part(meta) + "-" + suffix(identity(same_as, meta), length) def unique_work_key(same_as, meta, taken): """The work key, lengthened by two characters at a time while another identity already holds it.""" for length in range(SUFFIX_LENGTH, MAX_SUFFIX + 1, 2): key = work_key(same_as, meta, length) if key not in taken: return key raise ValueError("no free key for " + identity(same_as, meta)) def version_suffix(version): """The document key's suffix for a version: nothing for the version of record, `.si` for Supporting Information, `.v2` for a preprint or draft with a sequence number, else the version type.""" t = version.get("type") if t == "version-of-record": return "" if t == "supplement": return ".si" seq = version.get("seq") if t in ("preprint", "draft", "submitted", "accepted") and isinstance(seq, int): return ".v%d" % seq return "." + t def document_key(wkey, version): return wkey + version_suffix(version) def split_key(dkey): """A document key's work key and its suffix (with the leading dot, or empty).""" m = WORK_KEY_RE.match(dkey.split(".", 1)[0]) if dkey else None if not m or not DOCUMENT_KEY_RE.match(dkey): raise ValueError("%r is not a document key" % dkey) head, dot, tail = dkey.partition(".") return head, (dot + tail if dot else "") def suffix_of(wkey): return wkey.rsplit("-", 1)[1] def suffix_agrees(wkey, same_as, meta): """Whether a stored work key's hash part is what the identity gives: always checked when the identity is an identifier such as a DOI, which never changes; a metadata identity may have been corrected since, so it is not held to the key.""" ident = identity(same_as, meta) if ident.startswith("meta:"): return True sfx = suffix_of(wkey) return sfx == suffix(ident, len(sfx))