162 lines
6.5 KiB
Python
162 lines
6.5 KiB
Python
"""Work and document keys: a Scholar-style citation key plus a short hash of the work's identity.
|
||
|
||
Non-normative; see spec/02-model.md §2.4.
|
||
A work key looks like `vaswani2017attention-k3f9`. The first part is what Google Scholar would call the work; the
|
||
characters after the hyphen come from a hash of the work's identity, so two works Scholar would name alike never
|
||
share a key. A document key is the work key with a suffix for anything other than the version of record.
|
||
"""
|
||
|
||
import hashlib
|
||
import re
|
||
import unicodedata
|
||
|
||
from .text import ALPHABET
|
||
|
||
SCHEME = "scienting-key/1"
|
||
SUFFIX_LENGTH = 4
|
||
MAX_SUFFIX = 51
|
||
IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex")
|
||
WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$")
|
||
DOCUMENT_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$")
|
||
STOP_WORDS = frozenset("a an the of on in for to and or at by with from as into onto about toward towards via "
|
||
"is are be its it this that these those".split())
|
||
NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"])
|
||
_YEAR = re.compile(r"^\d{4}$")
|
||
_DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE)
|
||
|
||
|
||
def fold(text):
|
||
"""Lower-case ASCII with accents removed: `Müller` gives `muller`."""
|
||
raw = text if isinstance(text, str) else ""
|
||
return unicodedata.normalize("NFKD", raw).encode("ascii", "ignore").decode().lower()
|
||
|
||
|
||
def title_words(title):
|
||
"""A title's words, folded, with apostrophes removed and other punctuation splitting."""
|
||
t = re.sub(r"['’`]", "", fold(title))
|
||
return re.findall(r"[a-z0-9]+", t)
|
||
|
||
|
||
def family_name(name):
|
||
"""A best guess at the family name in a display name: the part before a comma, else the last word that is
|
||
not a suffix such as `Jr.` or `III`."""
|
||
name = (name or "").strip()
|
||
if "," in name:
|
||
head, _, tail = name.partition(",")
|
||
if tail.strip().rstrip(".").lower() not in NAME_SUFFIXES or " " not in head.strip():
|
||
return head.strip()
|
||
name = head
|
||
parts = name.split()
|
||
while len(parts) > 1 and parts[-1].rstrip(".").lower() in NAME_SUFFIXES:
|
||
parts.pop()
|
||
return parts[-1] if parts else ""
|
||
|
||
|
||
def first_family(meta):
|
||
"""The first author's family name from a CSL-JSON item, as the key spells it, or `anon`."""
|
||
authors = meta.get("author") or []
|
||
if not authors:
|
||
return "anon"
|
||
a = authors[0]
|
||
name = a.get("family") or family_name(a.get("literal") or a.get("name") or "")
|
||
fam = re.sub(r"[^a-z]", "", fold(name))
|
||
return fam or "anon"
|
||
|
||
|
||
def year_of(meta):
|
||
"""The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string."""
|
||
parts = (meta.get("issued") or {}).get("date-parts") or []
|
||
y = parts[0][0] if parts and parts[0] else None
|
||
s = str(y) if isinstance(y, int) and not isinstance(y, bool) else (y if isinstance(y, str) else "")
|
||
return s if _YEAR.match(s) else ""
|
||
|
||
|
||
def scholar_part(meta):
|
||
"""Family name, year and the first title word that is not a stop word: `vaswani2017attention`."""
|
||
word = next((w for w in title_words(meta.get("title")) if w not in STOP_WORDS), "")
|
||
return first_family(meta) + year_of(meta) + word
|
||
|
||
|
||
def norm_doi(value):
|
||
"""A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4)."""
|
||
s = _DOI_PREFIX.sub("", value.strip()).strip()
|
||
return s.translate(str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"))
|
||
|
||
|
||
def identity(same_as, meta):
|
||
"""What the key's hash is computed over: the first identifier of IDENTIFIER_ORDER among `same_as`, else the
|
||
folded metadata `meta:<title words>|<year>|<family>`."""
|
||
by_kind = {}
|
||
for s in same_as or []:
|
||
kind, _, value = s.partition(":")
|
||
by_kind.setdefault(kind, value)
|
||
for kind in IDENTIFIER_ORDER:
|
||
if kind in by_kind and by_kind[kind]:
|
||
value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip()
|
||
return "%s:%s" % (kind, value)
|
||
return "meta:%s|%s|%s" % (" ".join(title_words(meta.get("title"))), year_of(meta), first_family(meta))
|
||
|
||
|
||
def suffix(ident, length=SUFFIX_LENGTH):
|
||
"""The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32."""
|
||
if not 1 <= length <= MAX_SUFFIX:
|
||
raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX)
|
||
digest = int.from_bytes(hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big")
|
||
bits = digest >> (256 - 5 * length)
|
||
return "".join(ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length))
|
||
|
||
|
||
def work_key(same_as, meta, length=SUFFIX_LENGTH):
|
||
return scholar_part(meta) + "-" + suffix(identity(same_as, meta), length)
|
||
|
||
|
||
def unique_work_key(same_as, meta, taken):
|
||
"""The work key, lengthened by two characters at a time while another identity already holds it."""
|
||
for length in range(SUFFIX_LENGTH, MAX_SUFFIX + 1, 2):
|
||
key = work_key(same_as, meta, length)
|
||
if key not in taken:
|
||
return key
|
||
raise ValueError("no free key for " + identity(same_as, meta))
|
||
|
||
|
||
def version_suffix(version):
|
||
"""The document key's suffix for a version: nothing for the version of record, `.si` for Supporting
|
||
Information, `.v2` for a preprint or draft with a sequence number, else the version type."""
|
||
t = version.get("type")
|
||
if t == "version-of-record":
|
||
return ""
|
||
if t == "supplement":
|
||
return ".si"
|
||
seq = version.get("seq")
|
||
if t in ("preprint", "draft", "submitted", "accepted") and isinstance(seq, int):
|
||
return ".v%d" % seq
|
||
return "." + t
|
||
|
||
|
||
def document_key(wkey, version):
|
||
return wkey + version_suffix(version)
|
||
|
||
|
||
def split_key(dkey):
|
||
"""A document key's work key and its suffix (with the leading dot, or empty)."""
|
||
m = WORK_KEY_RE.match(dkey.split(".", 1)[0]) if dkey else None
|
||
if not m or not DOCUMENT_KEY_RE.match(dkey):
|
||
raise ValueError("%r is not a document key" % dkey)
|
||
head, dot, tail = dkey.partition(".")
|
||
return head, (dot + tail if dot else "")
|
||
|
||
|
||
def suffix_of(wkey):
|
||
return wkey.rsplit("-", 1)[1]
|
||
|
||
|
||
def suffix_agrees(wkey, same_as, meta):
|
||
"""Whether a stored work key's hash part is what the identity gives: always checked when the identity is an
|
||
identifier such as a DOI, which never changes; a metadata identity may have been corrected since, so it
|
||
is not held to the key."""
|
||
ident = identity(same_as, meta)
|
||
if ident.startswith("meta:"):
|
||
return True
|
||
sfx = suffix_of(wkey)
|
||
return sfx == suffix(ident, len(sfx))
|