Reviewed-on: #1
This commit is contained in:
57 files changed
+11446
No files matched your search
@@ -0,0 +1,161 @@
|
||||
"""Work and document keys: a Scholar-style citation key plus a short hash of the work's identity.
|
||||
|
||||
Non-normative; see spec/02-model.md §2.4.
|
||||
A work key looks like `vaswani2017attention-k3f9`. The first part is what Google Scholar would call the work; the
|
||||
characters after the hyphen come from a hash of the work's identity, so two works Scholar would name alike never
|
||||
share a key. A document key is the work key with a suffix for anything other than the version of record.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from .text import ALPHABET
|
||||
|
||||
SCHEME = "scienting-key/1"
|
||||
SUFFIX_LENGTH = 4
|
||||
MAX_SUFFIX = 51
|
||||
IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex")
|
||||
WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$")
|
||||
DOCUMENT_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$")
|
||||
STOP_WORDS = frozenset("a an the of on in for to and or at by with from as into onto about toward towards via "
|
||||
"is are be its it this that these those".split())
|
||||
NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"])
|
||||
_YEAR = re.compile(r"^\d{4}$")
|
||||
_DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE)
|
||||
|
||||
|
||||
def fold(text):
|
||||
"""Lower-case ASCII with accents removed: `Müller` gives `muller`."""
|
||||
raw = text if isinstance(text, str) else ""
|
||||
return unicodedata.normalize("NFKD", raw).encode("ascii", "ignore").decode().lower()
|
||||
|
||||
|
||||
def title_words(title):
|
||||
"""A title's words, folded, with apostrophes removed and other punctuation splitting."""
|
||||
t = re.sub(r"['’`]", "", fold(title))
|
||||
return re.findall(r"[a-z0-9]+", t)
|
||||
|
||||
|
||||
def family_name(name):
|
||||
"""A best guess at the family name in a display name: the part before a comma, else the last word that is
|
||||
not a suffix such as `Jr.` or `III`."""
|
||||
name = (name or "").strip()
|
||||
if "," in name:
|
||||
head, _, tail = name.partition(",")
|
||||
if tail.strip().rstrip(".").lower() not in NAME_SUFFIXES or " " not in head.strip():
|
||||
return head.strip()
|
||||
name = head
|
||||
parts = name.split()
|
||||
while len(parts) > 1 and parts[-1].rstrip(".").lower() in NAME_SUFFIXES:
|
||||
parts.pop()
|
||||
return parts[-1] if parts else ""
|
||||
|
||||
|
||||
def first_family(meta):
|
||||
"""The first author's family name from a CSL-JSON item, as the key spells it, or `anon`."""
|
||||
authors = meta.get("author") or []
|
||||
if not authors:
|
||||
return "anon"
|
||||
a = authors[0]
|
||||
name = a.get("family") or family_name(a.get("literal") or a.get("name") or "")
|
||||
fam = re.sub(r"[^a-z]", "", fold(name))
|
||||
return fam or "anon"
|
||||
|
||||
|
||||
def year_of(meta):
|
||||
"""The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string."""
|
||||
parts = (meta.get("issued") or {}).get("date-parts") or []
|
||||
y = parts[0][0] if parts and parts[0] else None
|
||||
s = str(y) if isinstance(y, int) and not isinstance(y, bool) else (y if isinstance(y, str) else "")
|
||||
return s if _YEAR.match(s) else ""
|
||||
|
||||
|
||||
def scholar_part(meta):
|
||||
"""Family name, year and the first title word that is not a stop word: `vaswani2017attention`."""
|
||||
word = next((w for w in title_words(meta.get("title")) if w not in STOP_WORDS), "")
|
||||
return first_family(meta) + year_of(meta) + word
|
||||
|
||||
|
||||
def norm_doi(value):
|
||||
"""A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4)."""
|
||||
s = _DOI_PREFIX.sub("", value.strip()).strip()
|
||||
return s.translate(str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"))
|
||||
|
||||
|
||||
def identity(same_as, meta):
|
||||
"""What the key's hash is computed over: the first identifier of IDENTIFIER_ORDER among `same_as`, else the
|
||||
folded metadata `meta:<title words>|<year>|<family>`."""
|
||||
by_kind = {}
|
||||
for s in same_as or []:
|
||||
kind, _, value = s.partition(":")
|
||||
by_kind.setdefault(kind, value)
|
||||
for kind in IDENTIFIER_ORDER:
|
||||
if kind in by_kind and by_kind[kind]:
|
||||
value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip()
|
||||
return "%s:%s" % (kind, value)
|
||||
return "meta:%s|%s|%s" % (" ".join(title_words(meta.get("title"))), year_of(meta), first_family(meta))
|
||||
|
||||
|
||||
def suffix(ident, length=SUFFIX_LENGTH):
|
||||
"""The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32."""
|
||||
if not 1 <= length <= MAX_SUFFIX:
|
||||
raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX)
|
||||
digest = int.from_bytes(hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big")
|
||||
bits = digest >> (256 - 5 * length)
|
||||
return "".join(ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length))
|
||||
|
||||
|
||||
def work_key(same_as, meta, length=SUFFIX_LENGTH):
|
||||
return scholar_part(meta) + "-" + suffix(identity(same_as, meta), length)
|
||||
|
||||
|
||||
def unique_work_key(same_as, meta, taken):
|
||||
"""The work key, lengthened by two characters at a time while another identity already holds it."""
|
||||
for length in range(SUFFIX_LENGTH, MAX_SUFFIX + 1, 2):
|
||||
key = work_key(same_as, meta, length)
|
||||
if key not in taken:
|
||||
return key
|
||||
raise ValueError("no free key for " + identity(same_as, meta))
|
||||
|
||||
|
||||
def version_suffix(version):
|
||||
"""The document key's suffix for a version: nothing for the version of record, `.si` for Supporting
|
||||
Information, `.v2` for a preprint or draft with a sequence number, else the version type."""
|
||||
t = version.get("type")
|
||||
if t == "version-of-record":
|
||||
return ""
|
||||
if t == "supplement":
|
||||
return ".si"
|
||||
seq = version.get("seq")
|
||||
if t in ("preprint", "draft", "submitted", "accepted") and isinstance(seq, int):
|
||||
return ".v%d" % seq
|
||||
return "." + t
|
||||
|
||||
|
||||
def document_key(wkey, version):
|
||||
return wkey + version_suffix(version)
|
||||
|
||||
|
||||
def split_key(dkey):
|
||||
"""A document key's work key and its suffix (with the leading dot, or empty)."""
|
||||
m = WORK_KEY_RE.match(dkey.split(".", 1)[0]) if dkey else None
|
||||
if not m or not DOCUMENT_KEY_RE.match(dkey):
|
||||
raise ValueError("%r is not a document key" % dkey)
|
||||
head, dot, tail = dkey.partition(".")
|
||||
return head, (dot + tail if dot else "")
|
||||
|
||||
|
||||
def suffix_of(wkey):
|
||||
return wkey.rsplit("-", 1)[1]
|
||||
|
||||
|
||||
def suffix_agrees(wkey, same_as, meta):
|
||||
"""Whether a stored work key's hash part is what the identity gives: always checked when the identity is an
|
||||
identifier such as a DOI, which never changes; a metadata identity may have been corrected since, so it
|
||||
is not held to the key."""
|
||||
ident = identity(same_as, meta)
|
||||
if ident.startswith("meta:"):
|
||||
return True
|
||||
sfx = suffix_of(wkey)
|
||||
return sfx == suffix(ident, len(sfx))
|
||||
Reference in new issue
Block a user