Reorganize paperspec commands and pin dependency versions (#2)
Reviewed-on: #2
This commit was merged in pull request #2.
This commit is contained in:
alexm committed
1 parent
4fa326dc76
commit
78101dc0db
65 files changed
+2360
-298
No files matched your search
@@ -0,0 +1,481 @@
|
||||
"""Simple notation and its canonical linear form.
|
||||
|
||||
Non-normative; the rules are spec/05-hashes.md, "The strict projection".
|
||||
A math source or a styled script either folds to one canonical linear form, or is complex.
|
||||
"""
|
||||
|
||||
import unicodedata
|
||||
|
||||
from .text import collapse_ws, is_ws, remove_ws, trim
|
||||
|
||||
MINUS = "\u2212"
|
||||
DOT = "\u22c5"
|
||||
PRIME = "\u2032"
|
||||
|
||||
# Characters that fold to one canonical character inside notation.
|
||||
CHAR_FOLD = {
|
||||
"-": MINUS,
|
||||
"\u2013": MINUS,
|
||||
"\u2212": MINUS,
|
||||
"\u00b7": DOT,
|
||||
"\u2219": DOT,
|
||||
"\u22c5": DOT,
|
||||
"'": PRIME,
|
||||
"\u2032": PRIME,
|
||||
}
|
||||
|
||||
OPERATORS = set("+=/<>.,()") | {MINUS, DOT, PRIME, "\u00d7", "\u2264", "\u2265", "\u00b1"}
|
||||
|
||||
TEX_GREEK = {
|
||||
"alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4",
|
||||
"epsilon": "\u03f5", "varepsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7",
|
||||
"theta": "\u03b8", "vartheta": "\u03d1", "iota": "\u03b9", "kappa": "\u03ba",
|
||||
"varkappa": "\u03f0", "lambda": "\u03bb", "mu": "\u03bc", "nu": "\u03bd",
|
||||
"xi": "\u03be", "pi": "\u03c0", "varpi": "\u03d6", "rho": "\u03c1",
|
||||
"varrho": "\u03f1", "sigma": "\u03c3", "varsigma": "\u03c2", "tau": "\u03c4",
|
||||
"upsilon": "\u03c5", "phi": "\u03d5", "varphi": "\u03c6", "chi": "\u03c7",
|
||||
"psi": "\u03c8", "omega": "\u03c9",
|
||||
"Gamma": "\u0393", "Delta": "\u0394", "Theta": "\u0398", "Lambda": "\u039b",
|
||||
"Xi": "\u039e", "Pi": "\u03a0", "Sigma": "\u03a3", "Upsilon": "\u03a5",
|
||||
"Phi": "\u03a6", "Psi": "\u03a8", "Omega": "\u03a9",
|
||||
}
|
||||
TEX_SYMBOLS = {
|
||||
"times": "\u00d7", "cdot": DOT, "pm": "\u00b1", "le": "\u2264", "leq": "\u2264",
|
||||
"ge": "\u2265", "geq": "\u2265", "lt": "<", "gt": ">", "prime": PRIME,
|
||||
"AA": "\u00c5",
|
||||
}
|
||||
TEX_WRAPPERS = {"mathrm", "text", "textrm", "mathup"}
|
||||
TEX_DECLARATIONS = {"rm"}
|
||||
TEX_SPACES = {"quad", "qquad", "enspace", "thinspace", "medspace", "thickspace"}
|
||||
TEX_SPACE_SYMBOLS = {",", ";", ":", "!", " "}
|
||||
|
||||
TYPST_GREEK = {
|
||||
"alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4",
|
||||
"epsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7", "theta": "\u03b8",
|
||||
"iota": "\u03b9", "kappa": "\u03ba", "lambda": "\u03bb", "mu": "\u03bc",
|
||||
"nu": "\u03bd", "xi": "\u03be", "omicron": "\u03bf", "pi": "\u03c0",
|
||||
"rho": "\u03c1", "sigma": "\u03c3", "tau": "\u03c4", "upsilon": "\u03c5",
|
||||
"phi": "\u03c6", "chi": "\u03c7", "psi": "\u03c8", "omega": "\u03c9",
|
||||
"Alpha": "\u0391", "Beta": "\u0392", "Gamma": "\u0393", "Delta": "\u0394",
|
||||
"Epsilon": "\u0395", "Zeta": "\u0396", "Eta": "\u0397", "Theta": "\u0398",
|
||||
"Iota": "\u0399", "Kappa": "\u039a", "Lambda": "\u039b", "Mu": "\u039c",
|
||||
"Nu": "\u039d", "Xi": "\u039e", "Omicron": "\u039f", "Pi": "\u03a0",
|
||||
"Rho": "\u03a1", "Sigma": "\u03a3", "Tau": "\u03a4", "Upsilon": "\u03a5",
|
||||
"Phi": "\u03a6", "Chi": "\u03a7", "Psi": "\u03a8", "Omega": "\u03a9",
|
||||
"epsilon.alt": "\u03f5", "theta.alt": "\u03d1", "phi.alt": "\u03d5",
|
||||
"pi.alt": "\u03d6", "rho.alt": "\u03f1", "sigma.alt": "\u03c2",
|
||||
"kappa.alt": "\u03f0", "beta.alt": "\u03d0",
|
||||
}
|
||||
TYPST_SYMBOLS = {
|
||||
"plus": "+", "minus": MINUS, "times": "\u00d7", "dot": DOT, "dot.op": DOT,
|
||||
"dot.c": DOT, "plus.minus": "\u00b1", "lt": "<", "gt": ">", "lt.eq": "\u2264",
|
||||
"gt.eq": "\u2265", "eq": "=", "prime": PRIME, "slash": "/",
|
||||
}
|
||||
TYPST_WRAPPERS = {"upright"}
|
||||
TYPST_SPACES = {"thin", "med", "thick", "quad", "wide", "space"}
|
||||
|
||||
|
||||
class NotSimple(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# The Latin and Greek letter blocks of spec/05 \u00a75.4: Basic Latin through Latin Extended-B, Greek and Coptic without
|
||||
# the Coptic letters, Latin Extended Additional, and Greek Extended.
|
||||
LETTER_RANGES = ((0x0041, 0x024F), (0x0370, 0x03E1), (0x03F0, 0x03FF), (0x1E00, 0x1EFF), (0x1F00, 0x1FFF))
|
||||
|
||||
|
||||
def is_letter(ch):
|
||||
"""A letter of General Category L in the Latin or Greek blocks, or the micro sign."""
|
||||
if ch == "\u00b5":
|
||||
return True
|
||||
if not unicodedata.category(ch).startswith("L"):
|
||||
return False
|
||||
cp = ord(ch)
|
||||
return any(lo <= cp <= hi for lo, hi in LETTER_RANGES)
|
||||
|
||||
|
||||
def fold_char(ch):
|
||||
return CHAR_FOLD.get(ch, ch)
|
||||
|
||||
|
||||
def is_simple_char(ch):
|
||||
ch = fold_char(ch)
|
||||
return is_letter(ch) or unicodedata.category(ch) == "Nd" or ch in OPERATORS
|
||||
|
||||
|
||||
def script_type(ch):
|
||||
d = unicodedata.decomposition(ch)
|
||||
if d.startswith("<super>"):
|
||||
return "^", "".join(chr(int(h, 16)) for h in d.split()[1:])
|
||||
if d.startswith("<sub>"):
|
||||
return "_", "".join(chr(int(h, 16)) for h in d.split()[1:])
|
||||
return None, None
|
||||
|
||||
|
||||
class Atom:
|
||||
__slots__ = ("base", "primes", "sub", "sup")
|
||||
|
||||
def __init__(self, base=""):
|
||||
self.base = base
|
||||
self.primes = 0
|
||||
self.sub = None
|
||||
self.sup = None
|
||||
|
||||
|
||||
def _canon(atoms):
|
||||
out = []
|
||||
for a in atoms:
|
||||
if a.sup is not None and a.sup and all(
|
||||
x.base == PRIME and x.sub is None and x.sup is None and x.primes == 0 for x in a.sup
|
||||
):
|
||||
a.primes += len(a.sup)
|
||||
a.sup = None
|
||||
s = a.base + PRIME * a.primes
|
||||
if a.sub is not None:
|
||||
s += "_{" + _canon(a.sub) + "}"
|
||||
if a.sup is not None:
|
||||
s += "^{" + _canon(a.sup) + "}"
|
||||
out.append(s)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _attach(atoms, op, arg):
|
||||
if not atoms:
|
||||
atoms.append(Atom(""))
|
||||
a = atoms[-1]
|
||||
if op == "_":
|
||||
if a.sub is not None:
|
||||
raise NotSimple("double subscript")
|
||||
a.sub = arg
|
||||
else:
|
||||
if a.sup is not None:
|
||||
raise NotSimple("double superscript")
|
||||
a.sup = arg
|
||||
|
||||
|
||||
def _char_atom(ch):
|
||||
if not is_simple_char(ch):
|
||||
raise NotSimple("character %r" % ch)
|
||||
return Atom(fold_char(ch))
|
||||
|
||||
|
||||
# --- TeX ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def _tex_tokens(src):
|
||||
toks = []
|
||||
i, n = 0, len(src)
|
||||
while i < n:
|
||||
ch = src[i]
|
||||
if ch == "\\":
|
||||
j = i + 1
|
||||
if j < n and src[j].isascii() and src[j].isalpha():
|
||||
while j < n and src[j].isascii() and src[j].isalpha():
|
||||
j += 1
|
||||
toks.append(("cw", src[i + 1:j]))
|
||||
while j < n and is_ws(src[j]):
|
||||
j += 1
|
||||
i = j
|
||||
continue
|
||||
if j < n:
|
||||
toks.append(("cs", src[j]))
|
||||
i = j + 1
|
||||
continue
|
||||
raise NotSimple("trailing backslash")
|
||||
if is_ws(ch) or ch == "~":
|
||||
i += 1
|
||||
continue
|
||||
toks.append(("ch", ch))
|
||||
i += 1
|
||||
return toks
|
||||
|
||||
|
||||
class _TexParser:
|
||||
def __init__(self, toks):
|
||||
self.t = toks
|
||||
self.i = 0
|
||||
|
||||
def peek(self):
|
||||
return self.t[self.i] if self.i < len(self.t) else None
|
||||
|
||||
def take(self):
|
||||
tok = self.peek()
|
||||
self.i += 1
|
||||
return tok
|
||||
|
||||
def seq(self, closing):
|
||||
atoms = []
|
||||
while True:
|
||||
tok = self.peek()
|
||||
if tok is None:
|
||||
if closing:
|
||||
raise NotSimple("unclosed group")
|
||||
return atoms
|
||||
if tok == ("ch", "}"):
|
||||
if not closing:
|
||||
raise NotSimple("unbalanced brace")
|
||||
self.take()
|
||||
return atoms
|
||||
self.take()
|
||||
kind, val = tok
|
||||
if kind == "ch" and val in "_^":
|
||||
_attach(atoms, val, self.arg())
|
||||
elif kind == "ch" and val == "'":
|
||||
if not atoms:
|
||||
atoms.append(Atom(""))
|
||||
atoms[-1].primes += 1
|
||||
elif kind == "ch" and val == "{":
|
||||
inner = self.seq(True)
|
||||
nxt = self.peek()
|
||||
if nxt in (("ch", "_"), ("ch", "^")) and len(inner) > 1:
|
||||
raise NotSimple("script on a multi-atom group")
|
||||
atoms.extend(inner) # an empty group leaves a following script to the previous atom or an empty base
|
||||
else:
|
||||
atoms.extend(self.command_or_char(kind, val))
|
||||
|
||||
def command_or_char(self, kind, val):
|
||||
if kind == "ch":
|
||||
return [_char_atom(val)]
|
||||
if kind == "cs":
|
||||
if val in TEX_SPACE_SYMBOLS:
|
||||
return []
|
||||
raise NotSimple("control symbol \\" + val)
|
||||
if val in TEX_GREEK:
|
||||
return [Atom(TEX_GREEK[val])]
|
||||
if val in TEX_SYMBOLS:
|
||||
return [Atom(TEX_SYMBOLS[val])]
|
||||
if val in TEX_SPACES or val in TEX_DECLARATIONS:
|
||||
return []
|
||||
if val in TEX_WRAPPERS:
|
||||
return self.arg()
|
||||
if val in ("left", "right"):
|
||||
tok = self.take()
|
||||
if tok == ("ch", "(" if val == "left" else ")"):
|
||||
return [Atom(tok[1])]
|
||||
raise NotSimple("\\left or \\right with another delimiter")
|
||||
raise NotSimple("command \\" + val)
|
||||
|
||||
def arg(self):
|
||||
tok = self.take()
|
||||
if tok is None:
|
||||
raise NotSimple("missing argument")
|
||||
kind, val = tok
|
||||
if tok == ("ch", "{"):
|
||||
return self.seq(True)
|
||||
if kind == "ch" and val in "}_^'":
|
||||
raise NotSimple("bad argument")
|
||||
return self.command_or_char(kind, val)
|
||||
|
||||
|
||||
def tex_atoms(src):
|
||||
return _TexParser(_tex_tokens(src)).seq(False)
|
||||
|
||||
|
||||
# --- Typst -------------------------------------------------------------------
|
||||
|
||||
|
||||
def _typst_tokens(src):
|
||||
toks = []
|
||||
i, n = 0, len(src)
|
||||
while i < n:
|
||||
ch = src[i]
|
||||
if is_ws(ch):
|
||||
i += 1
|
||||
continue
|
||||
if ch == '"':
|
||||
j = i + 1
|
||||
buf = []
|
||||
while j < n and src[j] != '"':
|
||||
if src[j] == "\\":
|
||||
raise NotSimple("escape in string")
|
||||
buf.append(src[j])
|
||||
j += 1
|
||||
if j >= n:
|
||||
raise NotSimple("unclosed string")
|
||||
toks.append(("str", "".join(buf)))
|
||||
i = j + 1
|
||||
continue
|
||||
if unicodedata.category(ch).startswith("L"):
|
||||
j = i
|
||||
while j < n and (
|
||||
unicodedata.category(src[j]).startswith("L")
|
||||
or (src[j] == "." and j + 1 < n and src[j + 1].isascii() and src[j + 1].isalpha() and j > i)
|
||||
):
|
||||
j += 1
|
||||
toks.append(("id", src[i:j]))
|
||||
i = j
|
||||
continue
|
||||
if unicodedata.category(ch) == "Nd":
|
||||
j = i
|
||||
while j < n and unicodedata.category(src[j]) == "Nd":
|
||||
j += 1
|
||||
if j + 1 < n and src[j] == "." and unicodedata.category(src[j + 1]) == "Nd":
|
||||
j += 1
|
||||
while j < n and unicodedata.category(src[j]) == "Nd":
|
||||
j += 1
|
||||
toks.append(("num", src[i:j]))
|
||||
i = j
|
||||
continue
|
||||
two = src[i:i + 2]
|
||||
if two in ("<=", ">="):
|
||||
toks.append(("ch", "\u2264" if two == "<=" else "\u2265"))
|
||||
i += 2
|
||||
continue
|
||||
if two in ("->", "=>", "!=", "<-", "==", ":="):
|
||||
raise NotSimple("operator " + two)
|
||||
toks.append(("ch", ch))
|
||||
i += 1
|
||||
return toks
|
||||
|
||||
|
||||
class _TypstParser:
|
||||
def __init__(self, toks):
|
||||
self.t = toks
|
||||
self.i = 0
|
||||
|
||||
def peek(self):
|
||||
return self.t[self.i] if self.i < len(self.t) else None
|
||||
|
||||
def take(self):
|
||||
tok = self.peek()
|
||||
self.i += 1
|
||||
return tok
|
||||
|
||||
def seq(self, closing):
|
||||
atoms = []
|
||||
while True:
|
||||
tok = self.peek()
|
||||
if tok is None:
|
||||
if closing:
|
||||
raise NotSimple("unclosed parenthesis")
|
||||
return atoms
|
||||
if closing and tok == ("ch", ")"):
|
||||
self.take()
|
||||
return atoms
|
||||
self.take()
|
||||
kind, val = tok
|
||||
if kind == "ch" and val in "_^":
|
||||
_attach(atoms, val, self.arg())
|
||||
elif kind == "ch" and val == "'":
|
||||
if not atoms:
|
||||
atoms.append(Atom(""))
|
||||
atoms[-1].primes += 1
|
||||
elif kind == "ch" and val == "(":
|
||||
atoms.append(Atom("("))
|
||||
atoms.extend(self.seq(True))
|
||||
atoms.append(Atom(")"))
|
||||
else:
|
||||
atoms.extend(self.single(kind, val))
|
||||
|
||||
def single(self, kind, val):
|
||||
if kind == "num":
|
||||
return [Atom(c) for c in val]
|
||||
if kind == "str":
|
||||
return [_char_atom(c) for c in remove_ws(val)]
|
||||
if kind == "id":
|
||||
if len(val) == 1:
|
||||
return [_char_atom(val)]
|
||||
if val in TYPST_GREEK:
|
||||
return [Atom(TYPST_GREEK[val])]
|
||||
if val in TYPST_SYMBOLS:
|
||||
return [Atom(TYPST_SYMBOLS[val])]
|
||||
if val in TYPST_SPACES:
|
||||
return []
|
||||
if val in TYPST_WRAPPERS and self.peek() == ("ch", "("):
|
||||
self.take()
|
||||
return self.seq(True)
|
||||
raise NotSimple("identifier " + val)
|
||||
if val in "/*#&[]{}\\$!:;|~@":
|
||||
raise NotSimple("character %r" % val)
|
||||
return [_char_atom(val)]
|
||||
|
||||
def arg(self):
|
||||
tok = self.take()
|
||||
if tok is None:
|
||||
raise NotSimple("missing argument")
|
||||
kind, val = tok
|
||||
if tok == ("ch", "("):
|
||||
return self.seq(True)
|
||||
if kind == "ch" and val in ")_^'":
|
||||
raise NotSimple("bad argument")
|
||||
return self.single(kind, val)
|
||||
|
||||
|
||||
def typst_atoms(src):
|
||||
return _TypstParser(_typst_tokens(src)).seq(False)
|
||||
|
||||
|
||||
# --- public ------------------------------------------------------------------
|
||||
|
||||
|
||||
def fold_math(notation, source):
|
||||
"""Return the canonical linear form of a math source, or None when it is complex."""
|
||||
src = unicodedata.normalize("NFC", source)
|
||||
try:
|
||||
atoms = tex_atoms(src) if notation == "tex" else typst_atoms(src)
|
||||
return _canon(atoms)
|
||||
except NotSimple:
|
||||
return None
|
||||
|
||||
|
||||
def complex_key(notation, source):
|
||||
return notation + ":" + remove_ws(unicodedata.normalize("NFC", source))
|
||||
|
||||
|
||||
def _fold_script_chars(chars):
|
||||
"""Fold a run of Unicode script characters of one type, or None if any base is not simple."""
|
||||
op = None
|
||||
bases = []
|
||||
for ch in chars:
|
||||
t, b = script_type(ch)
|
||||
if t is None or (op is not None and t != op):
|
||||
return None
|
||||
op = t
|
||||
bases.append(b)
|
||||
base = "".join(bases)
|
||||
if not all(is_simple_char(c) for c in base):
|
||||
return None
|
||||
return op + "{" + "".join(fold_char(c) for c in base) + "}"
|
||||
|
||||
|
||||
def _fold_chars(s, in_notation):
|
||||
"""Fold the characters of text: script characters always, operators only inside notation."""
|
||||
out = []
|
||||
i, n = 0, len(s)
|
||||
while i < n:
|
||||
ch = s[i]
|
||||
t, b = script_type(ch)
|
||||
if t is not None:
|
||||
j = i
|
||||
while j < n and script_type(s[j])[0] == t:
|
||||
j += 1
|
||||
folded = _fold_script_chars(s[i:j])
|
||||
out.append(folded if folded is not None else s[i:j])
|
||||
i = j
|
||||
continue
|
||||
if in_notation:
|
||||
out.append(fold_char(ch))
|
||||
elif ch in ("-", "\u2013") and i + 1 < n and unicodedata.category(s[i + 1]) == "Nd":
|
||||
out.append(MINUS)
|
||||
else:
|
||||
out.append(ch)
|
||||
i += 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def fold_plain_text(text):
|
||||
return _fold_chars(unicodedata.normalize("NFC", text), in_notation=False)
|
||||
|
||||
|
||||
def fold_styled_script(op, text):
|
||||
"""The canonical form of text styled as a subscript (`_`) or superscript (`^`)."""
|
||||
t = unicodedata.normalize("NFC", text)
|
||||
s = remove_ws(t)
|
||||
simple = all(is_simple_char(c) or script_type(c)[0] is not None for c in s)
|
||||
if simple:
|
||||
inner = _fold_chars(s, in_notation=True)
|
||||
if op == "^" and inner and all(c == PRIME for c in inner):
|
||||
return inner
|
||||
else:
|
||||
inner = trim(collapse_ws(t))
|
||||
return op + "{" + inner + "}"
|
||||
Reference in new issue
Block a user