""" Memorabilia universal card format (JVA-MEM-PROC-001), one shared module. Layer 1 TITLE -> format_title() Layer 2 SUBTITLE -> format_subtitle() (AUTHOR . PUBLISHER . YEAR, gold line) Layer 3 DESCRIPTION -> format_description() (real sentences, ends on a period) Deterministic only. No AI, no invented facts: everything comes from the source catalog record. Used by the live tracker AND by the data-repair script so old and new items always match. """ import html as _html import re # ---------------------------------------------------------------- titles _SMALL = { "a", "an", "and", "as", "at", "but", "by", "for", "from", "in", "into", "nor", "of", "on", "onto", "or", "over", "per", "the", "to", "up", "via", "vs", "with", "within", "without", } _ABBR = { "rev", "mr", "mrs", "ms", "dr", "st", "jr", "sr", "prof", "hon", "gen", "col", "capt", "lt", "sgt", "maj", "no", "vol", "ed", "eds", "etc", "vs", "co", "inc", "univ", "dept", "gov", "sen", "rep", "br", "bro", "messrs", "esq", "hon", "gov", "supt", "pres", "comr", "mt", "ft", "d", "dd", "geo", "wm", "jas", "thos", "chas", "benj", "jno", "robt", "saml", "hy", "edw", "fred", } _CODE_RE = re.compile(r"^\s*(?:ERIC\s+)?(?:[A-Z]{2,6}[\s-]?\d{3,}[A-Z0-9]*)\s*[:\-\u2013\u2014]\s*") _SEPS = (". ", " : ", ": ", " \u2014 ", " \u2013 ", " / ", " ; ", "; ", " -- ", "-- ") # phrases that start a descriptive tail on a title-page string _CONNECT = ( ", communicating", ", respecting", ", exposing", ", praying", ", containing", ", including", ", with ", ", being ", ", delivered", ", held ", ", presented", ", addressed", ", relative to", ", together with", ", in review", ", in reply", ", one of ", " relative to ", " together with ", ", read before", ", in compliance", ", in answer", ", before the ", ", late ", " by its board", " by the board", ) _MAX_LONG = 110 def _norm_space(s: str) -> str: s = (s or "").replace("\u00a0", " ").replace("\u200b", "") return re.sub(r"\s+", " ", s).strip() def _first_sep(t: str) -> int: best = -1 for sep in _SEPS: start = 0 while True: i = t.find(sep, start) if i == -1: break if sep == ". ": words = t[:i].split() raw = words[-1].strip(".") if words else "" if raw.lower() in _ABBR or (len(raw) == 1 and raw.isalpha()): start = i + 1 continue if re.fullmatch(r"(?:[A-Za-z]\.)+[A-Za-z]", raw): # "J.W. Loguen" start = i + 1 continue if re.fullmatch(r"\d{1,2}", raw): # "No. 9. " / "vol. 2. " start = i + 1 continue if i >= 8 and (best == -1 or i < best): best = i break return best def _fix_ordinals(t: str) -> str: return re.sub(r"\b(\d+)(St|Nd|Rd|Th)\b", lambda m: m.group(1) + m.group(2).lower(), t) def _cap(word: str) -> str: for i, ch in enumerate(word): if ch.isalpha(): return word[:i] + ch.upper() + word[i + 1:] return word def _title_case(t: str) -> str: if t and t.isupper(): t = t.lower() words = t.split() n = len(words) out = [] for i, w in enumerate(words): core = w.strip(".,;:!?()[]\"'\u2019\u201c\u201d") if not core: out.append(w) continue if core.isupper() and len(core) <= 5: out.append(w) continue if i not in (0, n - 1) and core.lower() in _SMALL: out.append(w.lower()) continue if "-" in w and not w.startswith("-"): parts = w.split("-") parts = [p if (j > 0 and p.lower() in _SMALL) else _cap(p) for j, p in enumerate(parts)] out.append("-".join(parts)) else: out.append(_cap(w)) return _fix_ordinals(" ".join(out)) def _unbracket(t: str) -> str: t = re.sub(r"\s*\[(?:manuscript|microform|electronic resource|sound recording)\]\s*", " ", t, flags=re.I) # trailing bracket with a date/range or number -> drop t = re.sub(r"\s*\[[\d\s,\-\u2013/c.?]+\]\s*$", "", t) # remaining bracketed words: keep the words, drop the brackets t = re.sub(r"\[([^\]]*)\]", r"\1", t) return _norm_space(t) def format_title(raw: str) -> str: """Clean card headline. Never returns empty.""" original = _norm_space(raw) t = re.sub(r"\s*(?:\.{3,}|\u2026)\s+(?:\w+\.?\s*){1,2}$", "", original) # "... Report" tail t = re.sub(r"\s*(?:\.{3,}|\u2026)\s*", " ", t) # "Speech... on X..." -> "Speech on X" t = _norm_space(t) if t.startswith("[") and "]" not in t: t = t[1:].lstrip() # unclosed "[Speeches of ..." t = _CODE_RE.sub("", t) t = re.sub(r"^\s*\[[^\]]*(?:SOURCE|HISTORICAL)[^\]]*\]\s*", "", t) t = _unbracket(t) i = _first_sep(t) if i != -1: head = t[:i].strip(" ,;:-") if len(head) >= 8: t = head if len(t) > 60: low = t.lower() cuts = [low.find(c) for c in _CONNECT if low.find(c) >= 20] if cuts: t = t[:min(cuts)].strip() if len(t) > _MAX_LONG: head = t[:_MAX_LONG] k = head.rfind(", ") if k < 40: ks = [head.rfind(c) for c in (" in ", " and ", " with ", " to ", " for ", " on ", " by ", " of ")] k = max(ks) if k >= 40: t = t[:k] # no dangling tail after a length cut: open paren, trailing month name if t.count("(") > t.count(")"): t = t[:t.rfind("(")] t = re.sub(r",?\s+(?:January|February|March|April|May|June|July|August|September|October|November|December)(?:\s+\d{1,2})?$", "", t) t = t.strip(" ,;:-\u2013\u2014/").lstrip(".\u2026 ") t = re.sub(r"\s+:\s+", ": ", t) t = re.sub(r"\s+&[Cc]\.?$", "", t).strip(" ,;:") if t.endswith("."): last = t.split()[-1].rstrip(".") if t.split() else "" if not (last.lower() in _ABBR or (len(last) <= 2 and last.isalpha() and last[:1].isupper())): t = t.rstrip(".") if not t: return original return _title_case(t) or original # -------------------------------------------------------------- subtitle _ROLE_RE = re.compile(r",?\s*\b(?:author|editor|ed\.?|eds\.?|compiler|comp\.?|translator|tr\.?|illustrator|joint author)\b\.?\s*$", re.I) _PREFIX_T = {"mrs", "mr", "miss", "sir", "rev", "dr", "hon", "prof", "gen", "col", "capt"} _SUFFIX_T = {"jr", "sr", "ii", "iii", "iv"} def _nodot(n: str) -> str: """Drop a trailing period unless it ends an initial/abbreviation.""" n = n.strip(" ,;") if n.endswith("."): last = n.split()[-1].rstrip(".") if n.split() else "" if not (last.lower() in _ABBR or last.lower() in _SUFFIX_T or (len(last) <= 2 and last[:1].isupper())): n = n.rstrip(".") return n def display_author(raw) -> str: if isinstance(raw, (list, tuple)): raw = raw[0] if raw else "" a = _norm_space(str(raw or "")) if not a: return "" a = re.sub(r"\[from old catalog\]", "", a, flags=re.I) a = re.sub(r"(\w)\[(\w+)\]", r"\1\2", a) # C[harles] -> Charles a = re.sub(r"\([^)]*\)", "", a) # (Felix Gregory) a = re.sub(r",?\s*(?:b\.|d\.|fl\.|ca\.)?\s*\d{4}\??\s*-\s*(?:\d{4}\??)?\.?", "", a) # 1813-1896 / 1864- a = re.sub(r",?\s*-\s*\d{4}\??\.?", "", a) # -1893 a = re.sub(r",?\s*\b(?:b|d|fl|ca)\.\s*\d{4}\??\.?\s*$", "", a) # ", d. 1839" a = re.sub(r",?\s*\d{4}\??\s*$", "", a) for _ in range(2): a = _ROLE_RE.sub("", a) a = _norm_space(a).strip(" ,;") if a.isupper() and len(a) > 6 and " " in a: # "CONKLIN, JULIA S." a = a.title() a = re.sub(r",\s*pseud\.?$", " (pseudonym)", a, flags=re.I) a = re.sub(r"^(?:Mrs|Mr|Miss|Ms)\.?\s+(?=\S+\s+\S)", "", a) # "Mrs. Jean T. Zimmermann" # corporate: "United States. Congress. House" if ". " in a and re.search(r"[a-z]\.\s+[A-Z]", a) and "," not in a.split(". ")[0]: segs = [s.strip(" .") for s in a.split(". ") if s.strip(" .")] return _nodot(", ".join(segs)) parts = [_nodot(p) for p in a.split(",") if p.strip(" .")] if len(parts) >= 2 and len(parts[1].split()) <= 5 and parts[1][:1].isupper() and not re.search(r"\b(society|committee|council|association|bureau|department|commission|convention|congress|university|college|church)\b", parts[0], re.I): last = parts[0] given, pre, suf = [], [], [] for p in parts[1:]: pk = p.lower().strip(".") toks = p.split() if pk in ("mrs", "mr", "miss", "ms"): continue # honorifics add nothing on the card if pk in _PREFIX_T: pre.append(p) elif pk in _SUFFIX_T: suf.append(p) else: # "Joseph W. jr." -> trailing suffix token while toks and toks[-1].lower().strip(".") in _SUFFIX_T: suf.insert(0, toks.pop()) given.append(" ".join(toks)) name = " ".join(x for x in pre + [" ".join(given).strip(), last] + suf if x) return _nodot(_norm_space(name)) return _nodot(a.strip(" ,;")) def _year(*vals) -> str: for v in vals: m = re.search(r"\b(1[5-9]\d{2}|20[0-2]\d)\b", str(v or "")) if m: return m.group(1) return "" def format_subtitle(author, publisher: str, year) -> str: parts = [] a = display_author(author) if a: parts.append(a) if publisher: parts.append(publisher) y = _year(year) if y: parts.append(y) return " \u00b7 ".join(parts) # ----------------------------------------------------------- description _ABBR_SENT = re.compile( r"(?:\b(?:Mr|Mrs|Ms|Dr|Rev|Hon|Gen|Col|Capt|Lt|Sgt|Maj|Sen|Rep|Gov|St|Jr|Sr|Prof|Vol|No|Co|Inc|Univ|Dept|vs|etc|Esq|Bro|Br|pp?|ed|eds|cf|ca|U\.S|D\.C|A\.M|P\.M)|\b[A-Z])\.$" ) _JUNK = re.compile( r"no edits|archive\.org/metadata|online edition of this book|public domain, i\.e\.|" r"Checklist Amer|\bt\.-p\.|Bound together|^JK\d|Bibliography of American literature|" r"Inscribed:|certifies that this is one of an edition|\(\d+ cards?\)|\bBlanck,|" r"Emory University Digital Library|Digital Library Publications Program|" r"^\"?Translated for|Added title|^Reprinted from|^Cover title|^Caption title|" r"metadata below describe|Follow the \"All Files|directory structure for the texts|digital reproduction made from|^Thesis \(|^Photocopy|^First edition|Series title|^Referred to the|" r"^Published also under|edition of 18\d\d|copies printed|^Source: |^Appended:|^Articles reviewed|" r"^v\. ?\d|^\"?Bibliography|^\"?Works of reference|^\"?Read at|^Published at|" r"^At head of title|issued first with title|identical with|^Includes (?:index|bibliograph)|^Paged continuously|^Also available|^\"?Read before", re.I, ) def _sentences(text: str): text = _norm_space(text) out, buf = [], "" for tok in re.split(r"(?<=[.!?])\s+(?=[\"'\u201c(\[]?[A-Z0-9])", text): buf = (buf + " " + tok).strip() if buf else tok if re.search(r"[.!?][\"'\u201d\u2019)\]]?$", buf) and not _ABBR_SENT.search(buf): out.append(buf) buf = "" return out, buf # buf = unfinished tail (dropped) def clean_catalog_description(desc, max_chars: int = 400): """Return 1-3 complete sentences or '' if the catalog text is not a real description.""" if isinstance(desc, (list, tuple)): cands = [str(d) for d in desc] else: cands = [str(desc or "")] best = "" for c in cands: c = _html.unescape(c) c = re.sub(r"<[^>]+>", " ", c) c = re.sub(r"also available in digital form(?: on the library of congress web ?site)?\.?", " ", c, flags=re.I) c = re.sub(r"\bLAC\s+[a-z]{2,4}\s+\d{4}-\d{2}-\d{2}(?:\s+(?:update|review))?(?:\s*\(\d+\s+cards?\))?", " ", c, flags=re.I) c = re.sub(r"\b(?:update|review)\s*\(\d+\s+cards?\)", " ", c, flags=re.I) c = re.sub(r"\bno edits\b(?:\s*\(\d+\s+cards?\))?", " ", c, flags=re.I) c = _norm_space(c) if len(c) < 60 or _JUNK.search(c) or c.count(" -- ") >= 3: continue if len(c) > len(best): best = c if not best: return "" sents, _tail = _sentences(best) out = "" for s in sents[:3]: if len((out + " " + s).strip()) > max_chars and out: break out = (out + " " + s).strip() if len(out) > 520 or len(out) < 40: return "" return out _SUBJ_DROP = re.compile(r"^(?:united states)?\.?$", re.I) def _subject(s: str) -> str: s = re.sub(r"\[from old catalog\]", "", s, flags=re.I) s = _norm_space(s).replace("--", ", ") s = re.sub(r",\s*,", ",", s) s = re.sub(r",?\s*\d{4}\s*-\s*\d{4}", "", s) s = re.sub(r"\.,", ",", s) s = re.sub(r"\s+,", ",", s) s = re.sub(r",(?=\S)", ", ", s) s = re.sub(r"\s{2,}", " ", s) if re.search(r"\d{4},?\s*-|,\s*-", s): # name-with-dates headings return "" return s.strip(" .,;") def build_fallback_description(title, year, author, subjects, institution) -> str: y = _year(year) a = display_author(author) art = "an" if y.startswith("18") else "a" head = f"{title} is {art} {y} publication" if y else f"{title} is a historical publication" if a and a.lower().replace(",", "") not in title.lower().replace(",", "") and title.lower() not in a.lower(): head += " by " + (a[0].lower() + a[1:] if re.match(r"(?:A|An) [a-z]", a) else a) out = head if head.endswith(".") else head + "." subs = [] for s in (subjects if isinstance(subjects, (list, tuple)) else str(subjects or "").split(";")): s = _subject(str(s)) if s and s.lower() not in [x.lower() for x in subs]: subs.append(s) subs = subs[:4] if subs: out += " Catalog subjects: " + "; ".join(subs) + "." out += f" Digitized primary source from the {institution}." return out def format_description(catalog_desc, title, year, author, subjects, institution) -> str: d = clean_catalog_description(catalog_desc) if d: return d return build_fallback_description(title, year, author, subjects, institution)