#!/usr/bin/env python3 """ Alexander von Humboldt's travel journals, correspondence and plant-geography notes from the edition humboldt digital (BBAW). Usage: python ehd_nature_travel.py # -> ehd_nature_travel.csv Reads the edition's TEI data set (https://github.com/telota/edition-humboldt-digital, CC BY-SA 4.0; the files are downloaded once into cache/ehd/) and keeps three parts: - the journals of Humboldt's journeys: the American journey 1799–1804, the European journeys 1790–1805 and the Russian-Siberian journey 1829. The journal folder also holds two working papers (an index of his manuscripts and a note on galvanism), which are left out. Title, date and languages come from the TEI header; the places tagged most often in the text go into the summary. - the letters, as one entry per correspondence (Humboldt and Ehrenberg, Kunth, Hooker, the Schlagintweits, ...), listing each letter with its date, places and link. Hundreds of single letters would otherwise crowd the catalogue. - Humboldt's notes and excerpts for the new edition of the Géographie des plantes (1810s–1820s), as one entry. The other documents (petitions, contracts, announcements, the German translations of documents already listed) and the edition's modern research articles are left out. The LLM on innkube (as in bhl_nature_travel.py) decides whether a journal or correspondence is travel, natural history or both; answers are cached in ehd_labels.json. Needs INNKUBE_API_KEY. """ import argparse import csv import json import re import sys import urllib.request from collections import Counter, defaultdict from concurrent.futures import ThreadPoolExecutor from pathlib import Path from bhl_nature_travel import label REPO = "https://api.github.com/repos/telota/edition-humboldt-digital/git/trees/HEAD?recursive=1" RAW = "https://raw.githubusercontent.com/telota/edition-humboldt-digital/HEAD/" WEB = "https://edition-humboldt.de/" UA = {"User-Agent": "NaturalHistoriesCatalogue/1.0 (https://thomashaider.net/natural-histories/)"} HUMBOLDT = "Humboldt, Alexander von (1769–1859)" HUMBOLDT_ID = "H0012069" NOT_JOURNALS = {"H0019162", "H0020388"} # Physiologie. Galvanismus; Index général de mes manuscrits FOLDERS = {"travel-journal": "", "letter": "letter", "document": "document"} # Géographie des plantes material: the lettered folders ("Aa – Physiognomie ...", "B / C – Geographie # der Moose"), the plan and materials for the new edition, the questions to and answers from Robert Brown PLANT_GEOGRAPHY = re.compile(r"^[A-Z][a-z]?( / [A-Z][a-z]?)? [–—] |Geographie der Pflanzen|Géographie des plantes|" r"Botanical Geography|Pflanzengeographie|végétation des îles", re.I) NOT_NOTES = re.compile(r"Übersetzung|Verlagsvertrag|Ankündigung", re.I) LANGS = {"fra": "fr", "deu": "de", "spa": "es", "lat": "la", "eng": "en", "ita": "it"} # Few journals declare their languages, and most switch between French and German, so the # languages are counted from function words; any with at least a fifth of the hits is listed. FUNCTION_WORDS = {"fr": set("le la les des du et est une dans que qui sur pour par pas".split()), "de": set("der die das und ist nicht ein eine den mit von sich auf dem".split()), "es": set("el los las y que del por con una para".split()), "en": set("the and of is was with that which".split())} def fetch(url): with urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=300) as r: return r.read() def download(cache, refresh): """{folder: [files]} for the parts of the data set used here.""" tree = json.loads(fetch(REPO))["tree"] todo, out = [], defaultdict(list) for x in tree: kind = x["path"].split("/")[1] if x["path"].startswith("data/") else None if x["type"] == "blob" and kind in FOLDERS: f = cache / FOLDERS[kind] / Path(x["path"]).name out[kind].append(f) if refresh or not f.exists(): todo.append((x["path"], f)) print(f"downloading {len(todo)} TEI files …", file=sys.stderr) for d in {f.parent for _, f in todo}: d.mkdir(parents=True, exist_ok=True) with ThreadPoolExecutor(8) as pool: list(pool.map(lambda pf: pf[1].write_bytes(fetch(RAW + pf[0])), todo)) return {k: sorted(v) for k, v in out.items()} def text_of(body): return re.sub(r"]*type=\"editorial\".*?|.*?|.*?|<[^>]+>", " ", body, flags=re.S).lower().split() def languages(text, head): hits = Counter(lang for w in text for lang, ws in FUNCTION_WORDS.items() if w in ws) langs = [l for l, n in hits.most_common() if n >= 0.2 * sum(hits.values())] return langs or [LANGS.get(l, l) for l in re.findall(r']+>", "", re.search(r"(.*?)", head, re.S).group(1)).split()) def parse_journal(path): s = path.read_text(encoding="utf-8") head, _, body = s.partition("") title = title_of(head) m = re.search(r".*?]*)/?>", head, re.S) dates = re.findall(r'(?:when|from|notBefore)="(\d{4})', m.group(1)) if m else [] years = re.findall(r"\b1[78]\d\d\b", title) # titles often name the year: "Reise. 1790. England." text = text_of(body) places = Counter(" ".join(re.sub(r"<[^>]+>", "", p).split()) for p in re.findall(r"]*>(.*?)", body, re.S)) return { "id": path.stem, "kind": "journal", "title": title, "author": HUMBOLDT, "year": int(dates[0]) if dates else int(years[0]) if years else "", "language": ",".join(languages(text, head)) or "de", "words": len(text), "places": " | ".join(p for p, _ in places.most_common(12) if len(p) > 2), "url": WEB + path.stem, "items": "", } def person(action): """(edition id, 'Surname, Forename (dates)', place) of a correspAction.""" if not action or "]*ref="https://edition-humboldt.de/(H\d+)[^"]*"[^>]*>(.*?)', action, re.S) \ or re.search(r"]*>(.*?)", action, re.S) name = " ".join(re.sub(r"<[^>]+>", "", m.group(2)).split()) name = re.sub(r"\s+\)", ")", re.sub(r"\(\s+", "(", re.sub(r"(\d)-(\d)", r"\1–\2", name))) place = re.search(r"]*>(.*?)", action, re.S) return m.group(1) or name, name, " ".join(re.sub(r"<[^>]+>", "", place.group(1)).split()) if place else "" def correspondences(files): by = defaultdict(list) names = defaultdict(Counter) for path in files: s = path.read_text(encoding="utf-8") head, _, body = s.partition("") sent = re.search(r'(.*?)', head, re.S) recv = re.search(r'(.*?)', head, re.S) (sid, sname, splace), (rid, rname, rplace) = person(sent and sent.group(1)), person(recv and recv.group(1)) other = (rid, rname) if sid == HUMBOLDT_ID or "Humboldt, Alexander" in sname else (sid, sname) if not other[0]: continue date = re.search(r']*(?:when|notBefore|from)="([\d-]+)', sent.group(1) if sent else "") names[other[0]][other[1]] += 1 by[other[0]].append({"id": path.stem, "t": title_of(head), "y": int(date.group(1)[:4]) if date else None, "date": date.group(1) if date else "", "places": [p for p in (splace, rplace) if p], "langs": languages(text_of(body), head), "words": len(text_of(body)), "named": [" ".join(re.sub(r"<[^>]+>", "", x).split()) for x in re.findall(r"]*>(.*?)", body, re.S)]}) rows = [] for pid, letters in by.items(): letters.sort(key=lambda l: l["date"] or "9999") name = max(names[pid], key=lambda n: ("," in n, names[pid][n])) years = [l["y"] for l in letters if l["y"]] span = f"{min(years)}–{max(years)}" if years and min(years) != max(years) else str(years[0]) if years else "" langs = Counter(lang for l in letters for lang in l["langs"][:1]) places = Counter(p for l in letters for p in l["named"] if len(p) > 2) # places the letters talk about short = re.sub(r",? (Freiherr|Graf|Ritter) von$", " von", re.sub(r"\s*\(.*", "", name)) short = " ".join(reversed(short.split(", ", 1))) if ", " in short else short rows.append({ "id": "corr-" + pid, "kind": "letters", "title": f"Briefwechsel mit {short}" + (f" ({span})" if span else ""), "author": f"{HUMBOLDT}; {name}", "year": min(years) if years else "", "language": ",".join(l for l, n in langs.most_common() if n >= 0.1 * len(letters)) or "de", "words": sum(l["words"] for l in letters), "places": " | ".join(p for p, _ in places.most_common(8)), "url": WEB + letters[0]["id"], "items": json.dumps([{"id": l["id"], "t": l["t"], "y": l["y"], "r": WEB + l["id"]} for l in letters], ensure_ascii=False), }) return rows def plant_notes(files): notes = [] for path in files: s = path.read_text(encoding="utf-8") head, _, body = s.partition("") t = title_of(head) if PLANT_GEOGRAPHY.search(t) and not NOT_NOTES.search(t): date = re.search(r'.*?]*(?:when|notBefore|from)="(\d{4})', head, re.S) text = text_of(body) notes.append({"id": path.stem, "t": t, "y": int(date.group(1)) if date else None, "r": WEB + path.stem, "langs": languages(text, head), "words": len(text)}) if not notes: return [] notes.sort(key=lambda n: n["t"]) years = [n["y"] for n in notes if n["y"]] langs = Counter(l for n in notes for l in n["langs"][:1]) return [{ "id": "geographie-des-plantes", "kind": "notes", "title": "Materialien zur Neuausgabe der Géographie des plantes", "author": HUMBOLDT, "year": min(years) if years else "", "language": ",".join(l for l, _ in langs.most_common()), "words": sum(n["words"] for n in notes), "places": "", "url": notes[0]["r"], "items": json.dumps([{k: n[k] for k in ("id", "t", "y", "r")} for n in notes], ensure_ascii=False), }] def main(): ap = argparse.ArgumentParser() ap.add_argument("--dir", default="cache/ehd") ap.add_argument("--refresh", action="store_true", help="download the TEI files again") ap.add_argument("--labels", default="ehd_labels.json") ap.add_argument("-o", "--out", default="ehd_nature_travel.csv") args = ap.parse_args() files = download(Path(args.dir), args.refresh) journals = [parse_journal(f) for f in files["travel-journal"] if f.stem not in NOT_JOURNALS] letters = correspondences(files["letter"]) rows = journals + letters + plant_notes(files["document"]) labels = label([(r["id"], f"{r['title']} | {r['author']} | {r['year']} | places named most often: {r['places'][:200]}") for r in journals + letters], args.labels, 1, what="travel journal or correspondence of Alexander von Humboldt", fields="title | authors | year | places") for r in rows: v = labels.get(r["id"], "natural_history" if r["kind"] == "notes" else "travel") if r["kind"] == "journal": r["category"] = "natural_history+travel" if v in ("both", "natural_history") else "travel" else: r["category"] = {"both": "natural_history+travel", "travel": "travel"}.get(v, "natural_history") with open(args.out, "w", encoding="utf-8", newline="") as f: w = csv.DictWriter(f, fieldnames=list(rows[0])) w.writeheader() w.writerows(sorted(rows, key=lambda r: (r["kind"] != "journal", r["year"] or 9999, r["title"]))) print(f"Wrote {len(journals)} journals, {len(letters)} correspondences and the plant-geography notes to {args.out}", file=sys.stderr) for r in rows: n = len(json.loads(r["items"])) if r["items"] else "" print(f" {r['year']} {r['language']:8} {r['category']:23} {r['words']:>7} {n:>4} {r['title'][:60]}", file=sys.stderr) if __name__ == "__main__": main()