#!/usr/bin/env python3
"""
Natural histories and travel accounts from the Biblioteca Nacional de España, the Universidad
Complutense de Madrid and the Biblioteca Nacional de Portugal, found through the Europeana API.

Usage:
    python europeana_nature_travel.py            # -> europeana_nature_travel.csv
    python europeana_nature_travel.py --cached   # reuse cache/europeana/*.jsonl

The BNE's own services (bne.es, the Biblioteca Digital Hispánica, datos.bne.es) refuse scripted
requests from this machine, so this asks Europeana, which aggregates the BDH's records with links
to the BNE viewer, and the Complutense's scans (made by Google; linked here to their copies in
HathiTrust, which unlike Google Books shows them in full outside the US). For each library it queries printed
texts dated 1450–1929 whose title names a journey, voyage, expedition, natural history, animals or
plants (most BNE records have no subject headings), or whose subject headings name travel or
natural history. Records in English, German, French, Spanish, Portuguese or Italian are kept; creators that are only
URIs are resolved through Europeana's entity API where possible. Copies and volumes of a work are
merged as in ia_nature_travel.py, and the LLM on innkube labels each work natural history, travel,
both or neither (answers cached in europeana_labels.json).

The kept works then get their library's own catalogue data, which is fuller than Europeana's search
results: for the Complutense, HathiTrust's MARC record (author with dates, imprint, extent, subject
headings, rights), fetched from HathiTrust's bibliographic API; for the BNE, Europeana's full record
(place of printing, printer, extent, subject headings, and author names where the search gave only
an identifier). Records are cached in cache/hathitrust/ and cache/europeana/records/.
Needs EUROPEANA_API_KEY and INNKUBE_API_KEY.
"""
import argparse
import csv
import json
import os
import re
import sys
import time
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from collections import Counter, defaultdict
from pathlib import Path

from bhl_nature_travel import label
from ia_nature_travel import VOLUME, norm

API = "https://api.europeana.eu/record/v2/search.json"
ENTITY = "https://api.europeana.eu/entity/agent/{}.json"
UA = {"User-Agent": "NaturalHistoriesCatalogue/1.0 (https://thomashaider.net/natural-histories/)"}
PROVIDERS = {"BNE": "National Library of Spain", "UCM": "Complutense University of Madrid",
             "BNP": "National Library of Portugal"}
# The Biblioteca Nazionale Centrale di Firenze was tried too (2026-10-22): its matching records are serial
# issues and plays, none of which survive the labelling, so it is left out.
LIBRARY = {"BNE": "Biblioteca Nacional de España", "UCM": "Universidad Complutense de Madrid",
           "BNP": "Biblioteca Nacional de Portugal"}
TITLE_WORDS = ('viaje OR viajes OR viage OR viages OR "historia natural" OR naturaleza OR expedición OR expedicion OR '
               'navegación OR navegacion OR derrotero OR peregrinación OR excursión OR excursiones OR travesía OR aves OR '
               'plantas OR animales OR botánica OR zoología OR insectos OR "noticias de" OR "relación del viaje" OR '
               '"relacion del viage" OR "diario del viaje" OR "diario de navegación" OR '
               # Portuguese and Italian
               'viagem OR viagens OR "história natural" OR expedição OR navegação OR animais OR '
               'viaggio OR viaggi OR "storia naturale" OR "istoria naturale" OR spedizione OR navigazione OR escursione OR '
               'uccelli OR piante OR animali')
SUBJECTS = '(viajes OR "historia natural" OR "descripciones y viajes" OR viagens OR "história natural" OR viaggi OR "storia naturale")'
LANGS = {"spa": "es", "es": "es", "fre": "fr", "fra": "fr", "fr": "fr", "ger": "de", "deu": "de", "de": "de",
         "eng": "en", "en": "en", "por": "pt", "pt": "pt", "ita": "it", "it": "it"}
NOT_TEXT = re.compile(r"\[(música|musica|grabado|mapa|material cartográfico|fotografía|dibujo)", re.I)


def get(url):
    for attempt in range(6):
        try:
            with urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=120) as r:
                return json.load(r)
        except Exception as e:
            print(f"  ! {e}; retrying", file=sys.stderr)
            time.sleep(5 * 2 ** attempt)
    raise RuntimeError(url.replace(os.environ["EUROPEANA_API_KEY"], "<key>"))


def search(query, out):
    """All records for a query (cursor paging, 100 per page), appended to out as JSON lines."""
    params = {"wskey": os.environ["EUROPEANA_API_KEY"], "query": query, "qf": "YEAR:[1450 TO 1929]",
              "rows": 100, "profile": "rich", "cursor": "*"}
    n = 0
    with open(out, "a", encoding="utf-8") as f:
        while params["cursor"]:
            d = get(API + "?" + urllib.parse.urlencode(params))
            for it in d.get("items", []):
                f.write(json.dumps(it, ensure_ascii=False) + "\n")
            n += len(d.get("items", []))
            params["cursor"] = d.get("nextCursor")
            time.sleep(0.3)
    return n


def harvest(cache):
    cache.mkdir(parents=True, exist_ok=True)
    for code, provider in PROVIDERS.items():
        out = cache / f"{code}.jsonl"
        out.unlink(missing_ok=True)
        base = f'DATA_PROVIDER:"{provider}" AND TYPE:TEXT'
        n = search(f"{base} AND title:({TITLE_WORDS})", out) + search(f"{base} AND subject:{SUBJECTS}", out)
        print(f"  {code}: {n} records", file=sys.stderr)


def creator_names(items, cache_path):
    """{uri: name} for creators given only as Europeana agent URIs (other URIs, e.g. datos.bne.es, stay unnamed)."""
    cache = json.loads(Path(cache_path).read_text(encoding="utf-8")) if Path(cache_path).exists() else {}
    uris = {c for it in items for c in it.get("dcCreator") or [] if c.startswith("http://data.europeana.eu/agent/")}
    for uri in sorted(uris - set(cache)):
        try:
            d = get(ENTITY.format(uri.rsplit("/", 1)[1]) + "?wskey=" + os.environ["EUROPEANA_API_KEY"])
            labels = d.get("prefLabel", {})
            cache[uri] = labels.get("es") or labels.get("en") or next(iter(labels.values()), "")
        except RuntimeError:
            cache[uri] = ""
        time.sleep(0.2)
    Path(cache_path).write_text(json.dumps(cache, ensure_ascii=False), encoding="utf-8")
    return cache


def hathitrust(url):
    """The Complutense's Google Books scans are also in HathiTrust (vid=UCM5320208538 -> ucm.5320208538), which shows
    pre-1880 books in full view everywhere; Google Books often shows only snippets outside the US."""
    m = re.search(r"books\.google\.com/books/ucm\?vid=UCM(\d+)", url)
    return f"https://babel.hathitrust.org/cgi/pt?id=ucm.{m.group(1)}" if m else url


HATHI = "https://catalog.hathitrust.org/api/volumes/full/htid/{}.json"
RECORD = "https://api.europeana.eu/record/v2/{}.json"
MARC = "{http://www.loc.gov/MARC21/slim}"
SUBJECT_TAGS = ("600", "610", "611", "630", "650", "651")


def cached_json(url, path):
    if path.exists():
        return json.loads(path.read_text(encoding="utf-8"))
    path.parent.mkdir(parents=True, exist_ok=True)
    d = get(url)
    path.write_text(json.dumps(d, ensure_ascii=False), encoding="utf-8")
    time.sleep(0.5)
    return d


def tidy(s):
    return re.sub(r"\s+([:;,.])", r"\1", " ".join(s.split())).strip(" ./:;,")


def from_marc(xml):
    """Author with dates, imprint, extent and subject headings from a MARC-XML record."""
    fields = defaultdict(list)
    for df in ET.fromstring(xml).iter(MARC + "datafield"):
        fields[df.get("tag")].append([(sf.get("code"), sf.text or "") for sf in df.iter(MARC + "subfield")])
    sub = lambda f, codes: [v for c, v in f if c in codes and not re.fullmatch(r"id\d+", v.strip())]  # skip local codes
    main = next(iter(fields.get("100") or fields.get("110") or []), [])
    author = tidy(", ".join(tidy(v) for v in sub(main, "abcd")))
    imp = next(iter(fields.get("264") or fields.get("260") or []), [])
    imprint = tidy(" ".join(sub(imp, "abc"))).replace(" :", ":").replace(" ;", ";")
    extent = tidy(" ".join(sub(next(iter(fields.get("300") or []), []), "abc")))
    subjects = []
    for tag in SUBJECT_TAGS:
        for f in fields.get(tag, []):
            head = tidy(" ".join(sub(f, "abcdt")))
            subjects.append(" -- ".join([head] + [tidy(v) for v in sub(f, "vxyz")]))
    return author, imprint, extent, subjects


def first(lang_aware):
    if not lang_aware:
        return []
    return lang_aware.get("def") or next(iter(lang_aware.values()), [])


def enrich(rows, cache):
    """Fill author, imprint, extent and subjects from the library's own record (see the module docstring)."""
    n = Counter()
    for r in rows:
        r.update(imprint="", extent="")
        try:
            if m := re.search(r"id=(ucm\.\d+)", r["read_url"]):
                d = cached_json(HATHI.format(m.group(1)), Path("cache/hathitrust") / f"{m.group(1)}.json")
                rec = next(iter(d.get("records", {}).values()), None)
                if not rec:
                    continue
                author, r["imprint"], r["extent"], subjects = from_marc(rec["marc-xml"])
                r["author"] = author or r["author"]
                r["subjects"] = " | ".join(subjects)[:400] or r["subjects"]
                n["hathitrust"] += 1
            else:
                d = cached_json(RECORD.format(r["id"]) + "?wskey=" + os.environ["EUROPEANA_API_KEY"],
                                cache / "records" / (r["id"].replace("/", "_") + ".json"))
                prox = next(p for p in d["object"]["proxies"] if not p.get("europeanaProxy"))
                literal = lambda k: [v for v in first(prox.get(k)) if not v.startswith("http")]
                if not r["author"]:
                    r["author"] = "; ".join(dict.fromkeys(literal("dcCreator")))[:200]
                # "España-Madrid" is the place of printing; a bare region ("Asturias ...") is what the book is about
                place = [v.split("-", 1)[1] for v in literal("dctermsSpatial") if re.fullmatch(r"[^-]+-[^-]+", v)]
                r["imprint"] = f"{place[0]}, {r['year']}" if place else ""
                r["extent"] = tidy("; ".join(literal("dctermsExtent") + literal("dcFormat")))[:120]
                subjects = literal("dcSubject")
                r["subjects"] = " | ".join(dict.fromkeys(subjects))[:400] or r["subjects"]
                n["europeana record"] += 1
        except (RuntimeError, StopIteration, KeyError, ET.ParseError) as e:
            print(f"  ! {r['id']}: {e}", file=sys.stderr)
    print(f"  enriched from {dict(n)}; with author {sum(1 for r in rows if r['author'])}/{len(rows)}, "
          f"with subjects {sum(1 for r in rows if r['subjects'])}/{len(rows)}", file=sys.stderr)


def parse(it, code, names):
    title = " ".join((it.get("title") or [""])[0].split())
    year = next((int(m.group(0)) for y in it.get("year") or [] if (m := re.search(r"1[4-9]\d\d", y))), None)
    lang = next((LANGS[l.lower()] for l in it.get("dcLanguage") or [] if l.lower() in LANGS), None)
    if not (title and year and lang) or NOT_TEXT.search(title):
        return None
    creators = [names.get(c, "") if c.startswith("http") else c for c in it.get("dcCreator") or []
                if not re.fullmatch(r"\s*id\d+\s*", c)]  # "id100": a local field code, not a name
    # Europeana repeats a name in several spellings ("Cortés, Gerónimo; Cortes, Geronimo"): keep the first of each
    creators = list({norm(re.sub(r",?\s*\d.*$", "", c)).replace(" ", ""): c for c in reversed(creators) if c}.values())[::-1]
    return {"id": it["id"].strip("/"), "provider": code, "title": re.sub(r"\s*\[Texto impreso\]", "", title).rstrip(" ./:;,"),
            "author": "; ".join(c for c in creators if c)[:200], "year": year, "language": lang,
            "subjects": " | ".join(it.get("dcSubject") or [])[:400],
            "read_url": hathitrust((it.get("edmIsShownAt") or [""])[0]),
            "url": "https://www.europeana.eu/item/" + it["id"].strip("/")}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--dir", default="cache/europeana")
    ap.add_argument("--cached", action="store_true", help="reuse the last harvest")
    ap.add_argument("--labels", default="europeana_labels.json")
    ap.add_argument("--workers", type=int, default=4)
    ap.add_argument("-o", "--out", default="europeana_nature_travel.csv")
    args = ap.parse_args()
    cache = Path(args.dir)
    if not args.cached:
        harvest(cache)

    rows = []
    for code in PROVIDERS:
        items = {it["id"]: it for it in map(json.loads, open(cache / f"{code}.jsonl", encoding="utf-8"))}
        names = creator_names(items.values(), cache / "agents.json")
        recs = [r for it in items.values() if (r := parse(it, code, names))]
        works = defaultdict(list)
        for r in recs:
            surname = norm(re.split(r"[,(]", r["author"])[0]).strip() if r["author"] else ""
            works[(r["language"], surname, " ".join(norm(VOLUME.sub("", r["title"])).split()[:8]))].append(r)
        for ws in works.values():
            lead = min(ws, key=lambda r: r["year"])
            rows.append({**lead, "volumes": len(ws)})
        print(f"{code}: {len(items)} records, {len(recs)} kept as texts in six languages up to 1929, {len(works)} works",
              file=sys.stderr)

    lines = [(r["id"], f"{r['title'][:160]} | {r['author'][:60] or 'anonymous'} | {r['year']} | {r['subjects'][:200]}")
             for r in rows]
    labels = label(lines, args.labels, args.workers, what="book in a Spanish, Portuguese or Italian national or university library")
    out = []
    for r in rows:
        v = labels.get(r["id"])
        if v and v != "neither":
            r["category"] = "natural_history+travel" if v == "both" else v
            out.append(r)
    enrich(out, cache)
    with open(args.out, "w", encoding="utf-8", newline="") as f:
        w = csv.DictWriter(f, fieldnames=list(out[0]))
        w.writeheader()
        w.writerows(sorted(out, key=lambda r: (r["provider"], r["year"])))
    print(f"Wrote {len(out)} of {len(rows)} works to {args.out}", file=sys.stderr)
    for k, n in sorted(Counter((r["provider"], r["language"], r["category"]) for r in out).items()):
        print("  ", *k, n, file=sys.stderr)


if __name__ == "__main__":
    main()