#!/usr/bin/env python3
"""
Natural histories and travel accounts in Gallica, found through the BnF catalogue.

Usage:
    python bnf_nature_travel.py            # -> bnf_nature_travel.csv
    python bnf_nature_travel.py --cached   # reuse the records in cache/bnf/

Gallica's own search API refuses scripted requests, so this asks the BnF catalogue's SRU
interface (https://catalogue.bnf.fr/api/SRU) for records that are freely readable in Gallica
(bib.digitized = freeAccess) and whose subject headings name voyages, description and travel
or natural history, or (for French books) whose title names a voyage, excursion or natural
history, since many older BnF records have no subject headings. From the UNIMARC records it keeps printed monographs
up to 1929 in French, German or English, with their Gallica link. Volumes and reissues of a
work are merged as in ia_nature_travel.py, and the LLM on innkube labels each work natural
history, travel, both or neither; answers are cached in bnf_labels.json. Needs INNKUBE_API_KEY.
"""
import argparse
import csv
import re
import sys
import time
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

from bhl_nature_travel import label
from ia_nature_travel import VOLUME, norm

SRU = "https://catalogue.bnf.fr/api/SRU"
UA = {"User-Agent": "NaturalHistoriesCatalogue/1.0 (https://thomashaider.net/natural-histories/)"}
FREE = 'bib.digitized all "freeAccess"'
QUERIES = [f'{FREE} and bib.subject all "{s}"' for s in
           ("voyages", "descriptions et voyages", "explorations", "histoire naturelle", "sciences naturelles",
            "naturalistes", "zoologie", "botanique", "ornithologie", "entomologie")] + \
          [f'{FREE} and bib.title all "{t}" and bib.language all "fre"' for t in
           ("voyage", "voyages", "histoire naturelle", "excursion", "excursions", "promenade", "promenades",
            "itinéraire", "naturaliste")]  # English and German titles: the Internet Archive and DTA cover these
LANGS = {"fre": "fr", "ger": "de", "eng": "en"}
NS = {"m": "info:lc/xmlns/marcxchange-v2", "srw": "http://www.loc.gov/zing/srw/"}


def fetch(query, start):
    params = {"version": "1.2", "operation": "searchRetrieve", "recordSchema": "unimarcXchange",
              "maximumRecords": 1000, "startRecord": start, "query": query}
    url = SRU + "?" + urllib.parse.urlencode(params)
    for attempt in range(6):
        try:
            with urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=600) as r:
                return r.read()
        except Exception as e:
            print(f"  ! {e}; retrying", file=sys.stderr)
            time.sleep(10 * 2 ** attempt)
    raise RuntimeError(url)


def harvest(cache, workers=3):
    """The SRU takes minutes per page of 1,000 records, so a few queries run side by side."""
    cache.mkdir(parents=True, exist_ok=True)
    with ThreadPoolExecutor(workers) as pool:
        list(pool.map(lambda qq: harvest_query(cache, *qq), enumerate(QUERIES)))


def harvest_query(cache, qi, q):
    start = 1
    while True:
        f = cache / f"q{qi:02d}_{start:06d}.xml"
        if not f.exists():
            f.write_bytes(fetch(q, start))
            time.sleep(1)
        root = ET.parse(f).getroot()
        total = int(root.findtext("srw:numberOfRecords", "0", NS))
        n = len(root.findall(".//srw:record", NS))
        print(f"  {q.split(' and ')[1]}: {start + n - 1}/{total}", file=sys.stderr)
        start += n
        if not n or start > total:
            break


def subfields(rec, tag, code=None):
    for df in rec.findall(f"m:datafield[@tag='{tag}']", NS):
        subs = [(sf.get("code"), (sf.text or "").strip()) for sf in df.findall("m:subfield", NS)]
        yield [v for c, v in subs if code is None or c in code]


def parse(rec):
    leader = rec.findtext("m:leader", "", NS)
    if leader[6:8] != "am":  # printed text, monograph
        return None
    f100 = next((vs[0] for vs in subfields(rec, "100", "a") if vs), "")
    year = f100[9:13] if f100[9:13].isdigit() else ""
    lang = next((LANGS[v] for vs in subfields(rec, "101", "a") for v in vs if v in LANGS), None)
    gallica = next((v for vs in subfields(rec, "856", "u") for v in vs if "gallica" in v), None)
    t = next(subfields(rec, "200", "ae"), [])
    if not (year and lang and gallica and t):
        return None
    authors = [", ".join(x for x in vs if x) for vs in subfields(rec, "700", "ab")] + \
              [", ".join(x for x in vs if x) for vs in subfields(rec, "701", "ab")]
    subjects = sorted({" -- ".join(vs) for tag in ("606", "607") for vs in subfields(rec, tag, "axyz") if vs})
    ark = rec.findtext("m:controlfield[@tag='003']", "", NS)
    title = re.sub(r"\s+([:;,])", r"\1", " : ".join(t)).replace("\u0098", "").replace("\u009c", "").rstrip(" ./:;,")
    return {"ark": ark, "title": title, "author": "; ".join(authors[:3]), "year": int(year), "language": lang,
            "subjects": " | ".join(subjects), "gallica": gallica.replace("http://", "https://"),
            "volume": next((v for vs in subfields(rec, "200", "h") for v in vs), "")}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--dir", default="cache/bnf")
    ap.add_argument("--cached", action="store_true", help="do not query again; use the cached records")
    ap.add_argument("--labels", default="bnf_labels.json")
    ap.add_argument("--workers", type=int, default=4)
    ap.add_argument("--max-year", type=int, default=1929)
    ap.add_argument("-o", "--out", default="bnf_nature_travel.csv")
    args = ap.parse_args()
    cache = Path(args.dir)
    if not args.cached:
        harvest(cache)

    recs = {}
    for f in sorted(cache.glob("q*.xml")):
        for rec in ET.parse(f).getroot().iter("{info:lc/xmlns/marcxchange-v2}record"):
            r = parse(rec)
            if r and r["year"] <= args.max_year:
                recs[r["ark"]] = r
    works = defaultdict(list)
    for r in recs.values():
        surname = norm(re.split(r"[,(]", r["author"])[0]).strip() if r["author"] else ""
        title = " ".join(norm(VOLUME.sub("", r["title"])).split()[:8])
        works[(r["language"], surname, title)].append(r)
    print(f"{len(recs)} digitised monographs up to {args.max_year}; {len(works)} works", file=sys.stderr)

    rows = []
    for items in works.values():
        lead = min(items, key=lambda r: (r["year"], r["volume"]))
        rows.append({**{k: lead[k] for k in ("ark", "title", "author", "year", "language", "subjects", "gallica")},
                     "volumes": len(items),
                     # the record id is already a URL (http://catalogue.bnf.fr/ark:/12148/cb...)
                     "url": re.sub(r"^(https?://catalogue\.bnf\.fr/)?", "https://catalogue.bnf.fr/", lead["ark"].replace("http://catalogue.bnf.fr/", ""))})
    lines = [(r["ark"], f"{r['title'][:160]} | {r['author'][:60] or 'anonymous'} | {r['year']} | {r['subjects'][:200]}")
             for r in rows]
    labels = label(lines, args.labels, args.workers, what="book in Gallica (Bibliothèque nationale de France)")
    out = []
    for r in rows:
        v = labels.get(r["ark"])
        if v and v != "neither":
            r["category"] = "natural_history+travel" if v == "both" else v
            out.append(r)
    with open(args.out, "w", encoding="utf-8", newline="") as f:
        w = csv.DictWriter(f, fieldnames=list(out[0].keys()))
        w.writeheader()
        w.writerows(sorted(out, key=lambda r: r["year"]))
    unlabelled = sum(1 for r in rows if r["ark"] not in labels)
    print(f"Wrote {len(out)} of {len(rows)} works to {args.out}"
          + (f" ({unlabelled} still unlabelled; rerun with --cached to retry)" if unlabelled else ""), file=sys.stderr)
    for k, n in sorted(Counter((r["language"], r["category"]) for r in out).items()):
        print("  ", *k, n, file=sys.stderr)


if __name__ == "__main__":
    main()