#!/usr/bin/env python3
"""
Merge the Gutenberg hits (gutenberg_nature_travel.csv), the Deutsches Textarchiv's
genre lists, the Wikisource harvest (wikisource_nature_travel.csv), the EcoCor picks
(ecocor_nature_travel.csv), the Biodiversity Heritage Library picks (bhl_nature_travel.csv),
Humboldt's travel journals (ehd_nature_travel.csv), Darwin and Wallace Online
(dwo_nature_travel.csv), the Internet Archive (ia_nature_travel.csv) and Gallica
(bnf_nature_travel.csv), and the Biblioteca Nacional de España and Universidad Complutense through
Europeana (europeana_nature_travel.csv) into one JSON file for the overview page.

Pipeline:
    python gutendex_nature_travel.py && python wikisource_nature_travel.py
    python ecocor_nature_travel.py && python bhl_nature_travel.py
    python ehd_nature_travel.py && python dwo_nature_travel.py
    python ia_nature_travel.py && python bnf_nature_travel.py && python europeana_nature_travel.py
    python build_overview.py          # -> overview/data.json
    python genres.py                  # genre per work; drops guidebooks, topographies, fiction ...
    python years.py                   # adds publication years to Gutenberg works
    python geolocate.py               # adds places, writes overview/places.json
    python keywords.py                # adds keywords
    python group.py                   # merges volumes and editions
    python keyword_types.py           # sorts keywords into kinds, -> overview/keywords.json
    python compact.py                 # drops links the page rebuilds from ids, and empty fields

Works found in more than one source are kept in each (they are different editions or
transcriptions) and marked with the other source in "also". The exception are the two large
scan libraries (the Internet Archive, Gallica, the BNE and the Complutense's Google Books): their copy of a work that another source
already has is dropped, and that source's entry is marked "also" in IA or Gallica.

DTA genres used (from deutschestextarchiv.de/list/browse):
    Belletristik, Gebrauchsliteratur > Reiseliteratur          -> travel
    Wissenschaft > Naturgeschichte, Zoologie, Biologie, Geologie -> natural_history
    Wissenschaft > Geographie   -> travel if the title names a journey, else natural_history
    mixed genres (Naturwissenschaft, Populärwissenschaft, Bericht, Ethnologie, ...) -> labelled by
        the LLM on innkube as in bhl_nature_travel.py (cached in dta_labels.json); "neither" is dropped
"""
import argparse
import csv
import difflib
import html
import json
import re
import sys
import unicodedata
import urllib.request
from collections import Counter, defaultdict
from pathlib import Path

from bhl_nature_travel import label
from gutendex_nature_travel import TITLE_RX

DTA = "https://www.deutschestextarchiv.de"
LLM = "llm"
DTA_GENRES = {
    "Belletristik": {"Reiseliteratur": ["travel"]},
    "Wissenschaft": {"Naturgeschichte": ["natural_history"], "Zoologie": ["natural_history"],
                     "Biologie": ["natural_history"], "Geologie": ["natural_history"], "Geographie": None,
                     "Naturwissenschaft": LLM, "Naturwissenschaft; Philosophie": LLM, "Ethnologie": LLM,
                     "Geographie; Ethnologie": LLM, "Geowissenschaften": LLM},
    "Gebrauchsliteratur": {"Reiseliteratur": ["travel"], "Populärwissenschaft": LLM, "Biologie": LLM,
                           "Zoologie": LLM, "Geologie": LLM, "Naturwissenschaft": LLM, "Pflanzenbuch": LLM,
                           "Bericht": LLM, "Sonstiges": LLM},
}


def fetch_dta(genre):
    # The DTA serves a JavaScript cookie check first; sending the cookie skips it.
    req = urllib.request.Request(f"{DTA}/list/browse?genre={genre}",
                                 headers={"User-Agent": "nature-travel-overview/1.0",
                                          "Cookie": "verified=1"})
    with urllib.request.urlopen(req, timeout=120) as r:
        return r.read().decode("utf-8")


def parse_dta_entry(text):
    """'Forster, Georg: Ansichten vom Niederrhein. Bd. 1. Berlin, 1791.' -> author, title, year"""
    text = " ".join(html.unescape(text).split())
    author, _, rest = text.partition(": ") if ": " in text else ("", "", text)
    years = re.findall(r"\b(1[4-9]\d\d)\b", rest)
    title = re.sub(r"\.\s*$", "", rest)
    # drop the trailing "Place, Year" imprint when present
    title = re.sub(r"\.\s*[^.]*,\s*\[?1[4-9]\d\d\]?\s*$", "", title)
    return author.strip("[] "), title, int(years[-1]) if years else None


def load_dta(labels_path="dta_labels.json"):
    rows, seen = [], {}
    for main, subs in DTA_GENRES.items():
        parts = re.split(r'<h2 id="([^"]+)">', fetch_dta(main))
        for i in range(1, len(parts), 2):
            sub = parts[i]
            if sub not in subs:
                continue
            for dta_id, text in re.findall(r'book/show/([^"]+)">\s*([^<]+)</a>', parts[i + 1]):
                author, title, year = parse_dta_entry(text)
                cats = subs[sub]
                if dta_id in seen:  # listed under two genres
                    if isinstance(cats, list):
                        seen[dta_id]["c"] = sorted(set(seen[dta_id]["c"] or []) | set(cats))
                    continue
                if cats is None:  # Geographie: decide by title
                    travel = any(rx.search(title) for rx in TITLE_RX["travel"])
                    cats = ["travel"] if travel else ["natural_history"]
                seen[dta_id] = row = {
                    "src": "DTA", "id": dta_id, "t": title, "a": author, "b": None, "y": year,
                    "k": "article" if re.search(r"\bIn: ", title) else "book",
                    "l": ["de"], "c": cats, "s": None, "f": [], "g": f"{main} › {sub}",
                    "u": f"{DTA}/book/show/{dta_id}",
                    "r": f"{DTA}/book/view/{dta_id}?p=1",  # online reader
                }
                rows.append(row)
    ask = [r for r in rows if r["c"] == LLM]
    labels = label([(r["id"], f"{r['a'] or 'anonymous'} | {r['t'][:200]} | {r['y']} | {r['g']}") for r in ask],
                   labels_path, 4, what="work in the Deutsches Textarchiv", fields="author | title | year | DTA genre")
    for r in ask:
        v = labels.get(r["id"])
        r["c"] = None if v in (None, "neither") else ["natural_history", "travel"] if v == "both" else [v]
    print(f"  DTA: {len(ask)} works in mixed genres labelled, {sum(1 for r in ask if r['c'])} kept", file=sys.stderr)
    return [r for r in rows if r["c"]]


def fmt_authors(authors):
    """'Darwin, Charles, 1809-1882; ...' -> 'Darwin, Charles (1809–1882); ...'"""
    return "; ".join(re.sub(r", (\d{0,4}\??)-(\d{0,4}\??)(?=( \[[^\]]+\])?$)", r" (\1–\2)", a)
                     for a in authors.split("; ")) if authors else ""


def load_gutenberg(path):
    rows = []
    for r in csv.DictReader(open(path, encoding="utf-8")):
        evidence = r["evidence"].split(" | ") if r["evidence"] else []
        kinds = sorted({e.split(":")[0].split("/")[1] for e in evidence})
        rows.append({
            "src": "PG", "id": r["id"], "t": r["title"],
            "a": fmt_authors(r["authors"]),
            "b": int(r["earliest_author_birth"]) if r["earliest_author_birth"] else None,
            "y": None, "k": "book", "l": r["languages"].split(","), "c": r["category"].split("+"),
            "s": int(r["score"]), "f": [f for f in r["flags"].split(",") if f],
            "g": " · ".join(kinds), "sub": r["subjects"],
            "u": r["url"], "r": f"https://www.gutenberg.org/cache/epub/{r['id']}/pg{r['id']}-images.html",  # static online reader
        })
    return rows


def load_wikisource(path):
    if not Path(path).exists():
        print(f"  ! {path} not found; run wikisource_nature_travel.py first")
        return []
    rows = []
    for r in csv.DictReader(open(path, encoding="utf-8")):
        rows.append({
            "src": "WS", "id": r["page"], "t": r["title"], "a": r["author"], "b": None,
            "y": int(r["year"]) if r["year"] else None, "k": r["type"], "l": [r.get("lang") or "de"],
            "c": r["category"].split("+"), "s": None, "f": [], "g": r["source"],
            "sum": r["summary"], "u": r["url"], "r": r["url"],
        })
    return rows


def load_ecocor(path):
    if not Path(path).exists():
        print(f"  ! {path} not found; run ecocor_nature_travel.py first")
        return []
    rows = []
    for r in csv.DictReader(open(path, encoding="utf-8")):
        rows.append({
            "src": "EC", "id": r["name"], "t": r["title"], "a": r["author"], "b": None,
            "y": int(r["year"]) if r["year"] else None, "k": "book", "l": [r["corpus"]],
            "c": r["category"].split("+"), "s": None, "f": [],
            "g": f"{r['animals']} animal and {r['plants']} plant species annotated",
            "u": r["url"], "r": r["url"],
        })
    return rows


def load_bhl(path):
    if not Path(path).exists():
        print(f"  ! {path} not found; run bhl_nature_travel.py first")
        return []
    rows = []
    for r in csv.DictReader(open(path, encoding="utf-8")):
        vols = int(r["volumes"] or 0)
        rows.append({
            "src": "BHL", "id": r["id"], "t": r["title"], "a": fmt_authors(r["author"]), "b": None,
            "y": int(r["year"]), "k": "book", "l": [r["language"]], "c": r["category"].split("+"),
            "s": None, "f": [], "g": f"{vols} volumes in BHL" if vols > 1 else "", "sub": r["subjects"],
            "u": r["url"], "r": r["read_url"],
        })
    return rows


def load_ehd(path):
    """Journals are single entries; a correspondence or the plant-geography notes come grouped, with
    their letters or notes listed under "ed" like the volumes of a work (group.py leaves them as they are)."""
    if not Path(path).exists():
        print(f"  ! {path} not found; run ehd_nature_travel.py first")
        return []
    rows = []
    kinds = {"journal": "travel journal", "letters": "correspondence", "notes": "working notes"}
    for r in csv.DictReader(open(path, encoding="utf-8")):
        row = {
            "src": "EHD", "id": r["id"], "t": r["title"], "a": r["author"], "b": None,
            "y": int(r["year"]) if r["year"] else None, "k": "book", "l": r["language"].split(","),
            "c": r["category"].split("+"), "s": None, "f": [],
            "g": f"{kinds[r['kind']]}, {int(r['words']):,} words",
            "u": r["url"], "r": r["url"],
        }
        if r["places"]:
            row["sum"] = "Places named most often: " + r["places"].replace(" | ", ", ")
        if r["items"]:
            row["ed"], row["ek"] = json.loads(r["items"]), r["kind"]
        rows.append(row)
    return rows


def load_dwo(path):
    if not Path(path).exists():
        print(f"  ! {path} not found; run dwo_nature_travel.py first")
        return []
    return [{
        "src": "DWO", "id": r["id"], "t": r["title"], "a": r["author"], "b": None,
        "y": int(r["year"]) if r["year"] else None, "k": "book", "l": [r["language"]],
        "c": r["category"].split("+"), "s": None, "f": [],
        "g": "Darwin Online" if r["site"] == "darwin" else "Wallace Online",
        "u": r["url"], "r": r["read_url"],
    } for r in csv.DictReader(open(path, encoding="utf-8"))]


def load_ia(path):
    if not Path(path).exists():
        print(f"  ! {path} not found; run ia_nature_travel.py first")
        return []
    rows = []
    for r in csv.DictReader(open(path, encoding="utf-8")):
        copies, vols = int(r["copies"]), int(r["volumes"] or 0)
        g = ", ".join(x for x in (f"{vols} volumes" if vols > 1 else "",
                                  f"{copies} scans" if copies > max(vols, 1) else "") if x)
        rows.append({
            "src": "IA", "id": r["id"], "t": r["title"], "a": fmt_authors(r["author"]), "b": None,
            "y": int(r["year"]) if r["year"] else None, "k": "book", "l": [r["language"]],
            "c": r["category"].split("+"), "s": None, "f": [], "g": g, "sub": r["subjects"],
            "u": r["url"], "r": r["url"],
        })
    return rows


def load_bnf(path):
    if not Path(path).exists():
        print(f"  ! {path} not found; run bnf_nature_travel.py first")
        return []
    return [{
        "src": "GA", "id": r["ark"].split("/")[-1], "t": r["title"], "a": r["author"], "b": None,
        "y": int(r["year"]), "k": "book", "l": [r["language"]], "c": r["category"].split("+"), "s": None, "f": [],
        "g": f"{r['volumes']} volumes" if int(r["volumes"]) > 1 else "", "sub": r["subjects"],
        "u": r["url"], "r": r["gallica"],
    } for r in csv.DictReader(open(path, encoding="utf-8"))]


def load_europeana(path):
    if not Path(path).exists():
        print(f"  ! {path} not found; run europeana_nature_travel.py first")
        return []
    return [{
        "src": r["provider"], "id": r["id"], "t": r["title"], "a": fmt_authors(r["author"]), "b": None,
        "y": int(r["year"]), "k": "book", "l": [r["language"]], "c": r["category"].split("+"), "s": None, "f": [],
        "g": f"{r['volumes']} volumes or copies" if int(r["volumes"]) > 1 else "", "sub": r["subjects"],
        "im": "; ".join(x for x in (r.get("imprint"), r.get("extent")) if x),
        "u": r["url"], "r": r["read_url"] or r["url"],
    } for r in csv.DictReader(open(path, encoding="utf-8"))]


SCANS = {"IA", "GA", "BNE", "UCM", "BNP"}


def drop_scan_duplicates(rows):
    """IA and Gallica entries for works another source already has: keep only the "also" mark."""
    keep = [r for r in rows if r["src"] not in SCANS or not set(r.get("also", [])) - SCANS]
    for src in sorted(SCANS):
        print(f"  {src}: {sum(1 for r in rows if r['src'] == src) - sum(1 for r in keep if r['src'] == src)} "
              f"works dropped as already in another source", file=sys.stderr)
    return keep


def _norm(s):
    s = unicodedata.normalize("NFD", s.replace("ſ", "s"))
    s = "".join(c for c in s if not unicodedata.combining(c)).lower()
    for a, b in (("th", "t"), ("ey", "ei"), ("ae", "a"), ("oe", "o"), ("ue", "u")):
        s = s.replace(a, b)  # old vs. modern spelling: Thier/Tier, Reyse/Reise
    return re.sub(r"[^a-z0-9 ]", "", s)


STOP = set("the of and a an in on to der die das und von vom im in nach ein eine zu um aus".split())


def _surname(author):
    first = re.sub(r"\(.*?\)|\[.*?\]", "", author.split(";")[0])
    words = _norm(first.split(",")[0] if "," in first else first).split()
    return words[-1] if words else ""


def mark_overlaps(rows):
    """Mark works that appear in more than one source (same language, same author surname
    and a similar title)."""
    by_author = defaultdict(list)
    for r in rows:
        a = _surname(r["a"])
        if a:
            by_author[a].append((r, _norm(r["t"])[:60]))
    pairs = ((x, y) for group in by_author.values() for i, x in enumerate(group) for y in group[i + 1:])
    for (r1, t1), (r2, t2) in pairs:
        if r1["src"] == r2["src"] or not set(r1["l"]) & set(r2["l"]):
            continue
        n = min(len(t1), len(t2))  # ignore trailing subtitles/translator notes
        w1, w2 = set(t1.split()) - STOP, set(t2.split()) - STOP
        words = len(w1 & w2) / max(1, len(w1 | w2))
        if difflib.SequenceMatcher(None, t1[:n], t2[:n]).ratio() > 0.8 or words >= 0.6:
            r1.setdefault("also", []).append(r2["src"])
            r2.setdefault("also", []).append(r1["src"])
    for r in rows:
        if "also" in r:
            r["also"] = sorted(set(r["also"]))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--csv", default="gutenberg_nature_travel.csv")
    ap.add_argument("--ws-csv", default="wikisource_nature_travel.csv")
    ap.add_argument("--ec-csv", default="ecocor_nature_travel.csv")
    ap.add_argument("--bhl-csv", default="bhl_nature_travel.csv")
    ap.add_argument("--ehd-csv", default="ehd_nature_travel.csv")
    ap.add_argument("--dwo-csv", default="dwo_nature_travel.csv")
    ap.add_argument("--ia-csv", default="ia_nature_travel.csv")
    ap.add_argument("--bnf-csv", default="bnf_nature_travel.csv")
    ap.add_argument("--europeana-csv", default="europeana_nature_travel.csv")
    ap.add_argument("-o", "--out", default="overview/data.json")
    args = ap.parse_args()

    rows = (load_gutenberg(args.csv) + load_dta() + load_wikisource(args.ws_csv)
            + load_ecocor(args.ec_csv) + load_bhl(args.bhl_csv) + load_ehd(args.ehd_csv) + load_dwo(args.dwo_csv)
            + load_ia(args.ia_csv) + load_bnf(args.bnf_csv) + load_europeana(args.europeana_csv))
    mark_overlaps(rows)
    rows = drop_scan_duplicates(rows)
    Path(args.out).parent.mkdir(parents=True, exist_ok=True)
    with open(args.out, "w", encoding="utf-8") as f:
        json.dump(rows, f, ensure_ascii=False, separators=(",", ":"))
    c = Counter((r["src"], lang, cat) for r in rows for lang in r["l"] if lang in ("en", "de", "fr", "es", "pt", "it")
                for cat in r["c"])
    print(f"Wrote {len(rows)} works to {args.out}")
    for r in rows:
        if "also" in r and r["src"] in ("EC", "EHD"):
            print(f"   overlap: {r['src']} {r['a']} | {r['t'][:60]} -> also in {', '.join(r['also'])}")
    for k, n in sorted(c.items()):
        print("  ", *k, n)


if __name__ == "__main__":
    main()