#!/usr/bin/env python3
"""
Collect travel writing and natural history from German, English, French, Spanish, Italian and
Portuguese Wikisource via the MediaWiki API.

Usage:
    python wikisource_nature_travel.py              # -> wikisource_nature_travel.csv
    python wikisource_nature_travel.py --langs de   # German only

German: reads every page in the categories below (minus the Meyers/Pauly encyclopedia entries
and journal index pages) and parses the page's metadata template ({{Textdaten}},
{{GartenlaubenArtikel}}, …) for author, title, year and source.

The other languages: the categories there are thinner and mix in poems, encyclopedia entries and
people, so the harvest is wider and then filtered. Candidates are the pages in the categories
below and their subcategories (two levels, minus people, periodicals, dictionaries and poetry),
plus every page whose Wikidata item has the genre travel literature, travel book, travel journal
or nature writing. Metadata comes from Wikidata (title, author, publication date) and the page
itself ({{header}} on en, {{Encabezado}} on es; the <pages> tag or the scan's Livre: index page
on fr). The LLM on
innkube (as in bhl_nature_travel.py) then labels each one natural history, travel, both or
neither; answers are cached in wikisource_labels.json. Needs INNKUBE_API_KEY.

Each work is typed as a book (standalone publication) or an article (in a periodical or
collection).
"""
import argparse
import csv
import json
import re
import sys
import time
import urllib.parse
import urllib.request
from collections import Counter

from bhl_nature_travel import label

API = "https://{lang}.wikisource.org/w/api.php"
WD_API = "https://www.wikidata.org/w/api.php"
WDQS = "https://query.wikidata.org/sparql"
UA = {"User-Agent": "nature-travel-catalogue/1.0 (https://thomashaider.net/natural-histories/)"}
CATEGORIES = {
    "travel": ["Reisebericht"],
    "natural_history": ["Zoologie", "Entomologie", "Ichthyologie", "Malakologie", "Ornithologie",
                        "Botanik", "Mykologie", "Biologie", "Naturschutz"],
}
# en/fr: root categories; their subcategories are followed two levels down.
OTHER = {
    "en": {"prefix": "Category", "cats": ["Travel", "Travelogues", "Exploration", "Natural history", "Zoology",
                                          "Botany", "Ornithology", "Entomology"]},
    "fr": {"prefix": "Catégorie", "cats": ["Récits de voyage", "Histoire naturelle", "Zoologie", "Botanique",
                                           "Ornithologie", "Entomologie", "Sciences naturelles"]},
    "es": {"prefix": "Categoría", "cats": ["Relatos de viajes", "Zoología", "Botánica", "Ciencias naturales"]},
    "it": {"prefix": "Categoria", "cats": ["Diari di viaggio", "Storia naturale", "Zoologia", "Botanica", "Scienze naturali"]},
    "pt": {"prefix": "Categoria", "cats": ["Botânica"]},
}
SKIP_CAT = re.compile(r"\b(EB1|dictionnaire|Reference Work|Periodicals|Périodiques|poetry|poésie|Guidebooks|Space|"
                      r"Botanists|Zoologists|Ornithologists|Naturalists|Travelers|Explorers|Botanistes|Zoologistes|"
                      r"Linnean|Illustrated Flora|Paleo|pathology|morphology|taxonomy|Veterinary|cognition|"
                      r"The Condor|The Zoologist|Pomme de terre|Tourist destinations)\b", re.I)
SKIP_TITLE = re.compile(r"^(RE:|Meyers|ADB:|BLKÖ:|Zedler:|Pierer:|Brockhaus:|Herders:|"
                        r"Dictionary of|The Encyclopedia|Encyclopædia|1911 Encyclop|The New Student|Nouveau Larousse)")
NOT_A_TEXT = re.compile(r"\{\{\s*(versions|disambiguation|translations|homonymie|éditions|portal)\b", re.I)
WD_GENRES = {"Q1072723": "travel", "Q1164267": "travel", "Q1503133": "travel", "Q3337126": "natural_history"}


def api(lang="de", **params):
    params.update(format="json", formatversion="2")
    url = API.format(lang=lang) + "?" + urllib.parse.urlencode(params)
    return get(url)


def get(url, data=None, accept="application/json"):
    for attempt in range(6):
        try:
            req = urllib.request.Request(url, data=data, headers={**UA, "Accept": accept})
            with urllib.request.urlopen(req, timeout=120) as r:
                return json.load(r)
        except Exception as e:
            print(f"  ! {e}; retrying", file=sys.stderr)
            time.sleep(5 * 2 ** attempt)
    raise RuntimeError(url)


def category_pages(cat, lang="de", prefix="Kategorie", kind="page"):
    params = dict(action="query", list="categorymembers", cmtitle=f"{prefix}:{cat}",
                  cmlimit=500, **({"cmnamespace": 0} if kind == "page" else {"cmtype": "subcat"}))
    while True:
        d = api(lang, **params)
        yield from (m["title"] if kind == "page" else m["title"].split(":", 1)[1]
                    for m in d["query"]["categorymembers"])
        if "continue" not in d:
            return
        params.update(d["continue"])
        time.sleep(0.5)


def wikitext(titles, lang="de", props=False):
    """Yield (title, wikitext[, wikidata id]) for up to 50 titles per request."""
    for i in range(0, len(titles), 50):
        d = api(lang, action="query", prop="revisions|pageprops" if props else "revisions", rvprop="content",
                rvslots="main", titles="|".join(titles[i:i + 50]))
        for p in d["query"]["pages"]:
            if "revisions" in p:
                text = p["revisions"][0]["slots"]["main"]["content"]
                yield (p["title"], text, p.get("pageprops", {}).get("wikibase_item")) if props else (p["title"], text)
        time.sleep(0.5)


def strip_markup(s):
    s = re.sub(r"\[\[(?:[^|\]]*\|)?([^\]]*)\]\]", r"\1", s)   # [[link|text]] -> text
    s = re.sub(r"\{\{[^{}]*\}\}", "", s)                      # simple templates
    s = re.sub(r"<[^>]+>|'''?", "", s)
    return " ".join(s.split()).strip(" .")


def parse_header(text):
    m = re.search(r"\{\{\s*([^|}\n]+?)\s*\n?\|(.*?)\n\}\}", text, re.S)
    if not m:
        return None, {}
    fields = {}
    for line in re.split(r"\n\s*\|", "\n|" + m.group(2)):
        k, _, v = line.partition("=")
        if k.strip():
            fields[k.strip().upper()] = v.strip()
    return m.group(1).strip(), fields


def template_fields(text, name="header"):
    """Fields of {{name|...}}, split at top-level pipes (en headers put several fields on one line)."""
    m = re.search(r"\{\{\s*" + name + r"\s*\|", text, re.I)
    if not m:
        return {}
    depth, i, start, parts = 0, m.end(), m.end(), []
    while i < len(text) and depth >= 0:
        two = text[i:i + 2]
        if two in ("{{", "[["):
            depth, i = depth + 1, i + 2
            continue
        if two in ("}}", "]]"):
            depth, i = depth - 1, i + 2
            if depth < 0:
                parts.append(text[start:i - 2])
            continue
        if text[i] == "|" and depth == 0:
            parts.append(text[start:i])
            start = i + 1
        i += 1
    fields = {}
    for part in parts:
        k, _, v = part.partition("=")
        fields[k.strip().upper()] = v.strip()
    return fields


def harvest_de():
    # Chapters of a multi-page work are subpages ("Work/Chapter"); collapse them into the work.
    cats = {}
    for cat_label, names in CATEGORIES.items():
        for name in names:
            for t in category_pages(name):
                if not SKIP_TITLE.match(t):
                    cats.setdefault(t.split("/")[0], set()).add(cat_label)
    print(f"de: {len(cats)} pages in the categories", file=sys.stderr)

    rows = []
    for title, text in wikitext(sorted(cats)):
        template, f = parse_header(text)
        author = strip_markup(f.get("AUTOR", "")) or "unbekannt"
        year = re.search(r"1[4-9]\d\d", f.get("ERSCHEINUNGSJAHR", "") or f.get("JAHR", "")
                         or f.get("ENTSTEHUNGSJAHR", ""))
        # HERKUNFT names the periodical or collection a piece appeared in; "off" marks a
        # standalone publication.
        source = strip_markup(f.get("HERKUNFT", "")).replace("&nbsp;", " ")
        if source.lower() == "off":
            source = ""
        if template and template.startswith("GartenlaubenArtikel"):
            source = f"Die Gartenlaube ({f.get('JAHR', '').strip()})"
        is_article = bool(source)
        rows.append({
            "title": strip_markup(f.get("TITEL", "")) or title,
            "page": title,
            "lang": "de",
            "author": author,
            "year": int(year.group(0)) if year else "",
            "type": "article" if is_article else "book",
            "source": source,
            "category": "+".join(sorted(cats[title])),
            "summary": strip_markup(f.get("KURZBESCHREIBUNG", "")),
            "url": "https://de.wikisource.org/wiki/" + urllib.parse.quote(title.replace(" ", "_")),
        })
    return rows


def walk(lang, cfg, depth=2):
    """Pages in the root categories and their subcategories -> {page: set of category names}."""
    pages, seen = {}, set()
    frontier = list(cfg["cats"])
    for level in range(depth + 1):
        nxt = []
        for cat in frontier:
            if cat in seen or SKIP_CAT.search(cat):
                continue
            seen.add(cat)
            for t in category_pages(cat, lang, cfg["prefix"]):
                if not SKIP_TITLE.match(t):
                    pages.setdefault(t, set()).add(cat)
            if level < depth:
                nxt += list(category_pages(cat, lang, cfg["prefix"], kind="subcat"))
        frontier = nxt
    print(f"{lang}: {len(pages)} pages in {len(seen)} categories", file=sys.stderr)
    return pages


def wikidata_genre_pages(lang):
    """{page title: genre category} for Wikisource pages whose Wikidata item has a travel or nature genre."""
    q = f"""SELECT ?genre ?page WHERE {{
      VALUES ?genre {{ {" ".join("wd:" + g for g in WD_GENRES)} }}
      ?item wdt:P136 ?genre . ?art schema:about ?item ; schema:isPartOf <https://{lang}.wikisource.org/> ; schema:name ?page .
    }}"""
    d = get(WDQS + "?" + urllib.parse.urlencode({"query": q}), accept="application/sparql-results+json")
    return {b["page"]["value"]: WD_GENRES[b["genre"]["value"].rsplit("/", 1)[1]] for b in d["results"]["bindings"]}


def wd_entities(ids):
    out = {}
    ids = sorted(set(ids))
    for i in range(0, len(ids), 50):
        d = get(WD_API + "?" + urllib.parse.urlencode({"action": "wbgetentities", "ids": "|".join(ids[i:i + 50]),
                                                         "props": "labels|claims", "format": "json"}))
        out.update(d.get("entities", {}))
        time.sleep(0.3)
    return out


def claim_values(e, prop):
    for c in e.get("claims", {}).get(prop, []):
        v = c["mainsnak"].get("datavalue", {}).get("value")
        if v is not None:
            yield v


def wd_meta(qids, lang):
    """{qid: (title, author, year)} from Wikidata; the year is the work's (P629) publication date if known."""
    items = wd_entities(qids)
    works = {v["id"] for e in items.values() for v in claim_values(e, "P629")}
    items.update(wd_entities(works - set(items)))
    people = {v["id"] for e in items.values() for v in claim_values(e, "P50")}
    people = wd_entities(people)
    lab = lambda e: (e.get("labels", {}).get(lang) or e.get("labels", {}).get("en") or {}).get("value", "")
    meta = {}
    for q in qids:
        e = items.get(q, {})
        work = next((items.get(v["id"], {}) for v in claim_values(e, "P629")), {})
        dates = [v["time"][1:5] for x in (work, e) for v in claim_values(x, "P577")]
        title = next((v["text"] for v in claim_values(e, "P1476")), "") or lab(e)
        authors = [lab(people.get(v["id"], {})) for v in claim_values(e, "P50")]
        meta[q] = (title, "; ".join(a for a in authors if a), int(dates[0]) if dates and dates[0].isdigit() else "")
    return meta


def harvest_other(lang, labels_path, workers):
    cfg = OTHER[lang]
    pages = walk(lang, cfg)
    for page, cat in wikidata_genre_pages(lang).items():
        if not SKIP_TITLE.match(page):
            pages.setdefault(page, set()).add(f"Wikidata genre: {cat.replace('_', ' ')}")
    # chapters are subpages of a work that is itself a candidate: keep only the work
    pages = {p: c for p, c in pages.items() if "/" not in p or p.split("/")[0] not in pages}

    texts = {t: (text, q) for t, text, q in wikitext(sorted(pages), lang, props=True) if not NOT_A_TEXT.search(text)}
    meta = wd_meta([q for _, q in texts.values() if q], lang)
    # fr: books transcluded from a scan without other metadata -> the scan's index page
    index_of = {t: m.group(1) for t, (text, _) in texts.items() if (m := re.search(r'<pages index="([^"]+)"', text))}
    index = {}
    if lang == "fr":
        for t, text in wikitext(sorted({"Livre:" + i for i in index_of.values()}), lang):
            _, f = parse_header(text)
            index[t[len("Livre:"):]] = f

    rows = []
    for page, (text, q) in texts.items():
        wt, wa, wy = meta.get(q, ("", "", ""))
        f = template_fields(text) if lang == "en" else {}
        if lang == "it":  # {{Intestazione|Nome e cognome dell'autore=|Titolo=|Anno di pubblicazione=}}
            e = template_fields(text, "intestazione")
            f = {"TITLE": e.get("TITOLO", ""), "AUTHOR": e.get("NOME E COGNOME DELL'AUTORE", ""),
                 "YEAR": e.get("ANNO DI PUBBLICAZIONE", "") or e.get("ANNO DI STESURA", "")}
        if lang == "pt":  # {{Cabeçalho|titulo=|autor=|ano=}}
            e = {k.replace("Í", "I"): v for k, v in (template_fields(text, "cabeçalho") or template_fields(text, "cabecalho")).items()}
            f = {"TITLE": e.get("TITULO", ""), "AUTHOR": e.get("AUTOR", ""), "YEAR": e.get("ANO", "")}
        if lang == "es":  # {{Encabezado|titulo=|autor=|año=|sección=|sección autor=}}; a sección is a piece in a periodical
            e = {k.replace("Í", "I"): v for k, v in template_fields(text, "encabezado").items()}
            f = {"TITLE": e.get("TITULO", ""), "AUTHOR": e.get("SECCIÓN AUTOR") or e.get("AUTOR", ""), "YEAR": e.get("AÑO", ""),
                 "SECTION": e.get("SECCIÓN", "") if e.get("SECCIÓN AUTOR") else ""}
        tag = dict(re.findall(r'(\w+)="([^"]*)"', re.search(r"<pages\b[^>]*>", text).group(0))) if "<pages" in text else {}
        idx = index.get(index_of.get(page, ""), {})
        source = ""
        if "/" in page:  # a piece in a periodical or collection
            source = page.split("/")[0]
        elif m := re.search(r"\[\[Catégorie:Articles d(?:e la|u|e l’|e l'|es|e) ([^\]]+)\]\]", text):
            source = m.group(1)
        elif lang == "es" and f.get("SECTION"):
            source = f["TITLE"]
        if source:  # the scan's title is the periodical's
            title = wt or strip_markup(f.get("SECTION", "")) or tag.get("titre") or page.split("/")[-1]
        else:
            title = (wt or strip_markup(f.get("TITLE", "")) or tag.get("titre") or strip_markup(idx.get("TITRE", ""))
                     or page.split("/")[-1])
        # fr: [[Auteur:Fritz von Dardel|M. {{sc|de Saint-Blaise}}]] -> the author page's name, not the byline
        fr_author = lambda v: "; ".join(re.findall(r"\[\[Auteur:([^|\]]+)", v or "")) or strip_markup(v or "")
        author = (wa or strip_markup(f.get("AUTHOR", "") or f.get("CONTRIBUTOR", "")) or fr_author(tag.get("auteur"))
                  or fr_author(idx.get("AUTEUR")))
        year = wy or next((int(m.group(0)) for s in (f.get("YEAR", ""), tag.get("annee", ""), idx.get("ANNEE", ""),
                                                      " ".join(re.findall(r"\[\[Cat[ée]gor(?:y|ie):(?:Articles de )?(\d{4})(?: works)?\]\]", text)))
                           if (m := re.search(r"1[4-9]\d\d", s or ""))), "")
        cats = sorted(pages[page])
        rows.append({
            "title": title, "page": page, "lang": lang, "author": author or "", "year": year,
            "type": "article" if source else "book", "source": strip_markup(source), "category": "",
            "summary": strip_markup(f.get("NOTES", ""))[:400], "cats": " | ".join(cats),
            "url": f"https://{lang}.wikisource.org/wiki/" + urllib.parse.quote(page.replace(" ", "_")),
        })

    labels = label([(f"{lang}:{r['page']}", f"{r['title'][:160]} | {r['author'][:60] or 'anonymous'} | {r['year']} | "
                     f"{r['cats'][:160]}") for r in rows], labels_path, workers,
                   what=f"text on {lang} Wikisource (poems, fiction and encyclopedia entries are 'neither')",
                   fields="title | author | year | Wikisource categories")
    out = []
    for r in rows:
        v = labels.get(f"{lang}:{r['page']}")
        if v and v != "neither":
            r["category"] = "natural_history+travel" if v == "both" else v
            out.append(r)
    print(f"{lang}: {len(out)} of {len(rows)} candidates kept", file=sys.stderr)
    return out


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("-o", "--out", default="wikisource_nature_travel.csv")
    ap.add_argument("--langs", default="de,en,fr,es,it,pt")
    ap.add_argument("--labels", default="wikisource_labels.json")
    ap.add_argument("--workers", type=int, default=4)
    args = ap.parse_args()

    rows = []
    for lang in args.langs.split(","):
        rows += harvest_de() if lang == "de" else harvest_other(lang, args.labels, args.workers)
    fields = ["title", "page", "lang", "author", "year", "type", "source", "category", "summary", "url"]
    with open(args.out, "w", newline="", encoding="utf-8") as fh:
        w = csv.DictWriter(fh, fieldnames=fields, extrasaction="ignore")
        w.writeheader()
        w.writerows(sorted(rows, key=lambda r: (r["lang"], r["year"] or 9999, r["title"])))
    c = Counter((r["lang"], r["type"], r["category"]) for r in rows)
    print(f"Wrote {len(rows)} works to {args.out}", file=sys.stderr)
    for k, n in sorted(c.items()):
        print("  ", *k, n, file=sys.stderr)


if __name__ == "__main__":
    main()