#!/usr/bin/env python3
"""
Charles Darwin's and Alfred Russel Wallace's books from Darwin Online and Wallace Online
(John van Wyhe, ed.; darwin-online.org.uk, wallace-online.org).

Usage:
    python dwo_nature_travel.py            # -> dwo_nature_travel.csv

Both sites run the same bibliographical database. Its search, filtered to the document type
"Book" and the author, lists every edition and translation with a concise reference
("Darwin, C. R. 1875. Reise eines Naturforschers um die Welt. Translated by J. V. Carus.
Stuttgart: ...") and links to the transcribed text or page images. Translations are recognised
by the title's function words; unless those clearly mark an English title, the language comes
from the record page, where Freeman's bibliography gives it
("[396 GERMAN"). English, German and French editions up to 1929 are kept; annotated copies are not.

With two authors and some eighty distinct titles, the selection is made by title rules rather
than the LLM (which called On the Origin of Species "neither"): the travel narratives (the
Beagle journal, the Amazon, the Malay Archipelago, Spruce's notes) are natural history and
travel; the other books on evolution, geology, distribution and the habits of plants and
animals are natural history; left out are the barnacle monographs and the specialists' parts
of The Zoology of the Voyage of H.M.S. Beagle, letters, lives and autobiographies, and
Wallace's books on spiritualism, vaccination and politics. Search and record pages are cached
in cache/dwo/.
"""
import argparse
import csv
import html
import re
import sys
import time
import urllib.parse
import urllib.request
from collections import Counter
from pathlib import Path

UA = {"User-Agent": "NaturalHistoriesCatalogue/1.0 (https://thomashaider.net/natural-histories/)"}
SITES = {"darwin": ("https://darwin-online.org.uk", "Darwin Charles Robert", "Darwin, Charles (1809–1882)"),
         "wallace": ("https://wallace-online.org", "Wallace Alfred Russel", "Wallace, Alfred Russel (1823–1913)")}
LANGS = {"ENGLISH": "en", "GERMAN": "de", "FRENCH": "fr"}
TRANSLATED = re.compile(r"\b(translated|übersetzt|übers\.|traduit|traduction|trad\.|vertaald|ausgabe)", re.I)
FUNCTION_WORDS = {"en": set("the of and on in a an with by from its their into".split()),
                  "de": set("der die das und des dem den ein eine einer eines um über von zur zum mit im aus nach".split()),
                  "fr": set("le la les des du de et un une sur dans autour chez par aux".split()),
                  "nl": set("het van en een uit den der op".split())}
TRAVEL = re.compile(r"journal of researches|voyage|beagle|reise|naturaliste autour|archipel|travels on the amazon|"
                    r"narrative of|what mr\. darwin saw|notes of a botanist", re.I)
LEAVE_OUT = re.compile(r"cirripedia|balanid|lepadid|\bpart \d.* of the zoology of the voyage|letters|autobiograph|"
                       r"selvbiografi|his life|vie et la correspondance|recollection|royal commission|vivisection|"
                       r"digest of evidence|celebrations|chronological register|portraits|woolner|barrington|"
                       r"zoology notes|land-tortoises|catalogue of|spiritualism|miracles|supernatural|vaccination|"
                       r"death penalty|land nationali|bad times|social|democracy|wonderful century|mars habitable|"
                       r"man's place|world of life|^reply to|members of parliament|^my life|erasmus darwin|"
                       r"kleinere|manual of geology|foundations of the origin|foreword", re.I)
# where the title ends in a concise reference: the imprint, translator, editor, edition or volume
TITLE_END = re.compile(r"\.\s+(?=(Translated|Edited|With|Volume|Vol\.|Part|\d+(st|nd|rd|th) |New |Revised|"
                       r"(First|Second|Third|Fourth|Fifth|Sixth|Seventh|Eighth|\w+) (and revised )?edition|"
                       r"Autori[sz]i[er]?te|Aus dem|Traduit|Traduction|Übers|"
                       r"[A-ZÄÖÜ][\w.'-]+(?:(?: |, | & | and )[A-Za-zäöü][\w.'-]+){0,6}(?: \[[^\]]*\])?:))")


def fetch(url, cache, refresh=False):
    f = cache / (re.sub(r"[^\w.-]+", "_", url.split("//", 1)[1])[:180] + ".html")
    if f.exists() and not refresh:
        return f.read_text(encoding="utf-8")
    for attempt in range(5):
        try:
            with urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=600) as r:
                text = r.read().decode("utf-8", errors="replace")
            f.write_text(text, encoding="utf-8")
            time.sleep(1)
            return text
        except Exception as e:
            print(f"  ! {e}; retrying", file=sys.stderr)
            time.sleep(10 * 2 ** attempt)
    raise RuntimeError(url)


def books(site, cache, refresh):
    base, name, _ = SITES[site]
    seen, page = {}, 1  # pages count from 1; 0 returns page 1 again
    while True:
        q = {"pagesize": 200, "sort": "identifier", "pageno": page, "name": name, "documenttype": "Book",
             **{k: "" for k in ("freetext", "allfields", "searchid", "dateafter", "datebefore", "searchtitle",
                                "description", "place", "publisher", "periodical")}}
        s = fetch(f"{base}/content/search-results?" + urllib.parse.urlencode(q), cache, refresh)
        hits = re.findall(r'<tr class="hit-item.*?</tr>', s, re.S)
        new = 0
        for h in hits:
            m = re.search(r"record\?itemID=([^\"&]+)\">", h)
            ref = re.findall(r"<span >(.*?)</span>", h, re.S)
            if not m or m.group(1) in seen or len(ref) < 2:
                continue
            new += 1
            seen[m.group(1)] = {"id": m.group(1), "ref": " ".join(html.unescape(re.sub(r"<[^>]+>", "", ref[1])).split()),
                                "text": "viewtype=text" in h, "image": "viewtype=image" in h}
        print(f"  {site} page {page}: {len(hits)} hits, {new} new", file=sys.stderr)
        if not new:
            return list(seen.values())
        page += 1


def parse_ref(ref):
    """'Darwin, C. R. 1859. On the origin of species ... London: John Murray. 1st ed.'
    -> (author as given, year, title)"""
    m = re.match(r"(.*?)\s*\[?(?:c\. )?\b(1[6-9]\d\d|20\d\d)\??[a-z]?(?:\]\.?|\.)\s+(.*)$", ref)
    if not m:
        return "", None, ref
    title = TITLE_END.split(m.group(3), 1)[0].rstrip(" .")
    return m.group(1).strip(" ."), int(m.group(2)), title


def language(site, item, cache, refresh):
    title = parse_ref(item["ref"])[2]
    if m := re.search(r"\[in (\w+)\]", title):
        return LANGS.get(m.group(1).upper(), m.group(1).lower())
    words = re.findall(r"[a-zäöüéèà]+", title.lower())
    hits = Counter(lang for w in words for lang, ws in FUNCTION_WORDS.items() if w in ws).most_common(2)
    if hits and hits[0][0] == "en" and (len(hits) == 1 or hits[0][1] > hits[1][1]) and not TRANSLATED.search(item["ref"]):
        return "en"  # other languages share too many function words (de, un, la): ask the record
    s = fetch(f"{SITES[site][0]}/content/record?itemID={item['id']}", cache, refresh)
    m = re.search(r"\[\s*\d+[a-z]?\s+([A-Z]{4,})", html.unescape(re.sub(r"<[^>]+>", " ", s)))
    if m:
        return LANGS.get(m.group(1), m.group(1).lower())
    # Freeman tags translations only: an untagged record is English unless the title says otherwise
    return "en" if (hits and hits[0][0] == "en") or (not hits and title.isascii()) else "?"


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--dir", default="cache/dwo")
    ap.add_argument("--refresh", action="store_true", help="fetch search and record pages again")
    ap.add_argument("--max-year", type=int, default=1929)
    ap.add_argument("-o", "--out", default="dwo_nature_travel.csv")
    args = ap.parse_args()
    max_year = args.max_year
    cache = Path(args.dir)
    cache.mkdir(parents=True, exist_ok=True)

    rows = []
    for site, (base, _, author) in SITES.items():
        items = books(site, cache, args.refresh)
        langs = Counter()
        for it in items:
            if not (it["text"] or it["image"]) or "Annotations in" in it["ref"]:
                continue  # bibliography entry without a digitised copy, or an annotated copy
            lang = language(site, it, cache, args.refresh)
            langs[lang] += 1
            if lang not in ("en", "de", "fr"):
                continue
            by, year, title = parse_ref(it["ref"])
            if year and year > max_year:
                continue  # modern editions and studies of Darwin's or Wallace's travels
            if not by.startswith(author.split(",")[0]):  # a book by someone else that Darwin or Wallace edited or used
                by = re.sub(r"\s+ed$", "", by)
            else:
                by = author
            view = "text" if it["text"] else "image"
            rows.append({"id": it["id"], "site": site, "title": title, "author": by, "year": year or "",
                         "language": lang, "reference": it["ref"], "url": f"{base}/content/record?itemID={it['id']}",
                         "read_url": f"{base}/content/frameset?itemID={it['id']}&viewtype={view}&pageseq=1"})
        print(f"{site}: {len(items)} book records; digitised by language: {dict(langs.most_common())}", file=sys.stderr)

    out = []
    for r in rows:
        if not LEAVE_OUT.search(r["title"]):
            r["category"] = "natural_history+travel" if TRAVEL.search(r["title"]) else "natural_history"
            if r["title"].startswith("Australasia"):
                r["category"] = "travel"  # a geographical compendium
            out.append(r)
    with open(args.out, "w", encoding="utf-8", newline="") as f:
        w = csv.DictWriter(f, fieldnames=list(out[0]))
        w.writeheader()
        w.writerows(out)
    print(f"Wrote {len(out)} of {len(rows)} editions to {args.out}", file=sys.stderr)
    for k, n in sorted(Counter((r["site"], r["language"], r["category"]) for r in out).items()):
        print("  ", *k, n, file=sys.stderr)


if __name__ == "__main__":
    main()