#!/usr/bin/env python3
"""
Shrink overview/data.json for the web: the last step of the pipeline.

Usage:
    python compact.py           # after group.py (and keyword_types.py)

The site serves data.json uncompressed, and with the Internet Archive and Gallica the
catalogue has some 29,000 works. Most catalogue and reading links follow a fixed pattern per
library (https://archive.org/details/{id}, https://www.gutenberg.org/ebooks/{id}, ...), so
this drops every link that the page can rebuild from the work's id with the same patterns
(URLS in index.html) and drops empty fields (b, s, f, sub ... when null, empty or ""). A link
that does not fit its library's pattern is kept. Running it twice changes nothing.

It also writes data.json.gz, which the page loads first and unpacks in the browser
(DecompressionStream), since the server does not compress JSON; data.json stays as the fallback.
"""
import argparse
import gzip
import json
import sys
from pathlib import Path

# Must match URLS in overview/index.html.
URLS = {
    "PG": {"u": "https://www.gutenberg.org/ebooks/{id}", "r": "https://www.gutenberg.org/cache/epub/{id}/pg{id}-images.html"},
    "IA": {"u": "https://archive.org/details/{id}", "r": "https://archive.org/details/{id}"},
    "DTA": {"u": "https://www.deutschestextarchiv.de/book/show/{id}", "r": "https://www.deutschestextarchiv.de/book/view/{id}?p=1"},
    "BHL": {"u": "https://www.biodiversitylibrary.org/bibliography/{id}"},
    "GA": {"u": "https://catalogue.bnf.fr/ark:/12148/{id}"},
    "EHD": {"u": "https://edition-humboldt.de/{id}", "r": "https://edition-humboldt.de/{id}"},
}
KEEP_EMPTY = {"t", "l", "c"}  # the page reads these on every row


def compact(entry, src):
    for k, pattern in URLS.get(src, {}).items():
        if entry.get(k) == pattern.format(id=entry["id"]):
            del entry[k]
    for k in [k for k, v in entry.items() if v in (None, "", [], {}) and k not in KEEP_EMPTY]:
        del entry[k]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--data", default="overview/data.json")
    args = ap.parse_args()
    path = Path(args.data)
    before = path.stat().st_size
    rows = json.loads(path.read_text(encoding="utf-8"))
    for r in rows:
        compact(r, r["src"])
        for e in r.get("ed", []):
            compact(e, r["src"])
    text = json.dumps(rows, ensure_ascii=False, separators=(",", ":"))
    path.write_text(text, encoding="utf-8")
    gz = path.with_suffix(".json.gz")
    gz.write_bytes(gzip.compress(text.encode("utf-8"), compresslevel=9, mtime=0))
    print(f"{path}: {before / 1e6:.1f} MB -> {path.stat().st_size / 1e6:.1f} MB; {gz.name}: {gz.stat().st_size / 1e6:.1f} MB",
          file=sys.stderr)


if __name__ == "__main__":
    main()