#!/usr/bin/env python3
"""Build data/dict-<lang>.txt, the word list behind the /<lang>/ tools.

    python3 build_dict.py es
    python3 build_dict.py fr

Sources (fetched at run time, nothing vendored), one hunspell dictionary
expanded with spylls (pure-python hunspell, MPL 2.0) into every inflected
form (plurals, feminines, full conjugations), plus an OpenSubtitles 2018
frequency list from hermitdave/FrequencyWords (MIT) for "common first":
  es  rla-es (Recursos Lingüísticos Abiertos del Español), pan-Spanish edition,
      https://github.com/sbosio/rla-es, GPLv3 / LGPLv3 / MPL 1.1; taken under MPL 1.1.
  de  igerman98 + frami (Björn Jacke, Franz Michael Baumann), GPL v2 or v3: the LibreOffice
      de_DE_frami dictionary, expanded the same way. Proper nouns are removed with the
      igerman98 source tarball, whose word files are sorted by category (namen, vornamen,
      geografie, marken, abkuerzungen...); compound-only stems (flag o), forbidden words
      (d) and affix-only stems (h) are dropped. igerman98 builds everyday compounds
      (Bahnhof, Autobahn, Fußball) by hunspell compounding rather than listing them, so
      affix expansion alone misses them: every frequency-list token seen 3+ times is also
      run through the full hunspell check (spylls lookup, compounding included) and kept
      if it passes. The derived list is GPL: see LICENSES.txt.
  fr  Grammalecte (Olivier R., https://grammalecte.net/), MPL 2.0: the tagged master
      FRANCAIS.dic (all spelling variants, with po:/lx: fields) from the Pofilo GitHub
      mirror, read with the LibreOffice fr.aff. The tags let us drop abbreviations,
      symbols and proper nouns (cd, cf, ab, Paris) that a plain word list keeps.

Normalisation follows how the games treat letters. Spanish Scrabble,
Apalabrados and the Spanish Wordles: accents and diaeresis stripped, Ñ its own
letter, so "cómo" and "como" are one entry. French Scrabble, Sutom and Tusmo:
accents and cedilla stripped, œ -> oe, æ -> ae, a-z only. German Scrabble, Wordfeud
and the German Wordles: ä ö ü are letters of their own, ß is written ss, other
accents stripped; nouns are lowercased. Capitalised entries (proper nouns) are
dropped for es/fr; hyphens and apostrophes dropped everywhere; 2..15 letters.

Format is the same as dict.txt: "word<TAB>bucket" with bucket 1-8 =
floor(log10(count))+1, no tab = never seen in the corpus. The tools label
bucket >= 4 (1,000+ occurrences) as common.

    pip install --target /tmp/pylib spylls && PYTHONPATH=/tmp/pylib python3 build_dict.py es
"""
import collections, io, os, re, sys, unicodedata, urllib.request, zipfile

HERE = os.path.dirname(os.path.abspath(__file__))
FREQ = "https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2018/{lang}/{lang}_full.txt"
IGERMAN98 = "https://www.j3e.de/ispell/igerman98/dict/igerman98-20161207.tar.bz2"
PROPER_DE = ("namen", "vornamen", "geografie", "marken", "zeitgeschichte", "orgabk", "abkuerz", "roemisch", "alphabeta", "abc-", "infoabk")
LANGS = {
    "es": {"oxt": "https://github.com/sbosio/rla-es/releases/download/v2.9/es.oxt", "base": "es", "keep": "ñ", "ok": re.compile(r"^[a-zñ]+$")},
    "fr": {"files": {"fr.dic": "https://raw.githubusercontent.com/Pofilo/grammalecte/master/gc_lang/fr/dictionnaire/orthographe/FRANCAIS.dic",
                     "fr.aff": "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/fr_FR/dictionaries/fr.aff"},
           "base": "fr", "keep": "", "ok": re.compile(r"^[a-z]+$"),
           # part-of-speech tags that are words a game accepts; everything else (npr/prn/patr proper nouns,
           # mg abbreviations, sign, ponc, symb, div, pfx, sfx, err, nbro ordinals like 1er) is dropped
           "pos": re.compile(r"^(nom|adj|v[123]|adv|interj|prep|det|pro|cj|negadv|nb|infi)$|^(nom|adj|v[123]|adv|det|pro|cj)[_a-z0-9]"),
           "drop_lx": {"abr", "sig", "symb"}},
    "de": {"files": {"de.dic": "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/de/de_DE_frami.dic",
                     "de.aff": "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/de/de_DE_frami.aff"},
           "base": "de", "keep": "äöü", "ok": re.compile(r"^[a-zäöü]+$"), "caps": True,
           "drop_flags": set("od"), "stem_only_flags": set("h"), "proper_tarball": IGERMAN98, "validate_min": 3},
}
LANG = sys.argv[1] if len(sys.argv) > 1 else "es"
CFG = LANGS[LANG]
OK = CFG["ok"]


def norm(w):
    w = w.lower().replace("œ", "oe").replace("æ", "ae").replace("ß", "ss")
    keep = CFG["keep"]
    for i, ch in enumerate(keep): w = w.replace(ch, chr(1 + i))
    w = "".join(c for c in unicodedata.normalize("NFD", w) if unicodedata.category(c) != "Mn")
    for i, ch in enumerate(keep): w = w.replace(chr(1 + i), ch)
    return w


def proper_nouns_de(work):
    """Stems of the igerman98 word files that hold names, places, brands and abbreviations."""
    import tarfile
    tb = os.path.join(work, "igerman98.tar.bz2")
    if not os.path.exists(tb):
        urllib.request.urlretrieve(IGERMAN98, tb)
    out = set()
    with tarfile.open(tb) as t:
        for m in t.getmembers():
            name = os.path.basename(m.name)
            if "/dicts/" in m.name and name.endswith(".txt") and name.startswith(PROPER_DE):
                for line in t.extractfile(m).read().decode("latin-1").split("\n"):
                    w = line.split("/")[0].split("#")[0].replace("--x", "").replace("qq", "").strip()
                    # igerman98 source encoding: a" -> ä, sS -> ß
                    w = w.replace('a"', "ä").replace('o"', "ö").replace('u"', "ü").replace('A"', "Ä").replace('O"', "Ö").replace('U"', "Ü").replace("sS", "ß")
                    if w: out.add(w)
    return out


def expand(dic):
    """All surface forms of a spylls Dictionary: stem, one or two suffixes, prefixes, cross products."""
    aff = dic.aff
    def sfx(word, flags):
        out = set()
        for f in flags:
            for s in aff.SFX.get(f, []):
                if s.cond_regexp.search(word) and (not s.strip or word.endswith(s.strip)):
                    out.add(((word[:-len(s.strip)] if s.strip else word) + s.add, frozenset(s.flags), s.crossproduct))
        return out
    def pfx(word, flags):
        out = set()
        for f in flags:
            for p in aff.PFX.get(f, []):
                if p.cond_regexp.search(word) and (not p.strip or word.startswith(p.strip)):
                    out.add((p.add + (word[len(p.strip):] if p.strip else word), frozenset(p.flags), p.crossproduct))
        return out
    words = set()
    for w in dic.dic.words:
        base, flags = w.stem, set(w.flags)
        forms = set() if flags & CFG.get("stem_only_flags", set()) else {base}
        sf = sfx(base, flags)
        for form, fl2, _ in sf:
            forms.add(form)
            for form2, _, _ in sfx(form, set(fl2)):
                forms.add(form2)
        for form, _, cross in pfx(base, flags):
            forms.add(form)
            if cross:
                for form2, _, c2 in sf:
                    if c2 and form.endswith(base):
                        forms.add(form[:-len(base)] + form2)
        words |= forms
    return words


def main():
    from spylls.hunspell import Dictionary
    work = os.path.join(HERE, f"_{LANG}_src"); os.makedirs(work, exist_ok=True)
    if "oxt" in CFG:
        oxt = os.path.join(work, f"{LANG}.oxt")
        if not os.path.exists(oxt):
            urllib.request.urlretrieve(CFG["oxt"], oxt)
        with zipfile.ZipFile(oxt) as z:
            for n in (f"{CFG['base']}.dic", f"{CFG['base']}.aff"):
                z.extract(n, work)
    else:
        for n, url in CFG["files"].items():
            if not os.path.exists(os.path.join(work, n)):
                urllib.request.urlretrieve(url, os.path.join(work, n))
    dic = Dictionary.from_files(os.path.join(work, CFG["base"]))
    if "drop_flags" in CFG:
        proper = proper_nouns_de(work) if CFG.get("proper_tarball") else set()
        keep = [w for w in dic.dic.words if not (set(w.flags) & CFG["drop_flags"]) and w.stem not in proper]
        print(f"{len(keep):,} of {len(dic.dic.words):,} entries kept ({len(proper):,} proper-noun stems, compound-only and forbidden flags dropped)", file=sys.stderr)
        dic.dic.words = keep
    if "pos" in CFG:
        keep = [w for w in dic.dic.words if any(CFG["pos"].match(t) for t in w.data.get("po", [])) and not (set(w.data.get("lx", [])) & CFG["drop_lx"])]
        print(f"{len(keep):,} of {len(dic.dic.words):,} entries kept by part of speech", file=sys.stderr)
        dic.dic.words = keep
    raw = expand(dic)
    words = set()
    for w in raw:
        if w and (w[0].islower() or CFG.get("caps")):
            n = norm(w)
            if OK.match(n) and 2 <= len(n) <= 15:
                words.add(n)
    fq = os.path.join(work, f"{LANG}_full.txt")
    if not os.path.exists(fq):
        urllib.request.urlretrieve(FREQ.format(lang=LANG), fq)
    if CFG.get("validate_min"):
        # compounds and anything else the affix expansion cannot produce: accept attested tokens the
        # full checker (compounding included) validates, in either case; proper nouns stay out
        lower_proper = {norm(w) for w in proper}
        cache = os.path.join(work, "validated.txt")   # the full check takes ~15 min; keep its verdicts
        if os.path.exists(cache):
            accepted = set(open(cache, encoding="utf-8").read().split())
        else:
            accepted = set()
            for line in open(fq, encoding="utf-8"):
                p = line.split()
                if len(p) != 2 or not p[1].isdigit() or int(p[1]) < CFG["validate_min"]: continue
                tok = p[0]
                n = norm(tok)
                if n in words or not OK.match(n) or not 2 <= len(n) <= 15: continue
                try:
                    ok = dic.lookup(tok) or dic.lookup(tok.capitalize())
                except Exception:
                    ok = False   # spylls trips on a few odd tokens; they are not words we want
                if ok: accepted.add(n)
            open(cache, "w", encoding="utf-8").write("\n".join(sorted(accepted)))
        added = {n for n in accepted if n not in words and n not in lower_proper}
        words |= added
        print(f"{len(added):,} attested words added by full hunspell validation", file=sys.stderr)
    freq = collections.Counter()
    for line in open(fq, encoding="utf-8"):
        p = line.split()
        if len(p) == 2 and p[1].isdigit():
            n = norm(p[0])
            if n in words:
                freq[n] += int(p[1])
    import math
    bucket = {w: min(8, int(math.log10(c)) + 1) for w, c in freq.items()}
    out = os.path.join(HERE, f"dict-{LANG}.txt")
    with open(out, "w", encoding="utf-8") as f:
        for w in sorted(words):
            f.write(f"{w}\t{bucket[w]}\n" if w in bucket else f"{w}\n")
    dist = collections.Counter(bucket.values())
    print(f"{len(words):,} words, {len(bucket):,} with a frequency bucket; common (>=4): {sum(v for k, v in dist.items() if k >= 4):,}; "
          f"5-letter: {sum(1 for w in words if len(w) == 5):,}; {os.path.getsize(out):,} bytes -> {out}")
    print("buckets:", sorted(dist.items()))


if __name__ == "__main__":
    main()