#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
# SPDX-FileCopyrightText: 2026 ICSL FZE
"""Tests of the reference reader (goethe_read.py) and the schemas (check_schema.py). Standard library only.

    python3 test_goethe_read.py                       the built-in package and exchange file, and the refusals
    GOETHE_SAMPLES=<folder> python3 test_goethe_read.py
                                                      also every .goethe/.goethex/.lexwrite in <folder> (Goethe Write's
                                                      self-test writes them: -goetheFormatTest … -goetheFormatFiles <folder>)

Licence: MIT (see LICENSE beside this file and ../LICENSE.md). Copyright (c) 2026 ICSL FZE.
"""

import io
import json
import os
import shutil
import sys
import tempfile
import unittest
import zipfile
from contextlib import redirect_stderr, redirect_stdout

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)

import check_schema  # noqa: E402
import goethe_read as g  # noqa: E402

MANUSCRIPT, PART, SCENE = "0B5E2A9C-1111-4000-8000-000000000001", "0B5E2A9C-1111-4000-8000-000000000002", "0B5E2A9C-1111-4000-8000-000000000003"
CHARACTERS, LENA, RESEARCH, PDF = ("0B5E2A9C-1111-4000-8000-000000000004", "0B5E2A9C-1111-4000-8000-000000000005",
                                   "0B5E2A9C-1111-4000-8000-000000000006", "0B5E2A9C-1111-4000-8000-000000000007")
EMPTY = b"{\\rtf1\\ansi\\ansicpg1252\\cocoartf2907\n\\cocoatextscaling0\\cocoaplatform0{\\fonttbl}\n{\\colortbl;\\red255\\green255\\blue255;}\n{\\*\\expandedcolortbl;;}\n}"
# As Goethe Write writes a text: Cocoa RTF, a heading and two paragraphs, a field token, a footnote group, cp1252 bytes,
# a character outside the BMP, and the record of the paragraph styles
SCENE_RTF = (
    b"{\\rtf1\\ansi\\ansicpg1252\\cocoartf2907\n"
    b"\\cocoatextscaling0\\cocoaplatform0{\\fonttbl\\f0\\fnil\\fcharset0 Charter-Roman;\\f1\\fnil\\fcharset0 HelveticaNeue-Bold;}\n"
    b"{\\colortbl;\\red255\\green255\\blue255;}\n"
    b"{\\*\\expandedcolortbl;;}{\\*\\lexxstyles v1 n=3 1 0*2}\n\n"
    b"\\deftab708\n"
    b"\\pard\\pardeftab708\\sl312\\slmult1\\sa160\\pardirnatural\\partightenfactor0\n\n"
    b"\\f1\\b\\fs42 \\cf0 Kapitel eins\\\n"
    b"\\f0\\b0\\fs30 Im Verfahren \\uc0\\u10214 var:Aktenzeichen\\u10215  traf Lena den Gr\\'fcnder."
    b"\\uc0{\\super\\chftn{\\footnote\\pard\\plain\\uc0\\f0\\fs20 {\\super\\chftn} \\u167  444 BGB, {\\i Palandt}.}}"
    b" Danach \\'84schwieg\\'93 sie.\\\n"
    b"Ein Zeichen \\uc0\\u55357 \\u56832  am Ende.}"
)
CANONICAL = "Kapitel eins\nIm Verfahren \ufffc traf Lena den Gründer.\ufffc Danach „schwieg“ sie.\nEin Zeichen \U0001F600 am Ende."
PLAIN = ("Kapitel eins\nIm Verfahren 12 O 345/26 traf Lena den Gründer.[^1] Danach „schwieg“ sie.\nEin Zeichen \U0001F600 am Ende."
         "\n\n[^1]: § 444 BGB, Palandt.")


# Character formats (6.2, addition 5): Kapitälchen, a hidden passage with a footnote in it, a passage in English and a
# mark of a later version, which readers skip
CHARS_TEXT = "Herr Berger nickte. Geheime Notiz" + chr(0xFFFC) + " hier. Ende" + chr(0xFFFC) + "."


def chars_rtf(record):
    return (b"{\\rtf1\\ansi\\ansicpg1252\\cocoartf2907\n"
            b"\\cocoatextscaling0\\cocoaplatform0{\\fonttbl\\f0\\fnil\\fcharset0 Charter-Roman;}\n"
            b"{\\colortbl;\\red255\\green255\\blue255;}\n"
            b"{\\*\\expandedcolortbl;;}{\\*\\lexxchars " + record.encode("ascii") + b"}\n\n"
            b"\\pard\\pardirnatural\\partightenfactor0\n\n"
            b"\\f0\\fs30 \\cf0 Herr Berger nickte. Geheime Notiz\\uc0{\\super\\chftn{\\footnote\\pard\\plain\\uc0\\f0\\fs20 {\\super\\chftn} Erste Note.}}"
            b" hier. Ende\\uc0{\\super\\chftn{\\footnote\\pard\\plain\\uc0\\f0\\fs20 {\\super\\chftn} Zweite Note.}}.}")


def span(s):
    i = CHARS_TEXT.index(s)
    return "%d+%d" % (i, len(s))


CHARS_RECORD = "v1 len=%d %s:sc %s:hidden %s:lang=en-GB,later" % (len(CHARS_TEXT), span("Berger"), span(" Geheime Notiz" + chr(0xFFFC) + " hier."), span("Ende"))


def utf16(s):
    return len(s.encode("utf-16-le", "surrogatepass")) // 2


def anchor(quote):
    i = CANONICAL.index(quote)
    return {"node": SCENE, "location": utf16(CANONICAL[:i]), "length": utf16(quote), "quote": quote}


def node(id, kind, title, children=(), **more):
    n = {"id": id, "kind": kind, "title": title, "synopsis": "", "status": "none", "includeInExport": True, "children": list(children),
         "notes": "", "target": 0, "pov": "", "place": "", "when": "", "threads": [], "links": [], "aliases": [], "fields": []}
    n.update(more)
    return n


PROJECT = {
    "format": 2, "dailyGoal": 0, "days": {"2026-11-12": 0}, "expanded": [PART], "selection": SCENE,
    "tree": [node(MANUSCRIPT, "manuscript", "Manuskript", [node(PART, "folder", "Teil 1", [node(SCENE, "text", "Die Akte", status="draft")])]),
             node(CHARACTERS, "characters", "Figuren", [node(LENA, "character", "Lena Berger", aliases=["Lena"])]),
             node(RESEARCH, "research", "Recherche", [node(PDF, "file", "Kaufvertrag", file="Kaufvertrag.pdf")])],
    "bible": {"variables": [{"id": "0B5E2A9C-2222-4000-8000-000000000001", "name": "Aktenzeichen", "value": "12 O 345/26", "note": "", "history": []}],
              "comments": [{"id": "0B5E2A9C-2222-4000-8000-000000000002", "anchor": anchor("Gründer"), "text": "Name prüfen", "author": "Lena",
                            "date": "2026-11-12T10:00:00Z", "resolved": False, "replies": []},
                           {"id": "0B5E2A9C-2222-4000-8000-000000000003", "anchor": anchor("am Ende"), "text": "nach dem Zeichen", "author": "Lena",
                            "date": "2026-11-12T10:00:00Z", "resolved": False, "replies": []},
                           {"id": "0B5E2A9C-2222-4000-8000-000000000004", "anchor": anchor("Gründer.\ufffc Danach"), "text": "über die Fußnote",
                            "author": "Lena", "date": "2026-11-12T10:00:00Z", "resolved": False, "replies": []}]},
    "settings": {"title": "Probe", "language": "de", "storyTools": True, "annexPrefix": "K",
                 # A heading with Umbruch of its own (5.7, Paket W5): optional keys, absent in older files
                 "styles": [{"style": 1, "family": "Helvetica Neue", "size": 21, "bold": True, "italic": False, "lineSpacing": 1.15,
                             "spaceBefore": 22, "spaceAfter": 8, "indent": 0, "alignment": "left", "pageBreakBefore": True, "keepWithNext": True},
                            {"style": 0, "family": "Charter", "size": 15, "bold": False, "italic": False, "lineSpacing": 1.3,
                             "spaceBefore": 0, "spaceAfter": 8, "indent": 0, "alignment": "left", "widowControl": False}]},
    "window": {"expanded": [PART]},
    "manuscriptGoal": {"amount": 0, "unit": "words", "bound": "atLeast", "withNotes": True},
}
MANIFEST = {"format": 1, "readableFrom": 1, "kind": "project", "generator": "test_goethe_read.py"}


def make_package(root, project=PROJECT, manifest=MANIFEST):
    os.makedirs(os.path.join(root, "texts"))
    os.makedirs(os.path.join(root, "files"))
    os.makedirs(os.path.join(root, "snapshots"))
    with open(os.path.join(root, "project.json"), "w", encoding="utf-8") as f:
        json.dump(project, f, ensure_ascii=False, indent=2)
    if manifest is not None:
        with open(os.path.join(root, "manifest.json"), "w", encoding="utf-8") as f:
            json.dump(manifest, f, indent=2)
    for id in (MANUSCRIPT, PART, CHARACTERS, RESEARCH):
        with open(os.path.join(root, "texts", id + ".rtf"), "wb") as f:
            f.write(EMPTY)
    with open(os.path.join(root, "texts", SCENE + ".rtf"), "wb") as f:
        f.write(SCENE_RTF)
    with open(os.path.join(root, "texts", LENA + ".rtf"), "wb") as f:
        f.write(EMPTY[:-1] + b"Anw\\'e4ltin.}")
    with open(os.path.join(root, "files", PDF + ".pdf"), "wb") as f:
        f.write(b"%PDF-1.7\n%%EOF\n")


def make_exchange(package, target, mime=g.MIME_TYPE, extra=()):
    """The profile of section 4: mimetype first and stored, manifest.json second, the rest in byte order, folders too"""
    entries = []
    for base, dirs, files in os.walk(package):
        rel = os.path.relpath(base, package).replace(os.sep, "/")
        if rel != ".":
            entries.append((rel + "/", None))
        for name in files:
            path = name if rel == "." else rel + "/" + name
            with open(os.path.join(base, name), "rb") as f:
                entries.append((path, f.read()))
    entries.sort(key=lambda e: e[0].rstrip("/").encode("utf-8"))
    entries.sort(key=lambda e: e[0] != "manifest.json")
    with zipfile.ZipFile(target, "w") as z:
        if mime is not None:
            z.writestr(zipfile.ZipInfo("mimetype"), mime.encode("ascii"), compress_type=zipfile.ZIP_STORED)
        for name, data in list(entries) + list(extra):
            if data is None:
                z.writestr(zipfile.ZipInfo(name), b"")
            else:
                z.writestr(name, data, compress_type=zipfile.ZIP_DEFLATED)


class Reader(unittest.TestCase):
    def setUp(self):
        self.tmp = tempfile.mkdtemp(prefix="goethe-read-")
        self.package = os.path.join(self.tmp, "Probe.goethe")
        make_package(self.package)
        self.exchange = os.path.join(self.tmp, "Probe.goethex")
        make_exchange(self.package, self.exchange)

    def tearDown(self):
        shutil.rmtree(self.tmp)

    def check_project(self, path):
        p = g.Project(path)
        self.assertEqual(p.outline(), ["Manuskript [manuscript]", "  Teil 1 [folder]", "    Die Akte [text] \u2013 draft",
                                       "Figuren [characters]", "  Lena Berger [character]",
                                       "Recherche [research]", "  Kaufvertrag [file] \u2013 Kaufvertrag.pdf"])
        texts = [(n["title"], rtf) for n, _d, rtf in p.texts()]
        self.assertEqual([t for t, _ in texts], ["Manuskript", "Teil 1", "Die Akte"])
        rtf = texts[2][1]
        self.assertEqual(rtf.text, CANONICAL)
        self.assertEqual(rtf.notes, ["§ 444 BGB, Palandt."])
        self.assertEqual(rtf.styles, [1, 0, 0])
        self.assertEqual(rtf.fields, [("var", "Aktenzeichen")])
        self.assertEqual(p.plain(rtf), PLAIN)
        # Anchors: UTF-16 code units, footnote and field one character each, the emoji two
        for c in p.bible["comments"]:
            a = c["anchor"]
            self.assertEqual(g.utf16_slice(rtf.text, a["location"], a["length"]), a["quote"])
        self.assertEqual([n["title"] for n, _d, _r in p.texts(every_root=True)][-2:], ["Lena Berger", "Recherche"])
        self.assertEqual(p.plain(p.text(LENA)), "Anwältin.")
        return p

    def test_package(self):
        p = self.check_project(self.package)
        self.assertEqual(p.manifest["format"], 1)

    def test_exchange(self):
        p = self.check_project(self.exchange)
        self.assertEqual(p.manifest["generator"], "test_goethe_read.py")
        with zipfile.ZipFile(self.exchange) as z:
            first = z.infolist()[0]
            self.assertEqual((first.filename, first.compress_type, first.extra), ("mimetype", zipfile.ZIP_STORED, b""))
        with open(self.exchange, "rb") as f:
            head = f.read(38 + len(g.MIME_TYPE))
        self.assertEqual(head[30:38], b"mimetype")
        self.assertEqual(head[38:].decode("ascii"), g.MIME_TYPE)

    def test_character_formats(self):
        p = g.Project(self.package)
        rtf = g.RTF(chars_rtf(CHARS_RECORD))
        self.assertEqual(rtf.raw, CHARS_TEXT)
        self.assertEqual([(c["start"], c["length"], c["marks"]) for c in rtf.chars],
                         [(5, 6, ["sc"]), (19, 21, ["hidden"]), (41, 4, ["lang=en-GB", "later"])])
        # The plain text leaves out the hidden passage with its footnote; the other footnote is the first
        self.assertEqual(p.plain(rtf), "Herr Berger nickte. Ende[^1].\n\n[^1]: Zweite Note.")
        self.assertEqual(rtf.notes, ["Erste Note.", "Zweite Note."])
        # A record for another length, or one against the rules, applies not at all
        everything = "Herr Berger nickte. Geheime Notiz[^1] hier. Ende[^2].\n\n[^1]: Erste Note.\n[^2]: Zweite Note."
        for record in (CHARS_RECORD.replace("len=%d" % len(CHARS_TEXT), "len=%d" % (len(CHARS_TEXT) + 1)),
                       "v1 len=%d 5+0:sc" % len(CHARS_TEXT), "v1 len=%d 40+9:hidden" % len(CHARS_TEXT), "v2 len=%d 0+4:hidden" % len(CHARS_TEXT)):
            rtf = g.RTF(chars_rtf(record))
            self.assertIsNone(rtf.chars, record)
            self.assertEqual(p.plain(rtf), everything, record)

    def test_paragraph_flow(self):
        """Umbruch and leaders (6.2, addition 6): per paragraph, only when the count agrees, unknown marks skipped"""
        def rtf(record):
            return (b"{\\rtf1\\ansi\\ansicpg1252\\cocoartf2907\n{\\fonttbl\\f0\\fnil\\fcharset0 Charter-Roman;}\n"
                    b"{\\*\\lexxparas " + record.encode("ascii") + b"}\n"
                    b"\\pard\\tqr\\tx4000\\pardirnatural\n\\f0\\fs30 Tenor:\\\nDie Klage.\\\nInhalt\tSeite 7\\\nEnde}")
        r = g.RTF(rtf("v1 n=4 keepNext,keepLines pageBreakBefore=0,widowControl=0,glanz tab4000=dot,tab12=wellen -"))
        self.assertEqual(r.paras, [{"keepWithNext": True, "keepLines": True}, {"pageBreakBefore": False, "widowControl": False},
                                   {"tabs": {4000: "dot"}}, None])
        self.assertEqual(g.RTF(rtf("v1 n=4 -*3 keepNext")).paras, [None, None, None, {"keepWithNext": True}])
        for record in ("v1 n=5 -*5", "v1 n=3 -*3", "v1 n=4 -*9", "v2 n=4 -*4", "v1 n=4 -*0 -*4", "v1 n=4 *2 -*2"):
            self.assertIsNone(g.RTF(rtf(record)).paras, record)
        # The text itself reads as before: the record is a destination
        self.assertEqual(g.RTF(rtf("v1 n=4 -*4")).text, "Tenor:\nDie Klage.\nInhalt\tSeite 7\nEnde")
        self.assertIsNone(g.RTF(SCENE_RTF).paras)

    def test_without_manifest(self):
        old = os.path.join(self.tmp, "Alt.lexwrite")
        make_package(old, manifest=None)
        self.assertEqual(g.Project(old).manifest, {"format": 1, "readableFrom": 1, "kind": "project"})

    def test_newer_versions(self):
        later = os.path.join(self.tmp, "Spaeter.goethe")
        make_package(later, manifest={"format": 2, "readableFrom": 1, "kind": "project"})
        self.assertEqual(g.Project(later).manifest["format"], 2)
        for manifest, why in (({"format": 2, "readableFrom": 2}, "format 2"), ({"format": 1, "readableFrom": 1, "kind": "screenplay"}, "screenplay")):
            path = os.path.join(self.tmp, "Neu-%s.goethe" % why.replace(" ", ""))
            make_package(path, manifest=manifest)
            with self.assertRaises(g.FormatError) as e:
                g.Project(path)
            self.assertIn(why, str(e.exception))

    def refused(self, path, why):
        with self.assertRaises(g.FormatError) as e:
            g.Project(path)
        self.assertIn(why, str(e.exception))

    def test_refusals(self):
        text = os.path.join(self.tmp, "text.goethex")
        with open(text, "w") as f:
            f.write("kein Archiv")
        self.refused(text, "not a ZIP")
        foreign = os.path.join(self.tmp, "fremd.goethex")
        make_exchange(self.package, foreign, mime=None)
        self.refused(foreign, "no entry \"mimetype\"")
        epub = os.path.join(self.tmp, "buch.goethex")
        make_exchange(self.package, epub, mime="application/epub+zip")
        self.refused(epub, "application/epub+zip")
        for name, why in (("../aussen.txt", "not allowed"), ("/etc/passwd", "not allowed"), ("C:/x.txt", "not allowed"),
                          ("texts\\..\\x", "not allowed"), ("Project.json", "doubled")):
            path = os.path.join(self.tmp, "boese-%d.goethex" % len(name))
            make_exchange(self.package, path, extra=[(name, b"x")])
            self.refused(path, why)
        link = os.path.join(self.tmp, "verweis.goethex")
        make_exchange(self.package, link)
        with zipfile.ZipFile(link, "a") as z:
            info = zipfile.ZipInfo("files/verweis")
            info.create_system = 3
            info.external_attr = (0o120777 << 16)
            z.writestr(info, b"/etc/passwd")
        self.refused(link, "symbolic link")

    def test_command_line(self):
        out, err = io.StringIO(), io.StringIO()
        with redirect_stdout(out), redirect_stderr(err):
            self.assertEqual(g.main([self.exchange]), 0)
            self.assertEqual(g.main([self.exchange, "--json"]), 0)
            self.assertEqual(g.main([os.path.join(self.tmp, "fehlt.goethex")]), 1)
        text = out.getvalue()
        self.assertIn("### Die Akte", text)
        self.assertIn("12 O 345/26", text)
        self.assertIn("\"chars\": null", text)
        bad = os.path.join(self.tmp, "kaputt.goethex")
        with open(bad, "wb") as f:
            f.write(b"PK\x03\x04 nicht mehr")
        with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
            self.assertEqual(g.main([bad]), 2)

    def test_schemas(self):
        v = check_schema.Validator(check_schema.load("project.schema.json"))
        errors, extra = [], []
        v.validate(PROJECT, v.root, "", errors, extra)
        self.assertEqual((errors, extra), ([], []))
        broken = json.loads(json.dumps(PROJECT))
        del broken["tree"][0]["id"]
        broken["bible"]["comments"][0]["anchor"]["location"] = "zwölf"
        errors = []
        v.validate(broken, v.root, "", errors, None)
        self.assertEqual(len(errors), 2)
        m = check_schema.Validator(check_schema.load("manifest.schema.json"))
        errors = []
        m.validate(MANIFEST, m.root, "", errors, [])
        self.assertEqual(errors, [])


@unittest.skipUnless(os.environ.get("GOETHE_SAMPLES"), "GOETHE_SAMPLES not set")
class Samples(unittest.TestCase):
    """What Goethe Write itself wrote: both forms read alike, every text decodes, anchors land on their quotes"""

    def test_samples(self):
        folder = os.environ["GOETHE_SAMPLES"]
        names = sorted(n for n in os.listdir(folder) if n.endswith((".goethe", ".lexwrite")))
        self.assertTrue(names, "no packages in %s" % folder)
        anchors = matched = 0
        for name in names:
            package = g.Project(os.path.join(folder, name))
            exchange_path = os.path.join(folder, name.rsplit(".", 1)[0] + ".goethex")
            texts = [(n.get("id"), package.plain(rtf)) for n, _d, rtf in package.texts(every_root=True)]
            if os.path.exists(exchange_path):
                exchange = g.Project(exchange_path)
                self.assertEqual(exchange.outline(), package.outline(), name)
                self.assertEqual([(n.get("id"), exchange.plain(rtf)) for n, _d, rtf in exchange.texts(every_root=True)], texts, name)
            cache = {}
            for key in ("comments", "sources", "roles", "tracked", "statements", "clauseUses"):
                for item in package.bible.get(key) or []:
                    a = item.get("anchor") if isinstance(item, dict) else None
                    if not a:
                        continue
                    node_id = a["node"].upper()
                    if node_id not in cache:
                        cache[node_id] = package.text(node_id)
                    if cache[node_id] is None:
                        continue
                    anchors += 1
                    got = g.utf16_slice(cache[node_id].text, a["location"], a["length"])
                    # A quote is refreshed only when the anchor is made or refreshed (GOETHE_FORMAT_1.md 6.4): one made
                    # before a footnote went into its passage lacks that footnote's character
                    self.assertIn(a["quote"], (got, got.replace("\ufffc", "")), "%s %s" % (name, key))
                    matched += got == a["quote"]
            out = io.StringIO()
            with redirect_stdout(out):
                status = check_schema.main([os.path.join(folder, name), "--strict"])
            self.assertEqual(status, 0, out.getvalue())
        print("\n%d packages, %d anchors (%d exactly on their quote)" % (len(names), anchors, matched))


if __name__ == "__main__":
    unittest.main()
