#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
# SPDX-FileCopyrightText: 2026 ICSL FZE
"""Reference reader for Goethe Format 1: a Goethe Write project (.goethe package) or exchange file (.goethex).

It prints the outline and the plain text of the manuscript, following only the published specification
(docs/format/GOETHE_FORMAT_1.md). Python 3.9 or later, standard library only.

    python3 goethe_read.py PROJECT                 outline, then the manuscript's text
    python3 goethe_read.py PROJECT --outline       only the outline
    python3 goethe_read.py PROJECT --text          only the text (--all: the texts of every root, not only the manuscript)
    python3 goethe_read.py PROJECT --json          outline, texts, footnotes, paragraph styles, character formats and Umbruch as JSON

The plain text leaves out hidden text (character format "hidden", GOETHE_FORMAT_1.md 6.2, addition 5), with the
footnotes and pictures in it, as Goethe Write's export does.

PROJECT is a .goethe (or older .lexwrite) folder or a .goethex file. Exit status 0 on success, 2 when the file is
refused (the reason is printed), 1 on other errors.

Licence: MIT (see LICENSE beside this file and ../LICENSE.md). Copyright (c) 2026 ICSL FZE.
"""

import json
import os
import re
import stat
import sys
import unicodedata
import zipfile

FORMAT = 1
MIME_TYPE = "application/vnd.goethecloud.goethex+zip"
OBJECT = 0xFFFC                     # one character for a footnote, a field or a picture
FIELD = re.compile("\u27e6(var|ref|anlage):([^\u27e7\n]{1,120})\u27e7")

# Reader limits of this implementation (GOETHE_FORMAT_1.md 4.3); not part of the format
MAX_ENTRIES = 50_000
MAX_TOTAL = 2 << 30
MAX_PATH = 1024
MAX_DEPTH = 32
RATIO_FROM = 16 << 20
RATIO = 200


class FormatError(Exception):
    """The file is refused; the message says why"""


# ---------------------------------------------------------------------------------------------------------------
# The two forms: a folder or a ZIP file, read through the same three calls

def valid_path(name):
    """A relative path inside the package: no "..", no ".", no empty part, no drive, no backslash, no control character"""
    if not name or len(name.encode("utf-8")) > MAX_PATH or name.startswith("/") or "\\" in name:
        return False
    if any(ord(c) < 0x20 or ord(c) == 0x7F for c in name):
        return False
    parts = name.split("/")
    if len(parts) > MAX_DEPTH:
        return False
    for i, part in enumerate(parts):
        if part in ("", ".", "..") or len(part.encode("utf-8")) > 255:
            return False
        if i == 0 and len(part) == 2 and part[1] == ":" and part[0].isalpha():
            return False
    return True


class Folder:
    """A .goethe package on disk. Symbolic links are never followed."""

    def __init__(self, root):
        self.root = root

    def _path(self, rel):
        if not valid_path(rel):
            raise FormatError("not allowed: %r" % rel)
        path = os.path.join(self.root, *rel.split("/"))
        here = self.root
        for part in rel.split("/"):
            here = os.path.join(here, part)
            if os.path.islink(here):
                raise FormatError("%s is a symbolic link; it is not followed" % rel)
        return path

    def read(self, rel):
        path = self._path(rel)
        if not os.path.isfile(path):
            return None
        with open(path, "rb") as f:
            return f.read()

    def isdir(self, rel):
        return os.path.isdir(self._path(rel))

    def listdir(self, rel):
        path = self._path(rel)
        return sorted(os.listdir(path)) if os.path.isdir(path) else []


class Exchange:
    """A .goethex file: a ZIP whose entry "mimetype" holds our type. Checked before anything is unpacked."""

    def __init__(self, path):
        try:
            self.zip = zipfile.ZipFile(path)
        except zipfile.BadZipFile:
            raise FormatError("not a ZIP archive, so not a Goethe exchange file")
        infos = self.zip.infolist()
        if len(infos) > MAX_ENTRIES:
            raise FormatError("more than %d entries" % MAX_ENTRIES)
        self.files, self.dirs, seen, total = {}, set(), {}, 0
        for info in infos:
            name = info.filename
            is_dir = name.endswith("/")
            path = name[:-1] if is_dir else name
            if not valid_path(path):
                raise FormatError("an entry has a name that is not allowed: %r" % name)
            parts = path.split("/")
            if parts[0] == "__MACOSX" or parts[-1] == ".DS_Store":
                continue                                            # what the Finder adds: ignored
            if info.flag_bits & 0x0001 or info.flag_bits & 0x0040 or info.flag_bits & 0x2000:
                raise FormatError("the archive is encrypted")
            if info.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
                raise FormatError("packed with an unsupported method (%d)" % info.compress_type)
            if info.create_system == 3 and stat.S_ISLNK(info.external_attr >> 16):
                raise FormatError("%r is a symbolic link" % name)
            if info.file_size >= 0xFFFFFFFF or info.compress_size >= 0xFFFFFFFF:
                raise FormatError("ZIP64 is not part of format 1")
            key = unicodedata.normalize("NFC", path).lower()        # doubled, also by case and Unicode form
            if key in seen and not (seen[key] and is_dir):
                raise FormatError("an entry is doubled: %r" % name)
            seen[key] = is_dir
            total += info.file_size
            if total > MAX_TOTAL:
                raise FormatError("larger than %d bytes unpacked" % MAX_TOTAL)
            if info.file_size >= RATIO_FROM and info.file_size > RATIO * max(1, info.compress_size):
                raise FormatError("%r unpacks suspiciously far" % name)
            if is_dir:
                self.dirs.add(path)
            else:
                self.files[path] = info
            for k in range(1, len(parts)):
                self.dirs.add("/".join(parts[:k]))
        for d in self.dirs:
            if d in self.files:
                raise FormatError("%r is a file and a folder" % d)
        mime = self.files.get("mimetype")
        if mime is None:
            raise FormatError("no entry \"mimetype\": a ZIP archive of another program")
        value = self._bytes(mime).decode("utf-8", "replace").strip()
        if value != MIME_TYPE:
            raise FormatError("its type is %r, not %r" % (value, MIME_TYPE))

    def _bytes(self, info):
        try:
            data = self.zip.read(info)                              # checks the CRC-32
        except zipfile.BadZipFile as e:
            raise FormatError("damaged: %s" % e)
        if len(data) != info.file_size:
            raise FormatError("damaged: %r has %d bytes instead of %d" % (info.filename, len(data), info.file_size))
        return data

    def read(self, rel):
        info = self.files.get(rel)
        return self._bytes(info) if info else None

    def isdir(self, rel):
        return rel in self.dirs

    def listdir(self, rel):
        prefix = rel + "/"
        names = {p[len(prefix):].split("/")[0] for p in list(self.files) + list(self.dirs) if p.startswith(prefix)}
        return sorted(n for n in names if n)


def open_project(path):
    source = Folder(path) if os.path.isdir(path) else Exchange(path)
    manifest = {"format": 1, "readableFrom": 1, "kind": "project"}
    raw = source.read("manifest.json")
    if raw is not None:
        try:
            m = json.loads(raw.decode("utf-8"))
        except ValueError:
            m = None                                                # unreadable: as if missing (format 1)
        if isinstance(m, dict):
            for key in ("format", "readableFrom"):
                v = m.get(key)
                if isinstance(v, (int, float)) and not isinstance(v, bool) and v == int(v) and v >= 0:
                    manifest[key] = int(v)
            if isinstance(m.get("kind"), str):
                manifest["kind"] = m["kind"]
            if isinstance(m.get("generator"), str):
                manifest["generator"] = m["generator"]
    if manifest["readableFrom"] > FORMAT:
        raise FormatError("needs a reader of format %d (this one reads format %d)" % (manifest["readableFrom"], FORMAT))
    if manifest["kind"] != "project":
        raise FormatError("holds content of the kind %r, not a project" % manifest["kind"])
    raw = source.read("project.json")
    if raw is None:
        raise FormatError("project.json is missing")
    try:
        project = json.loads(raw.decode("utf-8"))
    except ValueError as e:
        raise FormatError("project.json is no JSON: %s" % e)
    if not isinstance(project, dict) or not isinstance(project.get("tree"), list):
        raise FormatError("project.json has no outline (\"tree\")")
    return source, manifest, project


# ---------------------------------------------------------------------------------------------------------------
# RTF as Goethe Write writes it (GOETHE_FORMAT_1.md section 6): Cocoa RTF with footnote groups, field tokens,
# {\*\lexxstyles …}, and in RTFD {\NeXTGraphic …} pictures. Text is collected as UTF-16 code units, the unit the
# anchors of project.json count.

SKIP = {"fonttbl", "colortbl", "stylesheet", "info", "pict", "object", "header", "headerl", "headerr", "headerf",
        "footer", "footerl", "footerr", "footerf", "listtable", "listoverridetable", "revtbl", "rsidtbl", "xmlnstbl",
        "fldinst", "themedata", "colorschememapping", "latentstyles", "datastore", "generator", "filetbl",
        "pgdsctbl", "annotation", "atnid", "atnauthor", "atndate", "atntime", "atnref"}
WORDS = {"par": "\n", "sect": "\n", "cell": "\n", "nestcell": "\n", "line": "\u2028", "page": "\x0c", "tab": "\t",
         "emdash": "\u2014", "endash": "\u2013", "emspace": "\u2003", "enspace": "\u2002", "qmspace": "\u2005",
         "bullet": "\u2022", "lquote": "\u2018", "rquote": "\u2019", "ldblquote": "\u201c", "rdblquote": "\u201d",
         "zwj": "\u200d", "zwnj": "\u200c", "ltrmark": "\u200e", "rtlmark": "\u200f"}
SYMBOLS = {"\\": "\\", "{": "{", "}": "}", "~": "\u00a0", "_": "\u2011", "-": "\u00ad", "\n": "\n", "\r": "\n"}


def _letter(x):
    return 65 <= x <= 90 or 97 <= x <= 122


def _control(b, i):
    """b[i] is a backslash: (kind, name, parameter, next index)"""
    j = i + 1
    if j >= len(b):
        return "symbol", "", None, j
    if not _letter(b[j]):
        if b[j] == 0x27:                                            # \'hh
            try:
                return "hex", int(b[j + 1:j + 3].decode("ascii"), 16), None, j + 3
            except ValueError:
                return "symbol", "", None, j + 1
        return "symbol", chr(b[j]), None, j + 1
    k = j
    while k < len(b) and _letter(b[k]):
        k += 1
    m, param = k, None
    if m < len(b) and (b[m] == 0x2D or 48 <= b[m] <= 57):
        m += 1
        while m < len(b) and 48 <= b[m] <= 57:
            m += 1
        try:
            param = int(b[k:m])
        except ValueError:
            param = None
    if m < len(b) and b[m] == 0x20:                                 # the space that ends a control word
        m += 1
    return "word", b[j:k].decode("ascii"), param, m


def _group_end(b, i):
    """b[i] is "{": the index after its matching "}" """
    depth = 0
    while i < len(b):
        c = b[i]
        if c == 0x5C:
            kind, name, param, j = _control(b, i)
            i = j + (param or 0) if kind == "word" and name == "bin" else j
            continue
        if c == 0x7B:
            depth += 1
        elif c == 0x7D:
            depth -= 1
            if depth == 0:
                return i + 1
        i += 1
    return len(b)


def _units_to_text(units):
    return b"".join(u.to_bytes(2, "little") for u in units).decode("utf-16-le", "surrogatepass")


class RTF:
    """One text: `raw` (field tokens as written), `text` (as Goethe Write counts it: footnote, field and picture
    one U+FFFC each), `notes` (footnote texts in order), `objects` (what each U+FFFC is), `styles` (paragraph styles
    from {\\*\\lexxstyles}, or None), `chars` (character formats from {\\*\\lexxchars}: a list of
    {"start", "length", "marks"} counted in UTF-16 code units of `raw`, or None)"""

    def __init__(self, data):
        m = re.search(rb"\\ansicpg(\d+)", data[:400])
        self.codepage = "cp" + m.group(1).decode() if m else "cp1252"
        self.objects = []
        self.notes = []
        self.styles = _styles(data)
        # Umbruch and leaders per paragraph (6.2, addition 6); applies only when the count is the text's
        self.paras = _paras(data)
        main = self._parse(data)
        self.raw = _units_to_text(main)
        # The record applies only when it was written for a text of this length (6.2, addition 5)
        record = _chars(data)
        self.chars = None
        if record is not None and record[0] == len(main):
            self.chars = [{"start": s, "length": n, "marks": marks} for s, n, marks in record[1]]
        # The text as Goethe Write holds it: every field token is one character
        tokens = []
        for match in FIELD.finditer(self.raw):
            tokens.append((match.start(), "field", match.group(1), match.group(2)))
        self.fields = [(kind, key) for _, _, kind, key in tokens]
        self.text = FIELD.sub("\ufffc", self.raw)
        if self.paras is not None and len(self.paras) != paragraph_count(self.raw):
            self.paras = None

    def _byte(self, x):
        try:
            return ord(bytes([x]).decode(self.codepage))
        except (UnicodeDecodeError, LookupError):
            return x

    def _parse(self, b):
        main = []
        state = {"uc": 1, "buf": main, "note": False}
        stack = []
        skip = 0                  # fallback characters still to skip after \uN
        swallow = False           # after a picture: its attachment character "¬" (0xAC) is the picture itself
        i = 0
        while i < len(b):
            c = b[i]
            if c == 0x7B:                                           # {
                name, star, j = None, False, i + 1
                if j < len(b) and b[j] == 0x5C:
                    kind, name, _, j = _control(b, j)
                    if kind == "symbol" and name == "*":
                        star = True
                        kind, name, _, j = _control(b, j) if j < len(b) and b[j] == 0x5C else (None, None, None, j)
                    if kind != "word":
                        name = None
                if name == "footnote":                              # {\footnote …} and LibreOffice's {\*\footnote …}
                    skip = 0
                    swallow = False
                    if not state["note"]:
                        state["buf"].append(OBJECT)
                        self.objects.append(("footnote", len(self.notes)))
                    stack.append(state)
                    state = {"uc": state["uc"], "buf": [], "note": True, "root": True}
                    i = j
                    continue
                if name == "NeXTGraphic":                           # RTFD: {{\NeXTGraphic name \widthW \heightH …}¬}
                    end = _group_end(b, i)
                    picture = _picture(b[i:end], self.codepage)
                    state["buf"].append(OBJECT)
                    self.objects.append(("picture",) + picture)
                    swallow = True
                    i = end
                    continue
                if star or name in SKIP:
                    i = _group_end(b, i)
                    continue
                skip = 0
                stack.append(state)
                state = dict(state, root=False)
                i += 1
                continue
            if c == 0x7D:                                           # }
                skip = 0
                if state.get("root") and state["note"]:
                    self.notes.append(_note(state["buf"]))
                if stack:
                    state = stack.pop()
                i += 1
                continue
            if c == 0x5C:                                           # backslash
                kind, name, param, j = _control(b, i)
                if kind == "hex":
                    if skip:
                        skip -= 1
                    elif swallow and name == 0xAC:
                        swallow = False                             # the picture's own character: already counted
                    else:
                        swallow = False
                        state["buf"].append(self._byte(name))
                elif kind == "symbol":
                    if name in SYMBOLS:
                        skip, swallow = 0, False
                        state["buf"].append(ord(SYMBOLS[name]))
                elif name == "u" and param is not None:
                    skip, swallow = 0, False
                    state["buf"].append(param + 65536 if param < 0 else param)
                    skip = state["uc"]
                elif name == "uc":
                    state["uc"] = 1 if param is None else param
                elif name == "bin":
                    j += param or 0
                elif name in WORDS:
                    skip, swallow = 0, False
                    state["buf"].append(ord(WORDS[name]))
                else:
                    skip = 0                                        # formatting: no text
                i = j
                continue
            if c in (0x0A, 0x0D):                                   # line ends in the file are not text
                i += 1
                continue
            if skip:
                skip -= 1
            elif swallow and c == 0xAC:
                swallow = False                                     # the picture's own character: already counted
            else:
                swallow = False
                state["buf"].append(0x09 if c == 0x09 else self._byte(c))
            i += 1
        return main


def _picture(group, codepage):
    """(file name, width in points, height in points) of {\\NeXTGraphic name \\width… \\height…}"""
    text = group.decode(codepage, "replace")
    m = re.match(r"\{\\NeXTGraphic\s+(.*?)\s*(\\width(-?\d+))?\s*(\\height(-?\d+))?(\s|\\|\})", text, re.S)
    name = m.group(1).strip() if m else ""
    width = re.search(r"\\width(-?\d+)", text)
    height = re.search(r"\\height(-?\d+)", text)
    return name, (int(width.group(1)) / 20 if width else 0), (int(height.group(1)) / 20 if height else 0)


def _note(units):
    text = _units_to_text(units).replace("\ufffc", "").replace("\ufdd0", "")
    for a in ("\r\n", "\r", "\u2028", "\u2029", "\x0c", "\x0b"):
        text = text.replace(a, "\n")
    return text.strip()


def _styles(data):
    """{\\*\\lexxstyles v1 n=<P> <code>[*<count>] …}: one paragraph style per paragraph, or None"""
    m = re.search(rb"\{\\\*\\lexxstyles v1 n=(\d+)([^}]*)\}", data)
    if not m:
        return None
    styles = []
    for run in m.group(2).decode("ascii", "replace").split():
        code, _, count = run.partition("*")
        value = None if code == "-" else (int(code) if code.isdigit() else None)
        styles += [value] * (int(count) if count.isdigit() else 1)
    return styles if len(styles) == int(m.group(1)) else None


def _chars(data):
    """{\\*\\lexxchars v1 len=<L> <start>+<length>:<mark>[,<mark>] …}: (L, [(start, length, [marks])]), or None when there
    is none or it does not follow the rules (then none of it applies)"""
    m = re.search(rb"\{\\\*\\lexxchars v1 len=(\d+)([^}]*)\}", data)
    if not m:
        return None
    total, runs = int(m.group(1)), []
    for run in m.group(2).decode("ascii", "replace").split():
        span, colon, marks = run.partition(":")
        start, plus, count = span.partition("+")
        if not (colon and plus and start.isdigit() and count.isdigit() and marks):
            return None
        s, n = int(start), int(count)
        if n <= 0 or s + n > total:
            return None
        runs.append((s, n, [mark for mark in marks.split(",") if mark]))
    return total, runs


def paragraph_count(text):
    """Paragraphs as the records count them: ending at LF, CR, CRLF and U+2029; none after the last break"""
    if not text:
        return 0
    parts = re.split("\r\n|[\n\r ]", text)
    return len(parts) - (1 if parts[-1] == "" else 0)


PARA_MARKS = {"keepNext": "keepWithNext", "keepLines": "keepLines", "pageBreakBefore": "pageBreakBefore", "widowControl": "widowControl"}
LEADERS = {"dot", "hyphen", "underscore"}


def _paras(data):
    """{\\*\\lexxparas v1 n=<P> <marks>[*<count>] …}: per paragraph a dict of its own values (keepWithNext, keepLines,
    pageBreakBefore, widowControl: true or false) and "tabs" ({twips: leader}), or None for a paragraph without; None
    when there is no record or it breaks the rules (6.2, addition 6). Unknown marks are skipped."""
    m = re.search(rb"\{\\\*\\lexxparas v1 n=(\d+)([^}]*)\}", data)
    if not m:
        return None
    total, out = int(m.group(1)), []
    for run in m.group(2).decode("ascii", "replace").split():
        code, star, count = run.partition("*")
        if not code or (star and not count.isdigit()):
            return None
        n = int(count) if star else 1
        if n <= 0 or len(out) + n > total:
            return None
        value = None
        if code != "-":
            value = {}
            for mark in code.split(","):
                name, _, setting = mark.partition("=")
                if name in PARA_MARKS and setting in ("", "1", "0"):
                    value[PARA_MARKS[name]] = setting != "0"
                elif name.startswith("tab") and name[3:].isdigit() and setting in LEADERS:
                    value.setdefault("tabs", {})[int(name[3:])] = setting
        out += [value] * n
    return out if len(out) == total else None


def visible(rtf):
    """`raw` without its hidden text (character format "hidden"), and the objects of the U+FFFC characters that stay"""
    hidden = sorted((c["start"], c["start"] + c["length"]) for c in rtf.chars or [] if "hidden" in c["marks"])
    if not hidden:
        return rtf.raw, list(rtf.objects)
    objects = iter(rtf.objects)
    kept, kept_objects, pos, k = [], [], 0, 0
    for ch in rtf.raw:
        while k < len(hidden) and hidden[k][1] <= pos:
            k += 1
        shown = not (k < len(hidden) and hidden[k][0] <= pos)
        if ch == "￼":
            obj = next(objects, ("picture",))
            if shown:
                kept_objects.append(obj)
        if shown:
            kept.append(ch)
        pos += 2 if ord(ch) > 0xFFFF else 1
    return "".join(kept), kept_objects


def utf16_slice(text, location, length):
    """The passage an anchor names: `location` and `length` count UTF-16 code units"""
    b = text.encode("utf-16-le", "surrogatepass")
    n = len(b) // 2
    lo = max(0, min(location, n))
    hi = max(lo, min(location + max(0, length), n))
    return b[2 * lo:2 * hi].decode("utf-16-le", "surrogatepass")


# ---------------------------------------------------------------------------------------------------------------
# The project

class Project:
    def __init__(self, path):
        self.source, self.manifest, self.data = open_project(path)
        self.tree = [n for n in self.data["tree"] if isinstance(n, dict)]
        self.bible = self.data.get("bible") if isinstance(self.data.get("bible"), dict) else {}
        settings = self.data.get("settings")
        self.settings = settings if isinstance(settings, dict) else {}
        self.nodes = {}
        for node, _depth, _root in self.walk():
            self.nodes.setdefault(str(node.get("id", "")).upper(), node)
        self._texts = None

    def walk(self, nodes=None, depth=0, root=None):
        """Every node, depth first in outline order: (node, depth, the kind of its root)"""
        for n in self.tree if nodes is None else nodes:
            if not isinstance(n, dict):
                continue
            r = root or n.get("kind", "text")
            yield n, depth, r
            yield from self.walk(n.get("children") or [], depth + 1, r)

    def text_files(self):
        """Node id → the name of its text in texts/ (an .rtf file or an .rtfd folder)"""
        if self._texts is None:
            self._texts = {}
            for name in self.source.listdir("texts"):
                base, _, ext = name.rpartition(".")
                if ext in ("rtf", "rtfd") and base:
                    self._texts[base.upper()] = name
        return self._texts

    def text(self, node_id):
        """The RTF object of a node's text, or None"""
        name = self.text_files().get(str(node_id).upper())
        if name is None:
            return None
        rel = "texts/" + name
        if name.endswith(".rtfd"):
            inner = next((n for n in self.source.listdir(rel) if n.lower() == "txt.rtf"), None)
            if inner is None:
                return None
            rel = rel + "/" + inner
        data = self.source.read(rel)
        return RTF(data) if data is not None else None

    def field_value(self, kind, key):
        if kind == "var":
            name, _, day = key.partition("@")
            for v in self.bible.get("variables") or []:
                if isinstance(v, dict) and v.get("name") == name:
                    value = v.get("value", "")
                    if day:
                        for h in v.get("history") or []:
                            if isinstance(h, dict) and str(h.get("validFrom", ""))[:10] <= day:
                                value = h.get("value", value)
                    return value
            return "\u2039%s\u203a" % name
        node_id = key.split("#")[0].upper()
        target = self.nodes.get(node_id)
        if kind == "ref":
            if target is None:
                return "\u2039Verweis fehlt\u203a"
            heading = key.partition("#")[2]
            return "\u201e%s\u201c" % (heading or target.get("title", ""))
        if target is None:
            return "\u2039Anlage fehlt\u203a"
        prefix = self.settings.get("annexPrefix", "K")
        for n, _d, _r in self.walk():
            if n.get("kind") in ("research", "folder") and str(n.get("title", "")).lower().startswith("anlage"):
                ids = [str(c.get("id", "")).upper() for c in n.get("children") or [] if isinstance(c, dict)]
                if node_id in ids:
                    return "Anlage %s %d" % (prefix, ids.index(node_id) + 1)
        return "Anlage"

    def plain(self, rtf):
        """The text for people: fields as their values, footnotes as [^n] with the notes below, pictures left out, and
        hidden text left out with the footnotes in it"""
        # (footnotes and pictures are U+FFFC in `raw` too, in the order of `objects`; fields are still tokens there)
        raw, objects = visible(rtf)
        objects = iter(objects)
        text = FIELD.sub(lambda m: self.field_value(m.group(1), m.group(2)).replace("￼", ""), raw)
        out, notes = [], []
        for ch in text:
            if ch == "\ufffc":
                kind = next(objects, ("picture",))
                if kind[0] == "footnote":
                    notes.append(rtf.notes[kind[1]] if kind[1] < len(rtf.notes) else "")
                    out.append("[^%d]" % len(notes))
            elif ch in "\u2028\x0c":
                out.append("\n")
            else:
                out.append(ch)
        text = "".join(out)
        if notes:
            text = text.rstrip("\n") + "\n\n" + "\n".join("[^%d]: %s" % (i + 1, t.replace("\n", " ")) for i, t in enumerate(notes))
        return text

    def outline(self):
        lines = []
        for node, depth, _root in self.walk():
            title = node.get("title") or "(untitled)"
            extra = ""
            if node.get("kind") == "file" and node.get("file"):
                extra = " \u2013 %s" % node["file"]
            status = node.get("status")
            if status and status != "none":
                extra += " \u2013 %s" % status
            lines.append("%s%s [%s]%s" % ("  " * depth, title, node.get("kind", "text"), extra))
        return lines

    def texts(self, every_root=False):
        """(node, depth, RTF) for every text in reading order: the manuscript, or with every_root all roots"""
        for node, depth, root in self.walk():
            if node.get("kind") == "file" or (not every_root and root != "manuscript"):
                continue
            rtf = self.text(node.get("id", ""))
            if rtf is not None:
                yield node, depth, rtf


def main(argv):
    args = [a for a in argv if not a.startswith("--")]
    flags = {a for a in argv if a.startswith("--")}
    if len(args) != 1 or flags - {"--outline", "--text", "--json", "--all"}:
        sys.stderr.write(__doc__)
        return 1
    try:
        p = Project(args[0])
        if "--json" in flags:
            texts = []
            for node, depth, rtf in p.texts("--all" in flags):
                texts.append({"id": node.get("id"), "title": node.get("title", ""), "text": p.plain(rtf),
                              "notes": rtf.notes, "styles": rtf.styles, "chars": rtf.chars, "paras": rtf.paras, "length": len(rtf.text.encode("utf-16-le")) // 2})
            json.dump({"manifest": p.manifest, "title": p.settings.get("title", ""), "outline": p.outline(), "texts": texts},
                      sys.stdout, ensure_ascii=False, indent=2)
            sys.stdout.write("\n")
            return 0
        if "--text" not in flags:
            print("\n".join(p.outline()))
        if "--outline" not in flags:
            for node, depth, rtf in p.texts("--all" in flags):
                body = p.plain(rtf).strip("\n")
                if not body:
                    continue
                print()
                print("#" * min(6, depth + 1) + " " + (node.get("title") or "(untitled)"))
                print()
                print(body)
        return 0
    except FormatError as e:
        sys.stderr.write("refused: %s\n" % e)
        return 2
    except OSError as e:
        sys.stderr.write("cannot read: %s\n" % e)
        return 1


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))
