#!/usr/bin/env python3
import json, re, sys, zipfile
from pathlib import Path
from html.parser import HTMLParser

base = Path('/home/spazio/human_design_sources')
manifest = json.loads((base/'manifest.json').read_text(encoding='utf-8'))
textdir = Path(manifest['texts_dir'])
textdir.mkdir(parents=True, exist_ok=True)

class TextExtractor(HTMLParser):
    def __init__(self):
        super().__init__()
        self.parts = []
    def handle_data(self, data):
        data = data.strip()
        if data:
            self.parts.append(data)
    def text(self):
        return '\n'.join(self.parts)

def extract_pdf(path: Path) -> str:
    import fitz
    doc = fitz.open(str(path))
    chunks = []
    meta = doc.metadata or {}
    chunks.append(f"SOURCE_FILE: {path.name}\nTITLE: {meta.get('title') or path.stem}\nPAGES: {doc.page_count}\n")
    for i, page in enumerate(doc, 1):
        txt = page.get_text('text') or ''
        chunks.append(f"\n\n--- PAGE {i} ---\n{txt}")
    return ''.join(chunks)

def extract_epub(path: Path) -> str:
    chunks = [f"SOURCE_FILE: {path.name}\nTITLE: {path.stem}\nFORMAT: EPUB\n"]
    with zipfile.ZipFile(path) as z:
        names = [n for n in z.namelist() if n.lower().endswith(('.xhtml','.html','.htm'))]
        for n in names:
            raw = z.read(n).decode('utf-8', errors='ignore')
            parser = TextExtractor(); parser.feed(raw)
            text = parser.text()
            if text.strip():
                chunks.append(f"\n\n--- EPUB ITEM {n} ---\n{text}")
    return ''.join(chunks)

def main():
    ok = fail = skip = 0
    for rec in manifest['files']:
        src = Path(rec['path'])
        out = Path(rec['text_path'])
        if out.exists() and out.stat().st_size > 500:
            skip += 1
            continue
        try:
            if src.suffix.lower() == '.pdf':
                text = extract_pdf(src)
            elif src.suffix.lower() == '.epub':
                text = extract_epub(src)
            else:
                continue
            # Normalize excessive blank lines but preserve page markers.
            text = re.sub(r'\n{4,}', '\n\n\n', text)
            out.write_text(text, encoding='utf-8')
            ok += 1
            print(f"OK {src.name} -> {out.name} ({len(text)} chars)", flush=True)
        except Exception as e:
            fail += 1
            print(f"FAIL {src.name}: {type(e).__name__}: {e}", flush=True)
    print(f"DONE extracted={ok} skipped={skip} failed={fail}", flush=True)

if __name__ == '__main__':
    main()