#!/usr/bin/env python3
"""Mirror the public writer files listed by euskerazaleak.biz."""

from __future__ import annotations

import argparse
import csv
import json
import os
import time
from html.parser import HTMLParser
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import quote, urljoin, urlsplit, urlunsplit
from urllib.request import Request, urlopen


BASE_URL = "https://www.euskerazaleak.biz/"
INDEX_URL = urljoin(BASE_URL, "?module=Idazleak")
USER_AGENT = "Euskerazaleak-public-archive/1.0 (resumable downloader)"
PAGE_DELAY_SECONDS = 0.25
RETRIES = 4


class Links(HTMLParser):
    def __init__(self) -> None:
        super().__init__()
        self.hrefs: list[str] = []

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        if tag == "a":
            href = dict(attrs).get("href")
            if href:
                self.hrefs.append(href)


def request_url(url: str) -> str:
    """Encode a URL while retaining path and query separators."""
    parts = urlsplit(url)
    return urlunsplit(
        (parts.scheme, parts.netloc, quote(parts.path, safe="/%"), quote(parts.query, safe="=&/%"), "")
    )


def fetch_page(url: str) -> str:
    response = urlopen(Request(request_url(url), headers={"User-Agent": USER_AGENT}), timeout=60)
    with response:
        content = response.read()
        # The server labels HTML as ISO-8859-1, but its URLs and text are UTF-8.
        try:
            return content.decode("utf-8")
        except UnicodeDecodeError:
            charset = response.headers.get_content_charset() or "iso-8859-1"
            return content.decode(charset, errors="replace")


def links_from(page: str, source_url: str) -> list[str]:
    parser = Links()
    parser.feed(page)
    return [urljoin(source_url, repair_mojibake(href)) for href in parser.hrefs]


def repair_mojibake(value: str) -> str:
    """Repair UTF-8 bytes that the site has incorrectly emitted as Latin-1 text."""
    if "Ã" not in value and "Â" not in value:
        return value
    try:
        return value.encode("latin-1").decode("utf-8")
    except UnicodeError:
        return value


def writer_pages() -> list[str]:
    page = fetch_page(INDEX_URL)
    pages = []
    for link in links_from(page, INDEX_URL):
        query = urlsplit(link).query
        if "module=Liburuak" in query and "DirToIndex=" in query:
            pages.append(link)
    return list(dict.fromkeys(pages))


def file_links(pages: list[str]) -> list[str]:
    files: list[str] = []
    for number, page_url in enumerate(pages, start=1):
        print(f"[{number}/{len(pages)}] indexing {page_url}", flush=True)
        for link in links_from(fetch_page(page_url), page_url):
            path = urlsplit(link).path
            if path.startswith("/PDF/idazleak/"):
                files.append(link)
        time.sleep(PAGE_DELAY_SECONDS)
    return list(dict.fromkeys(files))


def relative_path(url: str) -> Path:
    path = Path(urlsplit(url).path.lstrip("/"))
    if not path.parts or path.parts[0] != "PDF" or ".." in path.parts:
        raise ValueError(f"unexpected file URL: {url}")
    return path


def write_manifest(files: list[str], output: Path) -> None:
    output.parent.mkdir(parents=True, exist_ok=True)
    with output.open("w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=["url", "local_path", "status", "bytes", "error"])
        writer.writeheader()
        for url in files:
            writer.writerow(
                {"url": url, "local_path": str(relative_path(url)), "status": "pending", "bytes": "", "error": ""}
            )


def load_manifest(path: Path) -> list[dict[str, str]]:
    with path.open(newline="", encoding="utf-8") as handle:
        return list(csv.DictReader(handle))


def save_manifest(rows: list[dict[str, str]], path: Path) -> None:
    with path.open("w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=["url", "local_path", "status", "bytes", "error"])
        writer.writeheader()
        writer.writerows(rows)


def content_length(response) -> int | None:
    value = response.headers.get("Content-Length")
    return int(value) if value and value.isdigit() else None


def download(url: str, target: Path) -> int:
    target.parent.mkdir(parents=True, exist_ok=True)
    partial = target.with_name(target.name + ".part")
    offset = partial.stat().st_size if partial.exists() else 0
    headers = {"User-Agent": USER_AGENT}
    if offset:
        headers["Range"] = f"bytes={offset}-"

    with urlopen(Request(request_url(url), headers=headers), timeout=120) as response:
        total = content_length(response)
        if offset and response.status != 206:
            offset = 0
            partial.unlink(missing_ok=True)
        mode = "ab" if offset else "wb"
        with partial.open(mode) as handle:
            while chunk := response.read(1024 * 1024):
                handle.write(chunk)

    final_size = partial.stat().st_size
    expected = offset + total if total is not None else None
    if expected is not None and final_size != expected:
        raise IOError(f"incomplete transfer: received {final_size} of {expected} bytes")
    os.replace(partial, target)
    return final_size


def run_download(rows: list[dict[str, str]], root: Path, manifest: Path) -> None:
    completed = failed = 0
    for number, row in enumerate(rows, start=1):
        target = root / row["local_path"]
        if row["status"] == "complete" and target.exists() and str(target.stat().st_size) == row["bytes"]:
            completed += 1
            continue
        if target.exists():
            row.update(status="complete", bytes=str(target.stat().st_size), error="")
            save_manifest(rows, manifest)
            completed += 1
            continue

        for attempt in range(1, RETRIES + 1):
            try:
                print(f"[{number}/{len(rows)}] downloading {row['local_path']}", flush=True)
                row.update(status="complete", bytes=str(download(row["url"], target)), error="")
                completed += 1
                break
            except (HTTPError, URLError, OSError, TimeoutError) as error:
                row.update(status="failed", error=f"attempt {attempt}: {error}")
                if attempt < RETRIES:
                    time.sleep(attempt * 2)
                else:
                    failed += 1
                    print(f"  failed: {row['error']}", flush=True)
            finally:
                save_manifest(rows, manifest)
    print(json.dumps({"complete": completed, "failed": failed, "total": len(rows)}))


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--output", type=Path, default=Path("euskerazaleak-idazleak"))
    parser.add_argument("--index-only", action="store_true")
    arguments = parser.parse_args()
    manifest = arguments.output / "manifest.csv"

    files = file_links(writer_pages())
    write_manifest(files, manifest)
    print(f"indexed {len(files)} files in {manifest}")
    if not arguments.index_only:
        run_download(load_manifest(manifest), arguments.output, manifest)


if __name__ == "__main__":
    main()