# 项目文件：crawler_project/src/catalogcrawl/runner.py
# 目录依赖下一页链接，先顺序发现；独立详情按小批并发，写入仍由当前调用层负责。
import asyncio
import logging
import math

import httpx

from .errors import CrawlError, ParseError
from .pacing import StartGate
from .parsers import parse_catalog, parse_detail
from .transport import get_response

logger = logging.getLogger(__name__)


def html_text(response):
    kind = response.headers.get("Content-Type", "").split(";", 1)[0].strip().lower()
    if kind not in {"text/html", "application/xhtml+xml"}:
        raise ParseError(f"{response.url}：期待 HTML，收到 {kind}")
    return response.text


async def discover_pages(client, state, gate, attempts):
    while (url := state.next_page()) is not None:
        response = await get_response(client, url, gate, attempts)
        links, next_url = parse_catalog(html_text(response), str(response.url))
        state.save_page(url, links, next_url)
        logger.info("目录已提交 url=%s cards=%s", url, len(links))


async def read_one(client, url, gate, attempts):
    try:
        response = await get_response(client, url, gate, attempts)
        return parse_detail(html_text(response), str(response.url))
    except CrawlError as error:
        # 预期的单项失败留待下一次启动；取消和程序错误继续向任务组传播。
        return error

# 续写同一文件：crawler_project/src/catalogcrawl/runner.py
async def download_details(client, state, gate, concurrency, attempts, stop_after):
    failures, saved, after_seq = [], 0, 0
    while stop_after is None or saved < stop_after:
        limit = (
            concurrency if stop_after is None else min(concurrency, stop_after - saved)
        )
        batch = state.pending_jobs(after_seq, limit)
        if not batch:
            break
        # 本轮只处理每项一次；失败任务仍留在库里，供下次启动重新选择。
        after_seq = batch[-1]["seq"]
        async with asyncio.TaskGroup() as group:
            tasks = [
                group.create_task(read_one(client, row["url"], gate, attempts))
                for row in batch
            ]
        for row, task in zip(batch, tasks):
            result = task.result()
            if isinstance(result, CrawlError):
                failures.append({"url": row["url"], "error": str(result)})
                logger.error("详情仍待采集 url=%s error=%s", row["url"], result)
            else:
                state.complete_job(row["url"], result)
                saved += 1
                logger.info("详情已提交 id=%s url=%s", result.source_id, row["url"])
    return {"saved_this_run": saved, "failures": failures}

# 续写同一文件：crawler_project/src/catalogcrawl/runner.py
# 先发现分页，再处理详情队列；同一个限速器和连接池供这些请求共同使用。
async def crawl(
    state,
    concurrency=2,
    interval=0.02,
    attempts=3,
    deadline=15,
    request_timeout=1,
    stop_after=None,
):
    if concurrency < 1 or attempts < 1:
        raise ValueError("并发数和尝试次数应至少为 1")
    if stop_after is not None and stop_after < 1:
        raise ValueError("暂停前本次保存的数量应至少为 1")
    if not all(
        math.isfinite(value) and value > 0 for value in (deadline, request_timeout)
    ):
        raise ValueError("超时应为正的有限秒数")
    gate = StartGate(interval)
    limits = httpx.Limits(
        max_connections=concurrency, max_keepalive_connections=concurrency
    )
    async with asyncio.timeout(deadline):
        async with httpx.AsyncClient(
            timeout=request_timeout,
            limits=limits,
            follow_redirects=True,
            trust_env=False,
            headers={"User-Agent": "ReadingNotes/1.0"},
        ) as client:
            await discover_pages(client, state, gate, attempts)
            result = await download_details(
                client, state, gate, concurrency, attempts, stop_after
            )
    return {**state.stats(), **result}