feat(p0): add favorite folder sync and sqlite data foundation

Implement P0 of the reaction dashboard: config loading with hot reload, SQLite migrations with backups, single-writer queue, read-only Bilibili fav client, folder import/diff sync, invalid-resource handling, blacklist seed, scheduler, and JSON redacted logging. Includes offline unit and transport-level contract tests; all existing and new tests pass.

reaction_board/bilibili/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+"""B 站接口访问层。"""

reaction_board/bilibili/auth.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+"""B 站 Cookie 与请求头工具。
+
+从 ``favsort.py`` 抽取，供现有排序工具和看板共用，避免出现两套不一致的
+身份 Cookie / 设备指纹处理逻辑。
+"""
+
+from __future__ import annotations
+
+import random
+import urllib.parse
+import uuid
+
+IDENTITY_KEYS = {
+    "sessdata",
+    "bili_jct",
+    "dedeuserid",
+    "dedeuserid__ckmd5",
+    "bili_ticket",
+    "bili_ticket_expires",
+    "sid",
+    "b_lsid",
+}
+
+DROP_STALE = {"sid", "b_lsid"}  # 会话级字段，刷新时应丢弃旧值
+
+
+def strip_identity(cookie: str) -> str:
+    """剥离账号身份类 cookie，保留 buvid3 / buvid4 / _uuid 等设备指纹。
+
+    实测结论：
+      - ids：完全无 cookie 也能读（连私密收藏夹都返回全量）；
+      - infos：不需要账号身份，但缺设备指纹 cookie 会被 HTTP 412 拦；
+      - sort：必须带完整 cookie + csrf，不能走这里。
+    """
+    kept = []
+    for part in (cookie or "").split(";"):
+        part = part.strip()
+        if not part:
+            continue
+        key = part.split("=", 1)[0].strip().lower()
+        if key in IDENTITY_KEYS:
+            continue
+        kept.append(part)
+    return "; ".join(kept)
+
+
+def make_headers(cfg: dict, auth: bool = False) -> dict[str, str]:
+    """构造 B 站 Web 请求头；``auth=False`` 时去掉账号身份 cookie。"""
+    return {
+        "accept": "*/*",
+        "accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+        "cookie": cfg["cookie"] if auth else strip_identity(cfg["cookie"]),
+        "dnt": "1",
+        "origin": "https://www.bilibili.com",
+        "referer": "https://www.bilibili.com/",
+        "sec-ch-ua": '"Google Chrome";v="155", "Chromium";v="155", "Not(A:Brand";v="24"',
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": '"Windows"',
+        "sec-fetch-dest": "empty",
+        "sec-fetch-mode": "cors",
+        "sec-fetch-site": "same-site",
+        "user-agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+            "(KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36"
+        ),
+    }
+
+
+def parse_cookie(cookie: str) -> list[tuple[str, str]]:
+    """把 cookie 字符串解析成 [(key, value)]，保留原顺序。"""
+    pairs = []
+    for part in (cookie or "").split(";"):
+        part = part.strip()
+        if not part or "=" not in part:
+            continue
+        k, v = part.split("=", 1)
+        pairs.append((k.strip(), v.strip()))
+    return pairs
+
+
+def cookie_value(cookie: str, name: str) -> str:
+    name = name.lower()
+    for k, v in parse_cookie(cookie):
+        if k.lower() == name:
+            return v
+    return ""
+
+
+def cookies_from_set_cookie(headers) -> dict[str, str]:
+    """从 Set-Cookie 头里取 name=value（忽略 Path/Domain/Expires 等属性）。"""
+    pairs = {}
+    for h in headers or []:
+        first = h.split(";", 1)[0].strip()
+        if "=" not in first:
+            continue
+        k, v = first.split("=", 1)
+        pairs[k.strip()] = v.strip()
+    return pairs
+
+
+def merge_cookie(old: str, new_pairs: dict[str, str]) -> str:
+    """只替换身份类 cookie，其它字段（设备指纹等）原样保留。
+
+    新值里没有的身份字段不会被删除（例如服务端没下发新的 sid 时保留旧值），
+    只有 DROP_STALE 里的会话字段会在刷新时清掉。
+    """
+    new_l = {
+        k.lower(): (k, v)
+        for k, v in new_pairs.items()
+        if k.lower() in IDENTITY_KEYS
+    }
+    out, seen = [], set()
+    for k, v in parse_cookie(old):
+        kl = k.lower()
+        if kl in new_l:
+            nk, nv = new_l[kl]
+            out.append((nk, nv))
+            seen.add(kl)
+        elif kl in DROP_STALE:
+            continue
+        else:
+            out.append((k, v))
+            seen.add(kl)
+    for kl, (k, v) in new_l.items():
+        if kl not in seen:
+            out.append((k, v))
+    return "; ".join(f"{k}={v}" for k, v in out)
+
+
+def gen_uuid() -> str:
+    """生成 B 站风格的 _uuid（前端本来就是本地生成的，服务端不下发）。"""
+    suffix = "".join(random.choices("0123456789ABCDEF", k=7))
+    return f"{str(uuid.uuid4()).upper()}{suffix}infoc"
+
+
+def sessdata_expiry(cookie: str) -> int | None:
+    """SESSDATA 形如 <值>%2C<过期时间戳>%2C<校验>，取中间的时间戳。"""
+    value = urllib.parse.unquote(cookie_value(cookie, "SESSDATA"))
+    parts = value.split(",")
+    if len(parts) >= 2 and parts[1].isdigit():
+        return int(parts[1])
+    return None

reaction_board/bilibili/client.py

@@ -0,0 +1,362 @@
+# -*- coding: utf-8 -*-
+"""只读 B 站 Web API 客户端（P0 范围）。
+
+P0 只需要收藏夹读取能力：``ids`` / ``infos`` / ``folder/created/list-all``。
+所有网络访问都经过 ``Transport`` 边界，测试可直接替换该边界，而不需要访问
+真实 B 站。写操作（``deal`` / ``sort``）不在本模块实现。
+"""
+
+from __future__ import annotations
+
+import json
+import random
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from typing import Any, Mapping, Protocol, Sequence
+
+from .auth import make_headers
+from .models import FavId, FavResource, FolderInfo, InfosResult
+
+BASE = "https://api.bilibili.com"
+DEFAULT_CHUNK = 200
+MAX_CHUNK = 400
+
+
+class BilibiliError(RuntimeError):
+    """B 站访问层基础异常。"""
+
+
+class TransportError(BilibiliError):
+    """网络层错误，可重试。"""
+
+    retryable = True
+
+
+class BilibiliHTTPError(BilibiliError):
+    def __init__(self, status: int, message: str = "") -> None:
+        self.status = status
+        self.message = message
+        self.retryable = status >= 500 or status == 429
+        super().__init__(f"HTTP {status}: {message}".rstrip())
+
+    def __str__(self) -> str:
+        return f"HTTP {self.status}: {self.message}".rstrip(": ")
+
+
+class BilibiliAPIError(BilibiliError):
+    def __init__(self, code: int, message: str = "") -> None:
+        self.code = code
+        self.message = message
+        super().__init__(f"B 站接口业务错误 code={code}: {message}".rstrip(": "))
+
+    def __str__(self) -> str:
+        return f"B 站接口业务错误 code={self.code}: {self.message}".rstrip(": ")
+
+
+class BilibiliProtocolError(BilibiliError):
+    """响应结构与已知契约不一致。"""
+
+
+@dataclass(frozen=True)
+class HTTPResponse:
+    status: int
+    headers: Mapping[str, str]
+    body: bytes
+
+    def json(self) -> Any:
+        try:
+            return json.loads(self.body.decode("utf-8", "replace"))
+        except json.JSONDecodeError as exc:
+            raise BilibiliProtocolError(f"响应不是合法 JSON: {exc}") from exc
+
+
+class Transport(Protocol):
+    def request(
+        self,
+        method: str,
+        url: str,
+        headers: Mapping[str, str],
+        data: bytes | None = None,
+        timeout: float = 30,
+    ) -> HTTPResponse: ...
+
+
+class UrllibTransport:
+    """基于 ``urllib.request`` 的默认传输实现。"""
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        headers: Mapping[str, str],
+        data: bytes | None = None,
+        timeout: float = 30,
+    ) -> HTTPResponse:
+        req = urllib.request.Request(
+            url, data=data, headers=dict(headers), method=method
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                return HTTPResponse(
+                    status=int(resp.status),
+                    headers=dict(resp.headers.items()),
+                    body=resp.read(),
+                )
+        except urllib.error.HTTPError as exc:
+            try:
+                body = exc.read()
+            except Exception:  # noqa: BLE001
+                body = b""
+            return HTTPResponse(
+                status=int(exc.code),
+                headers=dict(exc.headers.items()) if exc.headers else {},
+                body=body,
+            )
+        except (urllib.error.URLError, TimeoutError, OSError) as exc:
+            raise TransportError(str(exc)) from exc
+
+
+class Pacer:
+    """请求间随机等待；第一个请求不等待。"""
+
+    def __init__(
+        self,
+        min_delay: float = 1.5,
+        max_delay: float = 2.5,
+        *,
+        sleep=time.sleep,
+        rng: random.Random | None = None,
+    ) -> None:
+        if min_delay < 0 or max_delay < min_delay:
+            raise ValueError("非法的请求间隔")
+        self.min_delay = min_delay
+        self.max_delay = max_delay
+        self._sleep = sleep
+        self._rng = rng or random.Random()
+        self._first = True
+
+    def wait(self) -> float:
+        if self._first:
+            self._first = False
+            return 0.0
+        delay = self._rng.uniform(self.min_delay, self.max_delay)
+        if delay > 0:
+            self._sleep(delay)
+        return delay
+
+
+def _chunks(seq: Sequence[int], size: int):
+    for index in range(0, len(seq), size):
+        yield seq[index:index + size]
+
+
+class BilibiliClient:
+    """只读收藏夹 API 客户端。"""
+
+    def __init__(
+        self,
+        *,
+        cookie: str = "",
+        csrf: str = "",
+        mid: int = 0,
+        transport: Transport | None = None,
+        pacer: Pacer | None = None,
+        sleeper=time.sleep,
+        retries: int = 3,
+        backoff: float = 1.0,
+        timeout: float = 30,
+        logger=None,
+    ) -> None:
+        if retries < 1:
+            raise ValueError("retries 必须 >= 1")
+        self.cookie = cookie or ""
+        self.csrf = csrf or ""
+        self.mid = mid
+        self.transport = transport or UrllibTransport()
+        self.pacer = pacer or Pacer(sleep=sleeper)
+        self._sleeper = sleeper
+        self.retries = retries
+        self.backoff = backoff
+        self.timeout = timeout
+        self.logger = logger
+
+    # ------------------------------------------------------------ core
+
+    def _headers(self, auth: bool) -> dict[str, str]:
+        return make_headers({"cookie": self.cookie}, auth=auth)
+
+    def _request_json(self, url: str, *, auth: bool) -> dict[str, Any]:
+        self.pacer.wait()
+        resp = self.transport.request(
+            "GET", url, self._headers(auth), data=None, timeout=self.timeout
+        )
+        if resp.status >= 400:
+            raise BilibiliHTTPError(resp.status, resp.body.decode("utf-8", "replace")[:200])
+        payload = resp.json()
+        if not isinstance(payload, dict):
+            raise BilibiliProtocolError("接口响应顶层不是 JSON 对象")
+        return payload
+
+    def _request_with_retry(self, url: str, *, auth: bool) -> dict[str, Any]:
+        last: BilibiliError | None = None
+        for attempt in range(self.retries):
+            try:
+                return self._request_json(url, auth=auth)
+            except (TransportError, BilibiliHTTPError) as exc:
+                last = exc
+                if not exc.retryable or attempt >= self.retries - 1:
+                    raise
+                delay = self.backoff * (2 ** attempt)
+                if self.logger is not None:
+                    self.logger.warning(
+                        "bili.retry",
+                        attempt=attempt + 1,
+                        delay=delay,
+                        status=getattr(exc, "status", None),
+                    )
+                if delay > 0:
+                    self._sleeper(delay)
+        raise last or TransportError("未知网络错误")
+
+    @staticmethod
+    def _api_error(payload: Mapping[str, Any]) -> BilibiliAPIError:
+        code = payload.get("code")
+        if isinstance(code, bool) or not isinstance(code, int):
+            raise BilibiliProtocolError("接口响应缺少整数 code")
+        return BilibiliAPIError(code, str(payload.get("message") or ""))
+
+    def get_json(self, url: str, *, auth: bool = False) -> dict[str, Any]:
+        """GET JSON；匿名失败时自动带完整 Cookie 重试一次（与 favsort 一致）。"""
+        if auth:
+            payload = self._request_with_retry(url, auth=True)
+            if payload.get("code") != 0:
+                raise self._api_error(payload)
+            return payload
+
+        anon_error: BilibiliError | None = None
+        try:
+            payload = self._request_with_retry(url, auth=False)
+            if payload.get("code") == 0:
+                return payload
+            anon_error = self._api_error(payload)
+        except BilibiliError as exc:
+            anon_error = exc
+
+        if self.logger is not None and anon_error is not None:
+            self.logger.debug("bili.anon_fallback", error=str(anon_error))
+        payload = self._request_with_retry(url, auth=True)
+        if payload.get("code") != 0:
+            raise self._api_error(payload)
+        return payload
+
+    # ------------------------------------------------------------ fav
+
+    def fav_ids(self, media_id: int) -> list[FavId]:
+        url = (
+            f"{BASE}/x/v3/fav/resource/ids?media_id={int(media_id)}&platform=web"
+        )
+        payload = self.get_json(url)
+        data = payload.get("data")
+        if data is None:
+            data = []
+        if not isinstance(data, list):
+            raise BilibiliProtocolError("fav/resource/ids data 不是数组")
+        out: list[FavId] = []
+        for index, item in enumerate(data):
+            if not isinstance(item, Mapping):
+                raise BilibiliProtocolError(f"fav/resource/ids data[{index}] 不是对象")
+            aid = item.get("id")
+            if isinstance(aid, bool) or not isinstance(aid, int):
+                raise BilibiliProtocolError(f"fav/resource/ids data[{index}].id 不是整数")
+            raw_type = item.get("type", 2)
+            if isinstance(raw_type, bool) or not isinstance(raw_type, int):
+                raise BilibiliProtocolError(f"fav/resource/ids data[{index}].type 不是整数")
+            bvid = item.get("bvid") or item.get("bv_id")
+            if not isinstance(bvid, str) or not bvid:
+                bvid = None
+            out.append(FavId(aid=aid, type=raw_type, bvid=bvid))
+        return out
+
+    def fav_infos(
+        self,
+        media_id: int,
+        aids: Sequence[int],
+        *,
+        folder_mid: int | None = None,
+        chunk_size: int = DEFAULT_CHUNK,
+    ) -> InfosResult:
+        if not 1 <= chunk_size <= MAX_CHUNK:
+            raise ValueError(f"chunk_size 必须在 1..{MAX_CHUNK} 之间")
+        requested: list[int] = []
+        seen: set[int] = set()
+        for raw in aids:
+            aid = int(raw)
+            if aid not in seen:
+                seen.add(aid)
+                requested.append(aid)
+        if not requested:
+            return InfosResult(requested=(), items={}, missing=())
+
+        fmid = self.mid if folder_mid is None else int(folder_mid)
+        items: dict[int, FavResource] = {}
+        for chunk in _chunks(requested, chunk_size):
+            resources = ",".join(f"{aid}:2" for aid in chunk)
+            url = (
+                f"{BASE}/x/v3/fav/resource/infos"
+                f"?resources={urllib.parse.quote(resources)}"
+                f"&platform=web&folder_mid={fmid}&folder_id={int(media_id)}"
+            )
+            payload = self.get_json(url)
+            data = payload.get("data")
+            if data is None:
+                data = []
+            if not isinstance(data, list):
+                raise BilibiliProtocolError("fav/resource/infos data 不是数组")
+            chunk_set = set(chunk)
+            for item in data:
+                if not isinstance(item, Mapping):
+                    raise BilibiliProtocolError("fav/resource/infos item 不是对象")
+                try:
+                    parsed = FavResource.from_api(item)
+                except (TypeError, ValueError) as exc:
+                    raise BilibiliProtocolError(f"fav/resource/infos 解析失败: {exc}") from exc
+                if parsed.aid not in chunk_set:
+                    raise BilibiliProtocolError(
+                        f"fav/resource/infos 返回了未请求的 aid={parsed.aid}"
+                    )
+                items[parsed.aid] = parsed
+
+        missing = tuple(aid for aid in requested if aid not in items)
+        return InfosResult(requested=tuple(requested), items=items, missing=missing)
+
+    def fav_folders(self, up_mid: int | None = None) -> list[FolderInfo]:
+        mid = self.mid if up_mid is None else int(up_mid)
+        url = f"{BASE}/x/v3/fav/folder/created/list-all?up_mid={mid}&jsonp=jsonp"
+        payload = self.get_json(url, auth=True)
+        data = payload.get("data") or {}
+        if not isinstance(data, Mapping):
+            raise BilibiliProtocolError("fav/folder/created/list-all data 不是对象")
+        rows = data.get("list")
+        if rows is None:
+            return []
+        if not isinstance(rows, list):
+            raise BilibiliProtocolError("fav/folder/created/list-all data.list 不是数组")
+        out: list[FolderInfo] = []
+        for index, row in enumerate(rows):
+            if not isinstance(row, Mapping):
+                raise BilibiliProtocolError(f"folder list[{index}] 不是对象")
+            media_id = row.get("id")
+            if isinstance(media_id, bool) or not isinstance(media_id, int):
+                raise BilibiliProtocolError(f"folder list[{index}].id 不是整数")
+            count = row.get("media_count", 0)
+            if isinstance(count, bool) or not isinstance(count, int):
+                count = 0
+            out.append(FolderInfo(
+                media_id=media_id,
+                title=str(row.get("title") or ""),
+                media_count=count,
+            ))
+        return out

reaction_board/bilibili/models.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""B 站收藏夹接口的返回模型。"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Mapping
+
+
+@dataclass(frozen=True)
+class FavId:
+    """``fav/resource/ids`` 的一条资源。"""
+
+    aid: int
+    type: int
+    bvid: str | None = None
+
+
+@dataclass(frozen=True)
+class FavResource:
+    """``fav/resource/infos`` 的一条视频元数据。"""
+
+    aid: int
+    bvid: str | None
+    title: str
+    up_mid: int | None
+    up_name: str | None
+    pubdate: int
+    duration: int | None
+    cover: str | None
+    attr: int
+    fav_time: int | None
+
+    @property
+    def invalid(self) -> bool:
+        # 实测 attr=9 表示"已失效视频"。
+        return self.attr == 9
+
+    @classmethod
+    def from_api(cls, item: Mapping[str, Any]) -> "FavResource":
+        aid = item.get("id")
+        if isinstance(aid, bool) or not isinstance(aid, int):
+            raise ValueError("infos item.id 不是整数")
+        upper = item.get("upper") or {}
+        if not isinstance(upper, Mapping):
+            upper = {}
+        up_mid = upper.get("mid")
+        if isinstance(up_mid, bool) or not isinstance(up_mid, int):
+            up_mid = None
+        raw_duration = item.get("duration")
+        duration = raw_duration if isinstance(raw_duration, int) and not isinstance(
+            raw_duration, bool
+        ) else None
+        raw_attr = item.get("attr", 0)
+        attr = raw_attr if isinstance(raw_attr, int) and not isinstance(raw_attr, bool) else 0
+        fav_time = item.get("fav_time")
+        if isinstance(fav_time, bool) or not isinstance(fav_time, int) or fav_time <= 0:
+            fav_time = None
+        bvid = item.get("bvid") or item.get("bv_id")
+        if not isinstance(bvid, str) or not bvid:
+            bvid = None
+        return cls(
+            aid=aid,
+            bvid=bvid,
+            title=str(item.get("title") or ""),
+            up_mid=up_mid,
+            up_name=str(upper.get("name") or "") or None,
+            pubdate=int(item.get("pubtime") or item.get("ctime") or 0),
+            duration=duration,
+            cover=str(item.get("cover") or "") or None,
+            attr=attr,
+            fav_time=fav_time,
+        )
+
+
+@dataclass(frozen=True)
+class FolderInfo:
+    media_id: int
+    title: str
+    media_count: int
+
+
+@dataclass(frozen=True)
+class InfosResult:
+    requested: tuple[int, ...]
+    items: dict[int, FavResource]
+    missing: tuple[int, ...]