feat(p0): add favorite folder sync and sqlite data foundation
Implement P0 of the reaction dashboard: config loading with hot reload, SQLite migrations with backups, single-writer queue, read-only Bilibili fav client, folder import/diff sync, invalid-resource handling, blacklist seed, scheduler, and JSON redacted logging. Includes offline unit and transport-level contract tests; all existing and new tests pass.
This commit is contained in:
kairee committed
commit
e2cdf997af
39 files changed
+8575
No files matched your search
@@ -0,0 +1,362 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""只读 B 站 Web API 客户端（P0 范围）。
|
||||
|
||||
P0 只需要收藏夹读取能力：``ids`` / ``infos`` / ``folder/created/list-all``。
|
||||
所有网络访问都经过 ``Transport`` 边界，测试可直接替换该边界，而不需要访问
|
||||
真实 B 站。写操作（``deal`` / ``sort``）不在本模块实现。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import random
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Mapping, Protocol, Sequence
|
||||
|
||||
from .auth import make_headers
|
||||
from .models import FavId, FavResource, FolderInfo, InfosResult
|
||||
|
||||
BASE = "https://api.bilibili.com"
|
||||
DEFAULT_CHUNK = 200
|
||||
MAX_CHUNK = 400
|
||||
|
||||
|
||||
class BilibiliError(RuntimeError):
|
||||
"""B 站访问层基础异常。"""
|
||||
|
||||
|
||||
class TransportError(BilibiliError):
|
||||
"""网络层错误，可重试。"""
|
||||
|
||||
retryable = True
|
||||
|
||||
|
||||
class BilibiliHTTPError(BilibiliError):
|
||||
def __init__(self, status: int, message: str = "") -> None:
|
||||
self.status = status
|
||||
self.message = message
|
||||
self.retryable = status >= 500 or status == 429
|
||||
super().__init__(f"HTTP {status}: {message}".rstrip())
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"HTTP {self.status}: {self.message}".rstrip(": ")
|
||||
|
||||
|
||||
class BilibiliAPIError(BilibiliError):
|
||||
def __init__(self, code: int, message: str = "") -> None:
|
||||
self.code = code
|
||||
self.message = message
|
||||
super().__init__(f"B 站接口业务错误 code={code}: {message}".rstrip(": "))
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"B 站接口业务错误 code={self.code}: {self.message}".rstrip(": ")
|
||||
|
||||
|
||||
class BilibiliProtocolError(BilibiliError):
|
||||
"""响应结构与已知契约不一致。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HTTPResponse:
|
||||
status: int
|
||||
headers: Mapping[str, str]
|
||||
body: bytes
|
||||
|
||||
def json(self) -> Any:
|
||||
try:
|
||||
return json.loads(self.body.decode("utf-8", "replace"))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise BilibiliProtocolError(f"响应不是合法 JSON: {exc}") from exc
|
||||
|
||||
|
||||
class Transport(Protocol):
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
headers: Mapping[str, str],
|
||||
data: bytes | None = None,
|
||||
timeout: float = 30,
|
||||
) -> HTTPResponse: ...
|
||||
|
||||
|
||||
class UrllibTransport:
|
||||
"""基于 ``urllib.request`` 的默认传输实现。"""
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
headers: Mapping[str, str],
|
||||
data: bytes | None = None,
|
||||
timeout: float = 30,
|
||||
) -> HTTPResponse:
|
||||
req = urllib.request.Request(
|
||||
url, data=data, headers=dict(headers), method=method
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return HTTPResponse(
|
||||
status=int(resp.status),
|
||||
headers=dict(resp.headers.items()),
|
||||
body=resp.read(),
|
||||
)
|
||||
except urllib.error.HTTPError as exc:
|
||||
try:
|
||||
body = exc.read()
|
||||
except Exception: # noqa: BLE001
|
||||
body = b""
|
||||
return HTTPResponse(
|
||||
status=int(exc.code),
|
||||
headers=dict(exc.headers.items()) if exc.headers else {},
|
||||
body=body,
|
||||
)
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise TransportError(str(exc)) from exc
|
||||
|
||||
|
||||
class Pacer:
|
||||
"""请求间随机等待；第一个请求不等待。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
min_delay: float = 1.5,
|
||||
max_delay: float = 2.5,
|
||||
*,
|
||||
sleep=time.sleep,
|
||||
rng: random.Random | None = None,
|
||||
) -> None:
|
||||
if min_delay < 0 or max_delay < min_delay:
|
||||
raise ValueError("非法的请求间隔")
|
||||
self.min_delay = min_delay
|
||||
self.max_delay = max_delay
|
||||
self._sleep = sleep
|
||||
self._rng = rng or random.Random()
|
||||
self._first = True
|
||||
|
||||
def wait(self) -> float:
|
||||
if self._first:
|
||||
self._first = False
|
||||
return 0.0
|
||||
delay = self._rng.uniform(self.min_delay, self.max_delay)
|
||||
if delay > 0:
|
||||
self._sleep(delay)
|
||||
return delay
|
||||
|
||||
|
||||
def _chunks(seq: Sequence[int], size: int):
|
||||
for index in range(0, len(seq), size):
|
||||
yield seq[index:index + size]
|
||||
|
||||
|
||||
class BilibiliClient:
|
||||
"""只读收藏夹 API 客户端。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
cookie: str = "",
|
||||
csrf: str = "",
|
||||
mid: int = 0,
|
||||
transport: Transport | None = None,
|
||||
pacer: Pacer | None = None,
|
||||
sleeper=time.sleep,
|
||||
retries: int = 3,
|
||||
backoff: float = 1.0,
|
||||
timeout: float = 30,
|
||||
logger=None,
|
||||
) -> None:
|
||||
if retries < 1:
|
||||
raise ValueError("retries 必须 >= 1")
|
||||
self.cookie = cookie or ""
|
||||
self.csrf = csrf or ""
|
||||
self.mid = mid
|
||||
self.transport = transport or UrllibTransport()
|
||||
self.pacer = pacer or Pacer(sleep=sleeper)
|
||||
self._sleeper = sleeper
|
||||
self.retries = retries
|
||||
self.backoff = backoff
|
||||
self.timeout = timeout
|
||||
self.logger = logger
|
||||
|
||||
# ------------------------------------------------------------ core
|
||||
|
||||
def _headers(self, auth: bool) -> dict[str, str]:
|
||||
return make_headers({"cookie": self.cookie}, auth=auth)
|
||||
|
||||
def _request_json(self, url: str, *, auth: bool) -> dict[str, Any]:
|
||||
self.pacer.wait()
|
||||
resp = self.transport.request(
|
||||
"GET", url, self._headers(auth), data=None, timeout=self.timeout
|
||||
)
|
||||
if resp.status >= 400:
|
||||
raise BilibiliHTTPError(resp.status, resp.body.decode("utf-8", "replace")[:200])
|
||||
payload = resp.json()
|
||||
if not isinstance(payload, dict):
|
||||
raise BilibiliProtocolError("接口响应顶层不是 JSON 对象")
|
||||
return payload
|
||||
|
||||
def _request_with_retry(self, url: str, *, auth: bool) -> dict[str, Any]:
|
||||
last: BilibiliError | None = None
|
||||
for attempt in range(self.retries):
|
||||
try:
|
||||
return self._request_json(url, auth=auth)
|
||||
except (TransportError, BilibiliHTTPError) as exc:
|
||||
last = exc
|
||||
if not exc.retryable or attempt >= self.retries - 1:
|
||||
raise
|
||||
delay = self.backoff * (2 ** attempt)
|
||||
if self.logger is not None:
|
||||
self.logger.warning(
|
||||
"bili.retry",
|
||||
attempt=attempt + 1,
|
||||
delay=delay,
|
||||
status=getattr(exc, "status", None),
|
||||
)
|
||||
if delay > 0:
|
||||
self._sleeper(delay)
|
||||
raise last or TransportError("未知网络错误")
|
||||
|
||||
@staticmethod
|
||||
def _api_error(payload: Mapping[str, Any]) -> BilibiliAPIError:
|
||||
code = payload.get("code")
|
||||
if isinstance(code, bool) or not isinstance(code, int):
|
||||
raise BilibiliProtocolError("接口响应缺少整数 code")
|
||||
return BilibiliAPIError(code, str(payload.get("message") or ""))
|
||||
|
||||
def get_json(self, url: str, *, auth: bool = False) -> dict[str, Any]:
|
||||
"""GET JSON；匿名失败时自动带完整 Cookie 重试一次（与 favsort 一致）。"""
|
||||
if auth:
|
||||
payload = self._request_with_retry(url, auth=True)
|
||||
if payload.get("code") != 0:
|
||||
raise self._api_error(payload)
|
||||
return payload
|
||||
|
||||
anon_error: BilibiliError | None = None
|
||||
try:
|
||||
payload = self._request_with_retry(url, auth=False)
|
||||
if payload.get("code") == 0:
|
||||
return payload
|
||||
anon_error = self._api_error(payload)
|
||||
except BilibiliError as exc:
|
||||
anon_error = exc
|
||||
|
||||
if self.logger is not None and anon_error is not None:
|
||||
self.logger.debug("bili.anon_fallback", error=str(anon_error))
|
||||
payload = self._request_with_retry(url, auth=True)
|
||||
if payload.get("code") != 0:
|
||||
raise self._api_error(payload)
|
||||
return payload
|
||||
|
||||
# ------------------------------------------------------------ fav
|
||||
|
||||
def fav_ids(self, media_id: int) -> list[FavId]:
|
||||
url = (
|
||||
f"{BASE}/x/v3/fav/resource/ids?media_id={int(media_id)}&platform=web"
|
||||
)
|
||||
payload = self.get_json(url)
|
||||
data = payload.get("data")
|
||||
if data is None:
|
||||
data = []
|
||||
if not isinstance(data, list):
|
||||
raise BilibiliProtocolError("fav/resource/ids data 不是数组")
|
||||
out: list[FavId] = []
|
||||
for index, item in enumerate(data):
|
||||
if not isinstance(item, Mapping):
|
||||
raise BilibiliProtocolError(f"fav/resource/ids data[{index}] 不是对象")
|
||||
aid = item.get("id")
|
||||
if isinstance(aid, bool) or not isinstance(aid, int):
|
||||
raise BilibiliProtocolError(f"fav/resource/ids data[{index}].id 不是整数")
|
||||
raw_type = item.get("type", 2)
|
||||
if isinstance(raw_type, bool) or not isinstance(raw_type, int):
|
||||
raise BilibiliProtocolError(f"fav/resource/ids data[{index}].type 不是整数")
|
||||
bvid = item.get("bvid") or item.get("bv_id")
|
||||
if not isinstance(bvid, str) or not bvid:
|
||||
bvid = None
|
||||
out.append(FavId(aid=aid, type=raw_type, bvid=bvid))
|
||||
return out
|
||||
|
||||
def fav_infos(
|
||||
self,
|
||||
media_id: int,
|
||||
aids: Sequence[int],
|
||||
*,
|
||||
folder_mid: int | None = None,
|
||||
chunk_size: int = DEFAULT_CHUNK,
|
||||
) -> InfosResult:
|
||||
if not 1 <= chunk_size <= MAX_CHUNK:
|
||||
raise ValueError(f"chunk_size 必须在 1..{MAX_CHUNK} 之间")
|
||||
requested: list[int] = []
|
||||
seen: set[int] = set()
|
||||
for raw in aids:
|
||||
aid = int(raw)
|
||||
if aid not in seen:
|
||||
seen.add(aid)
|
||||
requested.append(aid)
|
||||
if not requested:
|
||||
return InfosResult(requested=(), items={}, missing=())
|
||||
|
||||
fmid = self.mid if folder_mid is None else int(folder_mid)
|
||||
items: dict[int, FavResource] = {}
|
||||
for chunk in _chunks(requested, chunk_size):
|
||||
resources = ",".join(f"{aid}:2" for aid in chunk)
|
||||
url = (
|
||||
f"{BASE}/x/v3/fav/resource/infos"
|
||||
f"?resources={urllib.parse.quote(resources)}"
|
||||
f"&platform=web&folder_mid={fmid}&folder_id={int(media_id)}"
|
||||
)
|
||||
payload = self.get_json(url)
|
||||
data = payload.get("data")
|
||||
if data is None:
|
||||
data = []
|
||||
if not isinstance(data, list):
|
||||
raise BilibiliProtocolError("fav/resource/infos data 不是数组")
|
||||
chunk_set = set(chunk)
|
||||
for item in data:
|
||||
if not isinstance(item, Mapping):
|
||||
raise BilibiliProtocolError("fav/resource/infos item 不是对象")
|
||||
try:
|
||||
parsed = FavResource.from_api(item)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise BilibiliProtocolError(f"fav/resource/infos 解析失败: {exc}") from exc
|
||||
if parsed.aid not in chunk_set:
|
||||
raise BilibiliProtocolError(
|
||||
f"fav/resource/infos 返回了未请求的 aid={parsed.aid}"
|
||||
)
|
||||
items[parsed.aid] = parsed
|
||||
|
||||
missing = tuple(aid for aid in requested if aid not in items)
|
||||
return InfosResult(requested=tuple(requested), items=items, missing=missing)
|
||||
|
||||
def fav_folders(self, up_mid: int | None = None) -> list[FolderInfo]:
|
||||
mid = self.mid if up_mid is None else int(up_mid)
|
||||
url = f"{BASE}/x/v3/fav/folder/created/list-all?up_mid={mid}&jsonp=jsonp"
|
||||
payload = self.get_json(url, auth=True)
|
||||
data = payload.get("data") or {}
|
||||
if not isinstance(data, Mapping):
|
||||
raise BilibiliProtocolError("fav/folder/created/list-all data 不是对象")
|
||||
rows = data.get("list")
|
||||
if rows is None:
|
||||
return []
|
||||
if not isinstance(rows, list):
|
||||
raise BilibiliProtocolError("fav/folder/created/list-all data.list 不是数组")
|
||||
out: list[FolderInfo] = []
|
||||
for index, row in enumerate(rows):
|
||||
if not isinstance(row, Mapping):
|
||||
raise BilibiliProtocolError(f"folder list[{index}] 不是对象")
|
||||
media_id = row.get("id")
|
||||
if isinstance(media_id, bool) or not isinstance(media_id, int):
|
||||
raise BilibiliProtocolError(f"folder list[{index}].id 不是整数")
|
||||
count = row.get("media_count", 0)
|
||||
if isinstance(count, bool) or not isinstance(count, int):
|
||||
count = 0
|
||||
out.append(FolderInfo(
|
||||
media_id=media_id,
|
||||
title=str(row.get("title") or ""),
|
||||
media_count=count,
|
||||
))
|
||||
return out
|
||||
Reference in new issue
Block a user