feat(p0): add favorite folder sync and sqlite data foundation
Implement P0 of the reaction dashboard: config loading with hot reload, SQLite migrations with backups, single-writer queue, read-only Bilibili fav client, folder import/diff sync, invalid-resource handling, blacklist seed, scheduler, and JSON redacted logging. Includes offline unit and transport-level contract tests; all existing and new tests pass.
This commit is contained in:
commit
e2cdf997af
39 files changed
+8575
No files matched your search
@@ -0,0 +1,2 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Reaction video dashboard backend package."""
|
||||
@@ -0,0 +1,2 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""B 站接口访问层。"""
|
||||
@@ -0,0 +1,143 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""B 站 Cookie 与请求头工具。
|
||||
|
||||
从 ``favsort.py`` 抽取,供现有排序工具和看板共用,避免出现两套不一致的
|
||||
身份 Cookie / 设备指纹处理逻辑。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import urllib.parse
|
||||
import uuid
|
||||
|
||||
IDENTITY_KEYS = {
|
||||
"sessdata",
|
||||
"bili_jct",
|
||||
"dedeuserid",
|
||||
"dedeuserid__ckmd5",
|
||||
"bili_ticket",
|
||||
"bili_ticket_expires",
|
||||
"sid",
|
||||
"b_lsid",
|
||||
}
|
||||
|
||||
DROP_STALE = {"sid", "b_lsid"} # 会话级字段,刷新时应丢弃旧值
|
||||
|
||||
|
||||
def strip_identity(cookie: str) -> str:
|
||||
"""剥离账号身份类 cookie,保留 buvid3 / buvid4 / _uuid 等设备指纹。
|
||||
|
||||
实测结论:
|
||||
- ids:完全无 cookie 也能读(连私密收藏夹都返回全量);
|
||||
- infos:不需要账号身份,但缺设备指纹 cookie 会被 HTTP 412 拦;
|
||||
- sort:必须带完整 cookie + csrf,不能走这里。
|
||||
"""
|
||||
kept = []
|
||||
for part in (cookie or "").split(";"):
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
key = part.split("=", 1)[0].strip().lower()
|
||||
if key in IDENTITY_KEYS:
|
||||
continue
|
||||
kept.append(part)
|
||||
return "; ".join(kept)
|
||||
|
||||
|
||||
def make_headers(cfg: dict, auth: bool = False) -> dict[str, str]:
|
||||
"""构造 B 站 Web 请求头;``auth=False`` 时去掉账号身份 cookie。"""
|
||||
return {
|
||||
"accept": "*/*",
|
||||
"accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"cookie": cfg["cookie"] if auth else strip_identity(cfg["cookie"]),
|
||||
"dnt": "1",
|
||||
"origin": "https://www.bilibili.com",
|
||||
"referer": "https://www.bilibili.com/",
|
||||
"sec-ch-ua": '"Google Chrome";v="155", "Chromium";v="155", "Not(A:Brand";v="24"',
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"Windows"',
|
||||
"sec-fetch-dest": "empty",
|
||||
"sec-fetch-mode": "cors",
|
||||
"sec-fetch-site": "same-site",
|
||||
"user-agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def parse_cookie(cookie: str) -> list[tuple[str, str]]:
|
||||
"""把 cookie 字符串解析成 [(key, value)],保留原顺序。"""
|
||||
pairs = []
|
||||
for part in (cookie or "").split(";"):
|
||||
part = part.strip()
|
||||
if not part or "=" not in part:
|
||||
continue
|
||||
k, v = part.split("=", 1)
|
||||
pairs.append((k.strip(), v.strip()))
|
||||
return pairs
|
||||
|
||||
|
||||
def cookie_value(cookie: str, name: str) -> str:
|
||||
name = name.lower()
|
||||
for k, v in parse_cookie(cookie):
|
||||
if k.lower() == name:
|
||||
return v
|
||||
return ""
|
||||
|
||||
|
||||
def cookies_from_set_cookie(headers) -> dict[str, str]:
|
||||
"""从 Set-Cookie 头里取 name=value(忽略 Path/Domain/Expires 等属性)。"""
|
||||
pairs = {}
|
||||
for h in headers or []:
|
||||
first = h.split(";", 1)[0].strip()
|
||||
if "=" not in first:
|
||||
continue
|
||||
k, v = first.split("=", 1)
|
||||
pairs[k.strip()] = v.strip()
|
||||
return pairs
|
||||
|
||||
|
||||
def merge_cookie(old: str, new_pairs: dict[str, str]) -> str:
|
||||
"""只替换身份类 cookie,其它字段(设备指纹等)原样保留。
|
||||
|
||||
新值里没有的身份字段不会被删除(例如服务端没下发新的 sid 时保留旧值),
|
||||
只有 DROP_STALE 里的会话字段会在刷新时清掉。
|
||||
"""
|
||||
new_l = {
|
||||
k.lower(): (k, v)
|
||||
for k, v in new_pairs.items()
|
||||
if k.lower() in IDENTITY_KEYS
|
||||
}
|
||||
out, seen = [], set()
|
||||
for k, v in parse_cookie(old):
|
||||
kl = k.lower()
|
||||
if kl in new_l:
|
||||
nk, nv = new_l[kl]
|
||||
out.append((nk, nv))
|
||||
seen.add(kl)
|
||||
elif kl in DROP_STALE:
|
||||
continue
|
||||
else:
|
||||
out.append((k, v))
|
||||
seen.add(kl)
|
||||
for kl, (k, v) in new_l.items():
|
||||
if kl not in seen:
|
||||
out.append((k, v))
|
||||
return "; ".join(f"{k}={v}" for k, v in out)
|
||||
|
||||
|
||||
def gen_uuid() -> str:
|
||||
"""生成 B 站风格的 _uuid(前端本来就是本地生成的,服务端不下发)。"""
|
||||
suffix = "".join(random.choices("0123456789ABCDEF", k=7))
|
||||
return f"{str(uuid.uuid4()).upper()}{suffix}infoc"
|
||||
|
||||
|
||||
def sessdata_expiry(cookie: str) -> int | None:
|
||||
"""SESSDATA 形如 <值>%2C<过期时间戳>%2C<校验>,取中间的时间戳。"""
|
||||
value = urllib.parse.unquote(cookie_value(cookie, "SESSDATA"))
|
||||
parts = value.split(",")
|
||||
if len(parts) >= 2 and parts[1].isdigit():
|
||||
return int(parts[1])
|
||||
return None
|
||||
@@ -0,0 +1,362 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""只读 B 站 Web API 客户端(P0 范围)。
|
||||
|
||||
P0 只需要收藏夹读取能力:``ids`` / ``infos`` / ``folder/created/list-all``。
|
||||
所有网络访问都经过 ``Transport`` 边界,测试可直接替换该边界,而不需要访问
|
||||
真实 B 站。写操作(``deal`` / ``sort``)不在本模块实现。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import random
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Mapping, Protocol, Sequence
|
||||
|
||||
from .auth import make_headers
|
||||
from .models import FavId, FavResource, FolderInfo, InfosResult
|
||||
|
||||
BASE = "https://api.bilibili.com"
|
||||
DEFAULT_CHUNK = 200
|
||||
MAX_CHUNK = 400
|
||||
|
||||
|
||||
class BilibiliError(RuntimeError):
|
||||
"""B 站访问层基础异常。"""
|
||||
|
||||
|
||||
class TransportError(BilibiliError):
|
||||
"""网络层错误,可重试。"""
|
||||
|
||||
retryable = True
|
||||
|
||||
|
||||
class BilibiliHTTPError(BilibiliError):
|
||||
def __init__(self, status: int, message: str = "") -> None:
|
||||
self.status = status
|
||||
self.message = message
|
||||
self.retryable = status >= 500 or status == 429
|
||||
super().__init__(f"HTTP {status}: {message}".rstrip())
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"HTTP {self.status}: {self.message}".rstrip(": ")
|
||||
|
||||
|
||||
class BilibiliAPIError(BilibiliError):
|
||||
def __init__(self, code: int, message: str = "") -> None:
|
||||
self.code = code
|
||||
self.message = message
|
||||
super().__init__(f"B 站接口业务错误 code={code}: {message}".rstrip(": "))
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"B 站接口业务错误 code={self.code}: {self.message}".rstrip(": ")
|
||||
|
||||
|
||||
class BilibiliProtocolError(BilibiliError):
|
||||
"""响应结构与已知契约不一致。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HTTPResponse:
|
||||
status: int
|
||||
headers: Mapping[str, str]
|
||||
body: bytes
|
||||
|
||||
def json(self) -> Any:
|
||||
try:
|
||||
return json.loads(self.body.decode("utf-8", "replace"))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise BilibiliProtocolError(f"响应不是合法 JSON: {exc}") from exc
|
||||
|
||||
|
||||
class Transport(Protocol):
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
headers: Mapping[str, str],
|
||||
data: bytes | None = None,
|
||||
timeout: float = 30,
|
||||
) -> HTTPResponse: ...
|
||||
|
||||
|
||||
class UrllibTransport:
|
||||
"""基于 ``urllib.request`` 的默认传输实现。"""
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
headers: Mapping[str, str],
|
||||
data: bytes | None = None,
|
||||
timeout: float = 30,
|
||||
) -> HTTPResponse:
|
||||
req = urllib.request.Request(
|
||||
url, data=data, headers=dict(headers), method=method
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return HTTPResponse(
|
||||
status=int(resp.status),
|
||||
headers=dict(resp.headers.items()),
|
||||
body=resp.read(),
|
||||
)
|
||||
except urllib.error.HTTPError as exc:
|
||||
try:
|
||||
body = exc.read()
|
||||
except Exception: # noqa: BLE001
|
||||
body = b""
|
||||
return HTTPResponse(
|
||||
status=int(exc.code),
|
||||
headers=dict(exc.headers.items()) if exc.headers else {},
|
||||
body=body,
|
||||
)
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise TransportError(str(exc)) from exc
|
||||
|
||||
|
||||
class Pacer:
|
||||
"""请求间随机等待;第一个请求不等待。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
min_delay: float = 1.5,
|
||||
max_delay: float = 2.5,
|
||||
*,
|
||||
sleep=time.sleep,
|
||||
rng: random.Random | None = None,
|
||||
) -> None:
|
||||
if min_delay < 0 or max_delay < min_delay:
|
||||
raise ValueError("非法的请求间隔")
|
||||
self.min_delay = min_delay
|
||||
self.max_delay = max_delay
|
||||
self._sleep = sleep
|
||||
self._rng = rng or random.Random()
|
||||
self._first = True
|
||||
|
||||
def wait(self) -> float:
|
||||
if self._first:
|
||||
self._first = False
|
||||
return 0.0
|
||||
delay = self._rng.uniform(self.min_delay, self.max_delay)
|
||||
if delay > 0:
|
||||
self._sleep(delay)
|
||||
return delay
|
||||
|
||||
|
||||
def _chunks(seq: Sequence[int], size: int):
|
||||
for index in range(0, len(seq), size):
|
||||
yield seq[index:index + size]
|
||||
|
||||
|
||||
class BilibiliClient:
|
||||
"""只读收藏夹 API 客户端。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
cookie: str = "",
|
||||
csrf: str = "",
|
||||
mid: int = 0,
|
||||
transport: Transport | None = None,
|
||||
pacer: Pacer | None = None,
|
||||
sleeper=time.sleep,
|
||||
retries: int = 3,
|
||||
backoff: float = 1.0,
|
||||
timeout: float = 30,
|
||||
logger=None,
|
||||
) -> None:
|
||||
if retries < 1:
|
||||
raise ValueError("retries 必须 >= 1")
|
||||
self.cookie = cookie or ""
|
||||
self.csrf = csrf or ""
|
||||
self.mid = mid
|
||||
self.transport = transport or UrllibTransport()
|
||||
self.pacer = pacer or Pacer(sleep=sleeper)
|
||||
self._sleeper = sleeper
|
||||
self.retries = retries
|
||||
self.backoff = backoff
|
||||
self.timeout = timeout
|
||||
self.logger = logger
|
||||
|
||||
# ------------------------------------------------------------ core
|
||||
|
||||
def _headers(self, auth: bool) -> dict[str, str]:
|
||||
return make_headers({"cookie": self.cookie}, auth=auth)
|
||||
|
||||
def _request_json(self, url: str, *, auth: bool) -> dict[str, Any]:
|
||||
self.pacer.wait()
|
||||
resp = self.transport.request(
|
||||
"GET", url, self._headers(auth), data=None, timeout=self.timeout
|
||||
)
|
||||
if resp.status >= 400:
|
||||
raise BilibiliHTTPError(resp.status, resp.body.decode("utf-8", "replace")[:200])
|
||||
payload = resp.json()
|
||||
if not isinstance(payload, dict):
|
||||
raise BilibiliProtocolError("接口响应顶层不是 JSON 对象")
|
||||
return payload
|
||||
|
||||
def _request_with_retry(self, url: str, *, auth: bool) -> dict[str, Any]:
|
||||
last: BilibiliError | None = None
|
||||
for attempt in range(self.retries):
|
||||
try:
|
||||
return self._request_json(url, auth=auth)
|
||||
except (TransportError, BilibiliHTTPError) as exc:
|
||||
last = exc
|
||||
if not exc.retryable or attempt >= self.retries - 1:
|
||||
raise
|
||||
delay = self.backoff * (2 ** attempt)
|
||||
if self.logger is not None:
|
||||
self.logger.warning(
|
||||
"bili.retry",
|
||||
attempt=attempt + 1,
|
||||
delay=delay,
|
||||
status=getattr(exc, "status", None),
|
||||
)
|
||||
if delay > 0:
|
||||
self._sleeper(delay)
|
||||
raise last or TransportError("未知网络错误")
|
||||
|
||||
@staticmethod
|
||||
def _api_error(payload: Mapping[str, Any]) -> BilibiliAPIError:
|
||||
code = payload.get("code")
|
||||
if isinstance(code, bool) or not isinstance(code, int):
|
||||
raise BilibiliProtocolError("接口响应缺少整数 code")
|
||||
return BilibiliAPIError(code, str(payload.get("message") or ""))
|
||||
|
||||
def get_json(self, url: str, *, auth: bool = False) -> dict[str, Any]:
|
||||
"""GET JSON;匿名失败时自动带完整 Cookie 重试一次(与 favsort 一致)。"""
|
||||
if auth:
|
||||
payload = self._request_with_retry(url, auth=True)
|
||||
if payload.get("code") != 0:
|
||||
raise self._api_error(payload)
|
||||
return payload
|
||||
|
||||
anon_error: BilibiliError | None = None
|
||||
try:
|
||||
payload = self._request_with_retry(url, auth=False)
|
||||
if payload.get("code") == 0:
|
||||
return payload
|
||||
anon_error = self._api_error(payload)
|
||||
except BilibiliError as exc:
|
||||
anon_error = exc
|
||||
|
||||
if self.logger is not None and anon_error is not None:
|
||||
self.logger.debug("bili.anon_fallback", error=str(anon_error))
|
||||
payload = self._request_with_retry(url, auth=True)
|
||||
if payload.get("code") != 0:
|
||||
raise self._api_error(payload)
|
||||
return payload
|
||||
|
||||
# ------------------------------------------------------------ fav
|
||||
|
||||
def fav_ids(self, media_id: int) -> list[FavId]:
|
||||
url = (
|
||||
f"{BASE}/x/v3/fav/resource/ids?media_id={int(media_id)}&platform=web"
|
||||
)
|
||||
payload = self.get_json(url)
|
||||
data = payload.get("data")
|
||||
if data is None:
|
||||
data = []
|
||||
if not isinstance(data, list):
|
||||
raise BilibiliProtocolError("fav/resource/ids data 不是数组")
|
||||
out: list[FavId] = []
|
||||
for index, item in enumerate(data):
|
||||
if not isinstance(item, Mapping):
|
||||
raise BilibiliProtocolError(f"fav/resource/ids data[{index}] 不是对象")
|
||||
aid = item.get("id")
|
||||
if isinstance(aid, bool) or not isinstance(aid, int):
|
||||
raise BilibiliProtocolError(f"fav/resource/ids data[{index}].id 不是整数")
|
||||
raw_type = item.get("type", 2)
|
||||
if isinstance(raw_type, bool) or not isinstance(raw_type, int):
|
||||
raise BilibiliProtocolError(f"fav/resource/ids data[{index}].type 不是整数")
|
||||
bvid = item.get("bvid") or item.get("bv_id")
|
||||
if not isinstance(bvid, str) or not bvid:
|
||||
bvid = None
|
||||
out.append(FavId(aid=aid, type=raw_type, bvid=bvid))
|
||||
return out
|
||||
|
||||
def fav_infos(
|
||||
self,
|
||||
media_id: int,
|
||||
aids: Sequence[int],
|
||||
*,
|
||||
folder_mid: int | None = None,
|
||||
chunk_size: int = DEFAULT_CHUNK,
|
||||
) -> InfosResult:
|
||||
if not 1 <= chunk_size <= MAX_CHUNK:
|
||||
raise ValueError(f"chunk_size 必须在 1..{MAX_CHUNK} 之间")
|
||||
requested: list[int] = []
|
||||
seen: set[int] = set()
|
||||
for raw in aids:
|
||||
aid = int(raw)
|
||||
if aid not in seen:
|
||||
seen.add(aid)
|
||||
requested.append(aid)
|
||||
if not requested:
|
||||
return InfosResult(requested=(), items={}, missing=())
|
||||
|
||||
fmid = self.mid if folder_mid is None else int(folder_mid)
|
||||
items: dict[int, FavResource] = {}
|
||||
for chunk in _chunks(requested, chunk_size):
|
||||
resources = ",".join(f"{aid}:2" for aid in chunk)
|
||||
url = (
|
||||
f"{BASE}/x/v3/fav/resource/infos"
|
||||
f"?resources={urllib.parse.quote(resources)}"
|
||||
f"&platform=web&folder_mid={fmid}&folder_id={int(media_id)}"
|
||||
)
|
||||
payload = self.get_json(url)
|
||||
data = payload.get("data")
|
||||
if data is None:
|
||||
data = []
|
||||
if not isinstance(data, list):
|
||||
raise BilibiliProtocolError("fav/resource/infos data 不是数组")
|
||||
chunk_set = set(chunk)
|
||||
for item in data:
|
||||
if not isinstance(item, Mapping):
|
||||
raise BilibiliProtocolError("fav/resource/infos item 不是对象")
|
||||
try:
|
||||
parsed = FavResource.from_api(item)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise BilibiliProtocolError(f"fav/resource/infos 解析失败: {exc}") from exc
|
||||
if parsed.aid not in chunk_set:
|
||||
raise BilibiliProtocolError(
|
||||
f"fav/resource/infos 返回了未请求的 aid={parsed.aid}"
|
||||
)
|
||||
items[parsed.aid] = parsed
|
||||
|
||||
missing = tuple(aid for aid in requested if aid not in items)
|
||||
return InfosResult(requested=tuple(requested), items=items, missing=missing)
|
||||
|
||||
def fav_folders(self, up_mid: int | None = None) -> list[FolderInfo]:
|
||||
mid = self.mid if up_mid is None else int(up_mid)
|
||||
url = f"{BASE}/x/v3/fav/folder/created/list-all?up_mid={mid}&jsonp=jsonp"
|
||||
payload = self.get_json(url, auth=True)
|
||||
data = payload.get("data") or {}
|
||||
if not isinstance(data, Mapping):
|
||||
raise BilibiliProtocolError("fav/folder/created/list-all data 不是对象")
|
||||
rows = data.get("list")
|
||||
if rows is None:
|
||||
return []
|
||||
if not isinstance(rows, list):
|
||||
raise BilibiliProtocolError("fav/folder/created/list-all data.list 不是数组")
|
||||
out: list[FolderInfo] = []
|
||||
for index, row in enumerate(rows):
|
||||
if not isinstance(row, Mapping):
|
||||
raise BilibiliProtocolError(f"folder list[{index}] 不是对象")
|
||||
media_id = row.get("id")
|
||||
if isinstance(media_id, bool) or not isinstance(media_id, int):
|
||||
raise BilibiliProtocolError(f"folder list[{index}].id 不是整数")
|
||||
count = row.get("media_count", 0)
|
||||
if isinstance(count, bool) or not isinstance(count, int):
|
||||
count = 0
|
||||
out.append(FolderInfo(
|
||||
media_id=media_id,
|
||||
title=str(row.get("title") or ""),
|
||||
media_count=count,
|
||||
))
|
||||
return out
|
||||
@@ -0,0 +1,87 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""B 站收藏夹接口的返回模型。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Mapping
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FavId:
|
||||
"""``fav/resource/ids`` 的一条资源。"""
|
||||
|
||||
aid: int
|
||||
type: int
|
||||
bvid: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FavResource:
|
||||
"""``fav/resource/infos`` 的一条视频元数据。"""
|
||||
|
||||
aid: int
|
||||
bvid: str | None
|
||||
title: str
|
||||
up_mid: int | None
|
||||
up_name: str | None
|
||||
pubdate: int
|
||||
duration: int | None
|
||||
cover: str | None
|
||||
attr: int
|
||||
fav_time: int | None
|
||||
|
||||
@property
|
||||
def invalid(self) -> bool:
|
||||
# 实测 attr=9 表示"已失效视频"。
|
||||
return self.attr == 9
|
||||
|
||||
@classmethod
|
||||
def from_api(cls, item: Mapping[str, Any]) -> "FavResource":
|
||||
aid = item.get("id")
|
||||
if isinstance(aid, bool) or not isinstance(aid, int):
|
||||
raise ValueError("infos item.id 不是整数")
|
||||
upper = item.get("upper") or {}
|
||||
if not isinstance(upper, Mapping):
|
||||
upper = {}
|
||||
up_mid = upper.get("mid")
|
||||
if isinstance(up_mid, bool) or not isinstance(up_mid, int):
|
||||
up_mid = None
|
||||
raw_duration = item.get("duration")
|
||||
duration = raw_duration if isinstance(raw_duration, int) and not isinstance(
|
||||
raw_duration, bool
|
||||
) else None
|
||||
raw_attr = item.get("attr", 0)
|
||||
attr = raw_attr if isinstance(raw_attr, int) and not isinstance(raw_attr, bool) else 0
|
||||
fav_time = item.get("fav_time")
|
||||
if isinstance(fav_time, bool) or not isinstance(fav_time, int) or fav_time <= 0:
|
||||
fav_time = None
|
||||
bvid = item.get("bvid") or item.get("bv_id")
|
||||
if not isinstance(bvid, str) or not bvid:
|
||||
bvid = None
|
||||
return cls(
|
||||
aid=aid,
|
||||
bvid=bvid,
|
||||
title=str(item.get("title") or ""),
|
||||
up_mid=up_mid,
|
||||
up_name=str(upper.get("name") or "") or None,
|
||||
pubdate=int(item.get("pubtime") or item.get("ctime") or 0),
|
||||
duration=duration,
|
||||
cover=str(item.get("cover") or "") or None,
|
||||
attr=attr,
|
||||
fav_time=fav_time,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FolderInfo:
|
||||
media_id: int
|
||||
title: str
|
||||
media_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InfosResult:
|
||||
requested: tuple[int, ...]
|
||||
items: dict[int, FavResource]
|
||||
missing: tuple[int, ...]
|
||||
@@ -0,0 +1,217 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""P0 命令行入口。
|
||||
|
||||
python app.py init # 建库 / 迁移 / 写入黑名单种子(离线)
|
||||
python app.py sync # 收藏夹只读同步(首次导入或增量)
|
||||
python app.py run # 按 folder_sync_minutes 常驻周期同步(--once 只跑一次)
|
||||
python app.py status # 查看库内状态(离线)
|
||||
|
||||
P2 起 ``python app.py`` 将在此基础上启动本地看板服务。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
from .bilibili.client import BilibiliClient
|
||||
from .config import ConfigError, ConfigStore
|
||||
from .database import Database
|
||||
from .logging_json import JsonLogger
|
||||
from .repository import Repository
|
||||
from .scheduler import ScheduledTask, Scheduler
|
||||
from .services.folder_sync import FolderSyncService
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_CONFIG = ROOT / "config.json"
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
common = argparse.ArgumentParser(add_help=False)
|
||||
common.add_argument(
|
||||
"--config",
|
||||
default=str(DEFAULT_CONFIG),
|
||||
help="配置文件路径(默认项目根目录 config.json)",
|
||||
)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="app.py",
|
||||
description="咻咻满 Reaction 数据看板 —— P0 数据底座",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
sub.add_parser("init", parents=[common], help="建库、迁移并写入黑名单种子")
|
||||
sync = sub.add_parser("sync", parents=[common], help="同步已配置的年度收藏夹")
|
||||
sync.add_argument(
|
||||
"--full", action="store_true", help="强制全量刷新元数据(忽略 6 小时周期)"
|
||||
)
|
||||
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行收藏夹同步")
|
||||
run.add_argument(
|
||||
"--once", action="store_true", help="只执行一次到期任务后退出"
|
||||
)
|
||||
sub.add_parser("status", parents=[common], help="查看数据库状态(离线)")
|
||||
return parser
|
||||
|
||||
|
||||
def _open_db(cfg, logger: JsonLogger) -> Database:
|
||||
db = Database(
|
||||
cfg.storage.database_path,
|
||||
backup_dir=cfg.storage.backup_dir,
|
||||
backup_keep_days=cfg.storage.backup_keep_days,
|
||||
logger=logger,
|
||||
)
|
||||
db.initialize()
|
||||
return db
|
||||
|
||||
|
||||
def _print(payload) -> None:
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2), flush=True)
|
||||
|
||||
|
||||
def cmd_init(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=False)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
for folder in cfg.folders:
|
||||
repo.ensure_folder(folder)
|
||||
seeded = repo.seed_blacklist(cfg.up_blacklist_seed)
|
||||
_print({
|
||||
"ok": True,
|
||||
"schema_version": db.schema_version,
|
||||
"database": str(cfg.storage.database_path),
|
||||
"blacklist_seeded": seeded,
|
||||
**repo.counts(),
|
||||
})
|
||||
finally:
|
||||
db.close()
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_sync(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=True)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
client = BilibiliClient(
|
||||
cookie=cfg.bilibili.cookie,
|
||||
csrf=cfg.bilibili.csrf,
|
||||
mid=cfg.bilibili.mid,
|
||||
logger=logger,
|
||||
)
|
||||
service = FolderSyncService(store, repo, client, logger=logger)
|
||||
summary = service.run(full_refresh=args.full)
|
||||
payload = summary.as_dict()
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
payload["counts"] = repo.counts()
|
||||
_print(payload)
|
||||
return 0 if summary.ok else 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def cmd_run(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=True)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
last: dict[str, object] = {}
|
||||
try:
|
||||
repo = Repository(db)
|
||||
client = BilibiliClient(
|
||||
cookie=cfg.bilibili.cookie,
|
||||
csrf=cfg.bilibili.csrf,
|
||||
mid=cfg.bilibili.mid,
|
||||
logger=logger,
|
||||
)
|
||||
service = FolderSyncService(store, repo, client, logger=logger)
|
||||
|
||||
def run_sync():
|
||||
summary = service.run()
|
||||
last["summary"] = summary
|
||||
logger.info("folder_sync.summary", **summary.as_dict())
|
||||
return summary
|
||||
|
||||
scheduler = Scheduler(
|
||||
[ScheduledTask(
|
||||
"folder_sync",
|
||||
float(cfg.schedule.folder_sync_minutes * 60),
|
||||
run_sync,
|
||||
)],
|
||||
logger=logger,
|
||||
)
|
||||
if args.once:
|
||||
results = scheduler.run_due()
|
||||
_print([result.as_dict() for result in results])
|
||||
else:
|
||||
logger.info(
|
||||
"scheduler.start",
|
||||
interval_seconds=cfg.schedule.folder_sync_minutes * 60,
|
||||
)
|
||||
scheduler.run_forever()
|
||||
summary = last.get("summary")
|
||||
return 0 if summary is None or getattr(summary, "ok", True) else 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def cmd_status(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=False)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
_print({
|
||||
"ok": True,
|
||||
"schema_version": db.schema_version,
|
||||
"database": str(cfg.storage.database_path),
|
||||
"configured_folders": [
|
||||
{
|
||||
"media_id": row.media_id,
|
||||
"year": row.year,
|
||||
"title": row.title,
|
||||
"media_count": row.media_count,
|
||||
"last_ids_at": row.last_ids_at,
|
||||
"last_full_at": row.last_full_at,
|
||||
"enabled": row.enabled,
|
||||
}
|
||||
for row in repo.list_folders()
|
||||
],
|
||||
**repo.counts(),
|
||||
})
|
||||
finally:
|
||||
db.close()
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = _build_parser().parse_args(list(argv) if argv is not None else None)
|
||||
logger = JsonLogger()
|
||||
try:
|
||||
if args.command == "init":
|
||||
return cmd_init(args, logger)
|
||||
if args.command == "sync":
|
||||
return cmd_sync(args, logger)
|
||||
if args.command == "run":
|
||||
return cmd_run(args, logger)
|
||||
if args.command == "status":
|
||||
return cmd_status(args, logger)
|
||||
except ConfigError as exc:
|
||||
logger.error("config.error", error=str(exc))
|
||||
return 2
|
||||
except KeyboardInterrupt:
|
||||
logger.warning("app.interrupted")
|
||||
return 130
|
||||
except Exception as exc: # noqa: BLE001 CLI 边界统一记录
|
||||
logger.error("app.error", error=str(exc))
|
||||
return 2
|
||||
logger.error("app.unknown_command", command=getattr(args, "command", None))
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,495 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""运行期配置加载、校验与热加载。
|
||||
|
||||
支持两种写法:
|
||||
|
||||
* 新格式:``{"bilibili": {"mid": ..., "cookie": ...}, ...}``;
|
||||
* 旧 ``favsort.py`` 格式:顶层 ``mid`` / ``cookie`` / ``csrf``。
|
||||
|
||||
旧的扁平字段仍然被读取,因此 ``bili-fav auth`` 写回的 config.json 无需迁移即可
|
||||
被看板使用。敏感值只保存在配置对象中,日志输出必须先经过 ``logging_json`` 脱敏。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
DEFAULT_CLASSIFY: dict[str, Any] = {
|
||||
"require_title": "咻咻满",
|
||||
"positive": [
|
||||
"reaction", "re", "反应", "看", "听", "初听", "首次",
|
||||
"点评", "锐评", "路人", "震惊", "双人",
|
||||
],
|
||||
"charge": ["充电", "加急", "专场"],
|
||||
"negative": ["歌切", "切片", "纯享", "搬运", "完整版"],
|
||||
"duration_boost": {"gt_30min": 1, "gt_60min": 2},
|
||||
}
|
||||
|
||||
DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
|
||||
"view": 0.30,
|
||||
"like": 0.20,
|
||||
"coin": 0.20,
|
||||
"favorite": 0.15,
|
||||
"share": 0.10,
|
||||
"reply": 0.03,
|
||||
"danmaku": 0.02,
|
||||
}
|
||||
|
||||
DEFAULT_CONFIG: dict[str, Any] = {
|
||||
"bilibili": {"mid": 0, "csrf": "", "cookie": ""},
|
||||
"telegram": {"enabled": False, "bot_token": "", "chat_id": 0},
|
||||
"web": {
|
||||
"host": "127.0.0.1",
|
||||
"port": 8080,
|
||||
"auth_token": "",
|
||||
"public_base_url": "",
|
||||
},
|
||||
"storage": {
|
||||
"database_path": "data/app.db",
|
||||
"backup_dir": "data/backup",
|
||||
"backup_keep_days": 14,
|
||||
},
|
||||
"folders": [],
|
||||
"track_fans": [],
|
||||
"up_blacklist_seed": [],
|
||||
"schedule": {
|
||||
"discover_minutes": 60,
|
||||
"folder_sync_minutes": 60,
|
||||
"full_folder_refresh_hours": 6,
|
||||
"snapshot_at": "00:01",
|
||||
"candidate_ttl_days": 7,
|
||||
},
|
||||
"notify": {"min_score": 2, "max_items_per_message": 10},
|
||||
"classify": DEFAULT_CLASSIFY,
|
||||
"heat_weights": DEFAULT_HEAT_WEIGHTS,
|
||||
}
|
||||
|
||||
# 环境变量只作为 CI / 临时调试的覆盖入口,不作为主路径。
|
||||
ENV_OVERRIDES: dict[str, tuple[tuple[str, ...], type]] = {
|
||||
"BILI_COOKIE": (("bilibili", "cookie"), str),
|
||||
"BILI_CSRF": (("bilibili", "csrf"), str),
|
||||
"BILI_MID": (("bilibili", "mid"), int),
|
||||
"TG_BOT_TOKEN": (("telegram", "bot_token"), str),
|
||||
"TG_CHAT_ID": (("telegram", "chat_id"), int),
|
||||
"DASHBOARD_AUTH_TOKEN": (("web", "auth_token"), str),
|
||||
"DATABASE_PATH": (("storage", "database_path"), str),
|
||||
}
|
||||
|
||||
_SNAPSHOT_AT_RE = re.compile(r"^([01]\d|2[0-3]):[0-5]\d$")
|
||||
_HEAT_METRICS = tuple(DEFAULT_HEAT_WEIGHTS)
|
||||
|
||||
|
||||
class ConfigError(ValueError):
|
||||
"""配置文件缺失、格式错误或字段非法。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BilibiliConfig:
|
||||
mid: int
|
||||
csrf: str
|
||||
cookie: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TelegramConfig:
|
||||
enabled: bool
|
||||
bot_token: str
|
||||
chat_id: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WebConfig:
|
||||
host: str
|
||||
port: int
|
||||
auth_token: str
|
||||
public_base_url: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StorageConfig:
|
||||
database_path: Path
|
||||
backup_dir: Path
|
||||
backup_keep_days: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FolderConfig:
|
||||
year: int
|
||||
media_id: int
|
||||
title: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScheduleConfig:
|
||||
discover_minutes: int
|
||||
folder_sync_minutes: int
|
||||
full_folder_refresh_hours: int
|
||||
snapshot_at: str
|
||||
candidate_ttl_days: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NotifyConfig:
|
||||
min_score: int
|
||||
max_items_per_message: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AppConfig:
|
||||
path: Path
|
||||
bilibili: BilibiliConfig
|
||||
telegram: TelegramConfig
|
||||
web: WebConfig
|
||||
storage: StorageConfig
|
||||
folders: tuple[FolderConfig, ...]
|
||||
track_fans: tuple[int, ...]
|
||||
up_blacklist_seed: tuple[int, ...]
|
||||
schedule: ScheduleConfig
|
||||
notify: NotifyConfig
|
||||
classify: Mapping[str, Any]
|
||||
heat_weights: Mapping[str, float]
|
||||
raw: Mapping[str, Any] = field(repr=False, compare=False)
|
||||
|
||||
def folder_for_year(self, year: int) -> FolderConfig | None:
|
||||
for folder in self.folders:
|
||||
if folder.year == year:
|
||||
return folder
|
||||
return None
|
||||
|
||||
def latest_folder(self) -> FolderConfig | None:
|
||||
if not self.folders:
|
||||
return None
|
||||
return max(self.folders, key=lambda f: f.year)
|
||||
|
||||
|
||||
def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> dict[str, Any]:
|
||||
out = copy.deepcopy(dict(base))
|
||||
for key, value in override.items():
|
||||
if isinstance(value, Mapping) and isinstance(out.get(key), Mapping):
|
||||
out[key] = _deep_merge(out[key], value)
|
||||
else:
|
||||
out[key] = copy.deepcopy(value)
|
||||
return out
|
||||
|
||||
|
||||
def _get_path(data: Mapping[str, Any], path: tuple[str, ...]) -> Any:
|
||||
cur: Any = data
|
||||
for key in path:
|
||||
if not isinstance(cur, Mapping) or key not in cur:
|
||||
return None
|
||||
cur = cur[key]
|
||||
return cur
|
||||
|
||||
|
||||
def _set_path(data: dict[str, Any], path: tuple[str, ...], value: Any) -> None:
|
||||
cur = data
|
||||
for key in path[:-1]:
|
||||
cur = cur.setdefault(key, {})
|
||||
cur[path[-1]] = value
|
||||
|
||||
|
||||
def _as_int(value: Any, name: str, *, minimum: int | None = None,
|
||||
maximum: int | None = None) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise ConfigError(f"{name} 必须是整数")
|
||||
if minimum is not None and value < minimum:
|
||||
raise ConfigError(f"{name} 不能小于 {minimum}")
|
||||
if maximum is not None and value > maximum:
|
||||
raise ConfigError(f"{name} 不能大于 {maximum}")
|
||||
return value
|
||||
|
||||
|
||||
def _as_str(value: Any, name: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ConfigError(f"{name} 必须是字符串")
|
||||
return value
|
||||
|
||||
|
||||
def _as_int_list(value: Any, name: str) -> tuple[int, ...]:
|
||||
if not isinstance(value, list):
|
||||
raise ConfigError(f"{name} 必须是数组")
|
||||
out: list[int] = []
|
||||
for item in value:
|
||||
mid = _as_int(item, f"{name}[]", minimum=1)
|
||||
if mid not in out:
|
||||
out.append(mid)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _as_str_list(value: Any, name: str) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
raise ConfigError(f"{name} 必须是数组")
|
||||
out = []
|
||||
for item in value:
|
||||
if not isinstance(item, str) or not item:
|
||||
raise ConfigError(f"{name}[] 必须是非空字符串")
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
|
||||
def _legacy_to_current(raw: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""把 favsort 的扁平字段映射到新格式,不修改调用方对象。"""
|
||||
data = copy.deepcopy(dict(raw))
|
||||
if "bilibili" not in data:
|
||||
data["bilibili"] = {
|
||||
"mid": raw.get("mid", 0),
|
||||
"csrf": raw.get("csrf", ""),
|
||||
"cookie": raw.get("cookie", ""),
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
def _validate_folders(value: Any) -> tuple[FolderConfig, ...]:
|
||||
if not isinstance(value, list):
|
||||
raise ConfigError("folders 必须是数组")
|
||||
folders: list[FolderConfig] = []
|
||||
seen_ids: set[int] = set()
|
||||
seen_years: set[int] = set()
|
||||
for index, item in enumerate(value):
|
||||
name = f"folders[{index}]"
|
||||
if not isinstance(item, Mapping):
|
||||
raise ConfigError(f"{name} 必须是对象")
|
||||
year = _as_int(item.get("year"), f"{name}.year", minimum=1900, maximum=9999)
|
||||
media_id = _as_int(item.get("media_id"), f"{name}.media_id", minimum=1)
|
||||
title = _as_str(item.get("title", "") or "", f"{name}.title")
|
||||
if media_id in seen_ids:
|
||||
raise ConfigError(f"{name}.media_id 重复: {media_id}")
|
||||
if year in seen_years:
|
||||
raise ConfigError(f"{name}.year 重复: {year}")
|
||||
seen_ids.add(media_id)
|
||||
seen_years.add(year)
|
||||
folders.append(FolderConfig(year=year, media_id=media_id, title=title))
|
||||
return tuple(folders)
|
||||
|
||||
|
||||
def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
||||
b = data["bilibili"]
|
||||
if not isinstance(b, Mapping):
|
||||
raise ConfigError("bilibili 必须是对象")
|
||||
bilibili = BilibiliConfig(
|
||||
mid=_as_int(b.get("mid", 0), "bilibili.mid", minimum=0),
|
||||
csrf=_as_str(b.get("csrf", "") or "", "bilibili.csrf"),
|
||||
cookie=_as_str(b.get("cookie", "") or "", "bilibili.cookie"),
|
||||
)
|
||||
|
||||
tg = data["telegram"]
|
||||
if not isinstance(tg, Mapping):
|
||||
raise ConfigError("telegram 必须是对象")
|
||||
telegram = TelegramConfig(
|
||||
enabled=bool(tg.get("enabled", False)),
|
||||
bot_token=_as_str(tg.get("bot_token", "") or "", "telegram.bot_token"),
|
||||
chat_id=_as_int(tg.get("chat_id", 0), "telegram.chat_id"),
|
||||
)
|
||||
|
||||
web_raw = data["web"]
|
||||
if not isinstance(web_raw, Mapping):
|
||||
raise ConfigError("web 必须是对象")
|
||||
web = WebConfig(
|
||||
host=_as_str(web_raw.get("host", "127.0.0.1"), "web.host"),
|
||||
port=_as_int(web_raw.get("port", 8080), "web.port", minimum=1, maximum=65535),
|
||||
auth_token=_as_str(web_raw.get("auth_token", "") or "", "web.auth_token"),
|
||||
public_base_url=_as_str(
|
||||
web_raw.get("public_base_url", "") or "", "web.public_base_url"
|
||||
),
|
||||
)
|
||||
|
||||
storage_raw = data["storage"]
|
||||
if not isinstance(storage_raw, Mapping):
|
||||
raise ConfigError("storage 必须是对象")
|
||||
base_dir = path.parent
|
||||
database_path = Path(_as_str(
|
||||
storage_raw.get("database_path", "data/app.db") or "data/app.db",
|
||||
"storage.database_path",
|
||||
))
|
||||
backup_dir = Path(_as_str(
|
||||
storage_raw.get("backup_dir", "data/backup") or "data/backup",
|
||||
"storage.backup_dir",
|
||||
))
|
||||
if not database_path.is_absolute():
|
||||
database_path = base_dir / database_path
|
||||
if not backup_dir.is_absolute():
|
||||
backup_dir = base_dir / backup_dir
|
||||
storage = StorageConfig(
|
||||
database_path=database_path,
|
||||
backup_dir=backup_dir,
|
||||
backup_keep_days=_as_int(
|
||||
storage_raw.get("backup_keep_days", 14),
|
||||
"storage.backup_keep_days",
|
||||
minimum=1,
|
||||
),
|
||||
)
|
||||
|
||||
sched_raw = data["schedule"]
|
||||
if not isinstance(sched_raw, Mapping):
|
||||
raise ConfigError("schedule 必须是对象")
|
||||
snapshot_at = _as_str(sched_raw.get("snapshot_at", "00:01"), "schedule.snapshot_at")
|
||||
if not _SNAPSHOT_AT_RE.match(snapshot_at):
|
||||
raise ConfigError("schedule.snapshot_at 必须是 HH:MM(24 小时制)")
|
||||
schedule = ScheduleConfig(
|
||||
discover_minutes=_as_int(
|
||||
sched_raw.get("discover_minutes", 60), "schedule.discover_minutes", minimum=1
|
||||
),
|
||||
folder_sync_minutes=_as_int(
|
||||
sched_raw.get("folder_sync_minutes", 60),
|
||||
"schedule.folder_sync_minutes",
|
||||
minimum=1,
|
||||
),
|
||||
full_folder_refresh_hours=_as_int(
|
||||
sched_raw.get("full_folder_refresh_hours", 6),
|
||||
"schedule.full_folder_refresh_hours",
|
||||
minimum=1,
|
||||
),
|
||||
snapshot_at=snapshot_at,
|
||||
candidate_ttl_days=_as_int(
|
||||
sched_raw.get("candidate_ttl_days", 7),
|
||||
"schedule.candidate_ttl_days",
|
||||
minimum=1,
|
||||
),
|
||||
)
|
||||
|
||||
notify_raw = data["notify"]
|
||||
if not isinstance(notify_raw, Mapping):
|
||||
raise ConfigError("notify 必须是对象")
|
||||
notify = NotifyConfig(
|
||||
min_score=_as_int(notify_raw.get("min_score", 2), "notify.min_score"),
|
||||
max_items_per_message=_as_int(
|
||||
notify_raw.get("max_items_per_message", 10),
|
||||
"notify.max_items_per_message",
|
||||
minimum=1,
|
||||
),
|
||||
)
|
||||
|
||||
classify_raw = data["classify"]
|
||||
if not isinstance(classify_raw, Mapping):
|
||||
raise ConfigError("classify 必须是对象")
|
||||
classify = copy.deepcopy(dict(classify_raw))
|
||||
classify["require_title"] = _as_str(
|
||||
classify.get("require_title", "咻咻满"), "classify.require_title"
|
||||
)
|
||||
if not classify["require_title"]:
|
||||
raise ConfigError("classify.require_title 不能为空")
|
||||
for key in ("positive", "charge", "negative"):
|
||||
classify[key] = _as_str_list(classify.get(key, []), f"classify.{key}")
|
||||
boost = classify.get("duration_boost", {})
|
||||
if not isinstance(boost, Mapping):
|
||||
raise ConfigError("classify.duration_boost 必须是对象")
|
||||
classify["duration_boost"] = {
|
||||
"gt_30min": _as_int(boost.get("gt_30min", 1), "classify.duration_boost.gt_30min"),
|
||||
"gt_60min": _as_int(boost.get("gt_60min", 2), "classify.duration_boost.gt_60min"),
|
||||
}
|
||||
|
||||
heat_raw = data["heat_weights"]
|
||||
if not isinstance(heat_raw, Mapping):
|
||||
raise ConfigError("heat_weights 必须是对象")
|
||||
heat: dict[str, float] = {}
|
||||
for metric in _HEAT_METRICS:
|
||||
value = heat_raw.get(metric, DEFAULT_HEAT_WEIGHTS[metric])
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0:
|
||||
raise ConfigError(f"heat_weights.{metric} 必须是非负数字")
|
||||
heat[metric] = float(value)
|
||||
|
||||
return AppConfig(
|
||||
path=path,
|
||||
bilibili=bilibili,
|
||||
telegram=telegram,
|
||||
web=web,
|
||||
storage=storage,
|
||||
folders=_validate_folders(data["folders"]),
|
||||
track_fans=_as_int_list(data["track_fans"], "track_fans"),
|
||||
up_blacklist_seed=_as_int_list(data["up_blacklist_seed"], "up_blacklist_seed"),
|
||||
schedule=schedule,
|
||||
notify=notify,
|
||||
classify=classify,
|
||||
heat_weights=heat,
|
||||
raw=data,
|
||||
)
|
||||
|
||||
|
||||
def load_config(
|
||||
path: str | os.PathLike[str],
|
||||
*,
|
||||
env: Mapping[str, str] | None = None,
|
||||
require_auth: bool = False,
|
||||
) -> AppConfig:
|
||||
"""读取并校验配置。``env`` 默认为 ``os.environ``。"""
|
||||
config_path = Path(path)
|
||||
if not config_path.exists():
|
||||
raise ConfigError(f"缺少配置文件: {config_path}")
|
||||
try:
|
||||
with open(config_path, encoding="utf-8") as fh:
|
||||
raw = json.load(fh)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ConfigError(f"配置文件不是合法 JSON: {config_path}: {exc}") from exc
|
||||
except OSError as exc:
|
||||
raise ConfigError(f"无法读取配置文件: {config_path}: {exc}") from exc
|
||||
if not isinstance(raw, Mapping):
|
||||
raise ConfigError("配置文件顶层必须是 JSON 对象")
|
||||
|
||||
data = _deep_merge(DEFAULT_CONFIG, _legacy_to_current(raw))
|
||||
|
||||
env_map = os.environ if env is None else env
|
||||
for env_name, (target, caster) in ENV_OVERRIDES.items():
|
||||
if env_name not in env_map:
|
||||
continue
|
||||
value: Any = env_map[env_name]
|
||||
if caster is int:
|
||||
try:
|
||||
value = int(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ConfigError(f"环境变量 {env_name} 必须是整数") from exc
|
||||
_set_path(data, target, value)
|
||||
|
||||
cfg = _build_config(data, config_path)
|
||||
if require_auth and (not cfg.bilibili.cookie or not cfg.bilibili.csrf):
|
||||
raise ConfigError(
|
||||
"config.json 中的 cookie / csrf 不能为空(可先运行 bili-fav auth)"
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
class ConfigStore:
|
||||
"""按文件 mtime 热加载配置;坏配置不会覆盖上一份可用配置。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: str | os.PathLike[str],
|
||||
*,
|
||||
env: Mapping[str, str] | None = None,
|
||||
require_auth: bool = False,
|
||||
) -> None:
|
||||
self.path = Path(path)
|
||||
self._env = env
|
||||
self._require_auth = require_auth
|
||||
self._lock = threading.RLock()
|
||||
self._config = load_config(path, env=env, require_auth=require_auth)
|
||||
self._mtime_ns = self._read_mtime_ns()
|
||||
self.last_error: str | None = None
|
||||
|
||||
def _read_mtime_ns(self) -> int | None:
|
||||
try:
|
||||
return self.path.stat().st_mtime_ns
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
def get(self) -> AppConfig:
|
||||
with self._lock:
|
||||
mtime_ns = self._read_mtime_ns()
|
||||
if mtime_ns is None or mtime_ns == self._mtime_ns:
|
||||
return self._config
|
||||
self._mtime_ns = mtime_ns
|
||||
try:
|
||||
self._config = load_config(
|
||||
self.path, env=self._env, require_auth=self._require_auth
|
||||
)
|
||||
except ConfigError as exc:
|
||||
self.last_error = str(exc)
|
||||
return self._config
|
||||
self.last_error = None
|
||||
return self._config
|
||||
@@ -0,0 +1,175 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""SQLite 连接、迁移入口与单写队列。
|
||||
|
||||
所有写操作经单个后台线程串行执行,避免并发任务出现 ``database is locked``;
|
||||
读操作使用短生命周期连接。网络请求绝不在写事务内执行。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import sqlite3
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, TypeVar
|
||||
|
||||
from .migrations import current_version, run_migrations
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class DatabaseClosed(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Job:
|
||||
fn: Callable[[sqlite3.Connection], Any]
|
||||
done: threading.Event = field(default_factory=threading.Event)
|
||||
result: Any = None
|
||||
error: BaseException | None = None
|
||||
|
||||
|
||||
def _configure(conn: sqlite3.Connection) -> sqlite3.Connection:
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("PRAGMA busy_timeout = 5000")
|
||||
conn.execute("PRAGMA synchronous = NORMAL")
|
||||
conn.execute("PRAGMA journal_mode = WAL")
|
||||
return conn
|
||||
|
||||
|
||||
class Database:
|
||||
"""线程安全的 SQLite 访问入口。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: str | Path,
|
||||
*,
|
||||
backup_dir: str | Path | None = None,
|
||||
backup_keep_days: int = 14,
|
||||
logger=None,
|
||||
) -> None:
|
||||
self.path = Path(path)
|
||||
self.backup_dir = Path(backup_dir) if backup_dir is not None else None
|
||||
self.backup_keep_days = backup_keep_days
|
||||
self.logger = logger
|
||||
self.schema_version: int | None = None
|
||||
self._queue: queue.Queue[_Job | None] = queue.Queue()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._ready = threading.Event()
|
||||
self._writer_error: BaseException | None = None
|
||||
self._closed = False
|
||||
|
||||
# ------------------------------------------------------------ setup
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
# isolation_level=None:单条 DML 立即提交;显式事务由 transaction() 管理。
|
||||
return _configure(
|
||||
sqlite3.connect(self.path, timeout=5.0, isolation_level=None)
|
||||
)
|
||||
|
||||
def initialize(self) -> list[int]:
|
||||
"""创建目录、执行迁移并启动单写线程。"""
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = self._connect()
|
||||
try:
|
||||
applied = run_migrations(
|
||||
conn,
|
||||
backup_dir=self.backup_dir,
|
||||
backup_keep_days=self.backup_keep_days,
|
||||
logger=self.logger,
|
||||
)
|
||||
self.schema_version = current_version(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
self._start_writer()
|
||||
if self.logger is not None:
|
||||
self.logger.info(
|
||||
"db.ready", path=str(self.path), schema_version=self.schema_version
|
||||
)
|
||||
return applied
|
||||
|
||||
def _start_writer(self) -> None:
|
||||
if self._thread is not None:
|
||||
return
|
||||
self._thread = threading.Thread(target=self._writer_loop, name="sqlite-writer", daemon=True)
|
||||
self._thread.start()
|
||||
self._ready.wait(timeout=10)
|
||||
if self._writer_error is not None:
|
||||
raise RuntimeError(f"SQLite 写线程启动失败: {self._writer_error}") from self._writer_error
|
||||
|
||||
def _writer_loop(self) -> None:
|
||||
conn: sqlite3.Connection | None = None
|
||||
try:
|
||||
conn = self._connect()
|
||||
except BaseException as exc: # noqa: BLE001
|
||||
self._writer_error = exc
|
||||
self._ready.set()
|
||||
return
|
||||
self._ready.set()
|
||||
try:
|
||||
while True:
|
||||
job = self._queue.get()
|
||||
if job is None:
|
||||
break
|
||||
try:
|
||||
job.result = job.fn(conn)
|
||||
except BaseException as exc: # noqa: BLE001 原样返回给调用方
|
||||
job.error = exc
|
||||
finally:
|
||||
job.done.set()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# ------------------------------------------------------------ access
|
||||
|
||||
def write(self, fn: Callable[[sqlite3.Connection], T]) -> T:
|
||||
"""把函数交给单写线程执行,并同步等待结果。"""
|
||||
if self._closed:
|
||||
raise DatabaseClosed("数据库已关闭")
|
||||
if self._thread is None:
|
||||
raise RuntimeError("请先调用 initialize()")
|
||||
job = _Job(fn=fn)
|
||||
self._queue.put(job)
|
||||
job.done.wait()
|
||||
if job.error is not None:
|
||||
raise job.error
|
||||
return job.result
|
||||
|
||||
def transaction(self, fn: Callable[[sqlite3.Connection], T]) -> T:
|
||||
"""在单写线程内执行一个显式事务。"""
|
||||
def run(conn: sqlite3.Connection) -> T:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
result = fn(conn)
|
||||
conn.execute("COMMIT")
|
||||
return result
|
||||
except BaseException:
|
||||
try:
|
||||
conn.execute("ROLLBACK")
|
||||
except sqlite3.Error:
|
||||
pass
|
||||
raise
|
||||
|
||||
return self.write(run)
|
||||
|
||||
def read(self, fn: Callable[[sqlite3.Connection], T]) -> T:
|
||||
"""使用独立的短生命周期连接执行只读查询。"""
|
||||
if self._closed:
|
||||
raise DatabaseClosed("数据库已关闭")
|
||||
conn = self._connect()
|
||||
try:
|
||||
return fn(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
if self._thread is not None:
|
||||
self._queue.put(None)
|
||||
self._thread.join(timeout=10)
|
||||
self._thread = None
|
||||
@@ -0,0 +1,101 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""脱敏 JSON 行日志。
|
||||
|
||||
日志只写 stdout,敏感字段(cookie / csrf / token / secret 等)在序列化前统一
|
||||
替换为 ``***``;即使调用方误把异常文本或 Cookie 塞进普通字段,也会在字符串
|
||||
层面再脱敏一次。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, IO, Mapping
|
||||
|
||||
REDACTED = "***"
|
||||
|
||||
_SENSITIVE_KEY_PARTS = (
|
||||
"cookie",
|
||||
"csrf",
|
||||
"token",
|
||||
"secret",
|
||||
"password",
|
||||
"authorization",
|
||||
"sessdata",
|
||||
"bili_jct",
|
||||
)
|
||||
|
||||
_STRING_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(
|
||||
re.compile(
|
||||
r"(?i)\b(SESSDATA|bili_jct|buvid3|buvid4|DedeUserID__ckMd5|DedeUserID)"
|
||||
r"\s*=\s*([^;\s]+)"
|
||||
),
|
||||
r"\1=" + REDACTED,
|
||||
),
|
||||
(re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+"), "Bearer " + REDACTED),
|
||||
(re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{25,}\b"), REDACTED),
|
||||
)
|
||||
|
||||
|
||||
def _is_sensitive_key(key: str) -> bool:
|
||||
normalized = key.lower().replace("-", "_")
|
||||
return any(part in normalized for part in _SENSITIVE_KEY_PARTS)
|
||||
|
||||
|
||||
def _redact_string(value: str) -> str:
|
||||
for pattern, replacement in _STRING_PATTERNS:
|
||||
value = pattern.sub(replacement, value)
|
||||
return value
|
||||
|
||||
|
||||
def redact(value: Any, *, key: str | None = None) -> Any:
|
||||
"""递归脱敏。敏感键的整个值替换为 ``***``。"""
|
||||
if key is not None and _is_sensitive_key(key):
|
||||
return REDACTED
|
||||
if isinstance(value, str):
|
||||
return _redact_string(value)
|
||||
if isinstance(value, Mapping):
|
||||
return {k: redact(v, key=str(k)) for k, v in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [redact(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
class JsonLogger:
|
||||
"""线程安全的 JSON 行 logger。"""
|
||||
|
||||
def __init__(self, stream: IO[str] | None = None) -> None:
|
||||
self.stream = stream if stream is not None else sys.stdout
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def emit(self, level: str, event: str, **fields: Any) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"ts": _now_iso(),
|
||||
"level": level,
|
||||
"event": event,
|
||||
}
|
||||
payload.update(redact(fields))
|
||||
line = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
||||
with self._lock:
|
||||
print(line, file=self.stream, flush=True)
|
||||
return payload
|
||||
|
||||
def debug(self, event: str, **fields: Any) -> dict[str, Any]:
|
||||
return self.emit("debug", event, **fields)
|
||||
|
||||
def info(self, event: str, **fields: Any) -> dict[str, Any]:
|
||||
return self.emit("info", event, **fields)
|
||||
|
||||
def warning(self, event: str, **fields: Any) -> dict[str, Any]:
|
||||
return self.emit("warning", event, **fields)
|
||||
|
||||
def error(self, event: str, **fields: Any) -> dict[str, Any]:
|
||||
return self.emit("error", event, **fields)
|
||||
@@ -0,0 +1,305 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""SQLite schema 迁移。
|
||||
|
||||
规则(DEVELOPMENT.md §6.3):
|
||||
|
||||
* 使用 ``schema_version(version, applied_at)`` 记录已应用版本;
|
||||
* 每个版本包在单一事务里,失败整体回滚;
|
||||
* 只向前追加,不允许删库重建;
|
||||
* 升级已有数据库前自动创建 SQLite 备份。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Migration:
|
||||
version: int
|
||||
name: str
|
||||
statements: tuple[str, ...]
|
||||
|
||||
|
||||
MIGRATIONS: tuple[Migration, ...] = (
|
||||
Migration(
|
||||
version=1,
|
||||
name="initial schema",
|
||||
statements=(
|
||||
"""
|
||||
CREATE TABLE video (
|
||||
aid INTEGER PRIMARY KEY,
|
||||
bvid TEXT UNIQUE,
|
||||
title TEXT NOT NULL,
|
||||
up_mid INTEGER,
|
||||
up_name TEXT,
|
||||
pubdate INTEGER NOT NULL,
|
||||
duration INTEGER,
|
||||
cover TEXT,
|
||||
folder_id INTEGER,
|
||||
fav_time INTEGER,
|
||||
status TEXT NOT NULL,
|
||||
source TEXT,
|
||||
first_seen INTEGER NOT NULL,
|
||||
last_sync INTEGER,
|
||||
attr INTEGER
|
||||
)
|
||||
""",
|
||||
"CREATE INDEX idx_video_status ON video(status, pubdate DESC)",
|
||||
"CREATE INDEX idx_video_up ON video(up_mid, pubdate DESC)",
|
||||
"""
|
||||
CREATE TABLE snapshot (
|
||||
stat_date TEXT NOT NULL,
|
||||
aid INTEGER NOT NULL,
|
||||
view INTEGER,
|
||||
like INTEGER,
|
||||
favorite INTEGER,
|
||||
coin INTEGER,
|
||||
share INTEGER,
|
||||
reply INTEGER,
|
||||
danmaku INTEGER,
|
||||
captured_at INTEGER NOT NULL,
|
||||
ok INTEGER NOT NULL DEFAULT 1,
|
||||
err TEXT,
|
||||
src TEXT DEFAULT 'view',
|
||||
PRIMARY KEY (stat_date, aid)
|
||||
)
|
||||
""",
|
||||
"CREATE INDEX idx_snap_aid ON snapshot(aid, stat_date)",
|
||||
"""
|
||||
CREATE TABLE folder (
|
||||
media_id INTEGER PRIMARY KEY,
|
||||
year INTEGER NOT NULL,
|
||||
title TEXT,
|
||||
media_count INTEGER,
|
||||
last_ids_at INTEGER,
|
||||
last_full_at INTEGER,
|
||||
enabled INTEGER NOT NULL DEFAULT 1
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE folder_resource (
|
||||
media_id INTEGER NOT NULL,
|
||||
aid INTEGER NOT NULL,
|
||||
first_seen INTEGER NOT NULL,
|
||||
active INTEGER NOT NULL DEFAULT 1,
|
||||
last_seen INTEGER,
|
||||
PRIMARY KEY (media_id, aid)
|
||||
)
|
||||
""",
|
||||
"CREATE INDEX idx_folder_resource_aid ON folder_resource(aid, active)",
|
||||
"""
|
||||
CREATE TABLE up_blacklist (
|
||||
mid INTEGER PRIMARY KEY,
|
||||
name TEXT,
|
||||
reason TEXT,
|
||||
created_at INTEGER NOT NULL
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE up_stat (
|
||||
stat_date TEXT NOT NULL,
|
||||
mid INTEGER NOT NULL,
|
||||
follower INTEGER,
|
||||
following INTEGER,
|
||||
captured_at INTEGER NOT NULL,
|
||||
ok INTEGER NOT NULL DEFAULT 1,
|
||||
PRIMARY KEY (stat_date, mid)
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE candidate (
|
||||
aid INTEGER PRIMARY KEY,
|
||||
bvid TEXT,
|
||||
title TEXT,
|
||||
up_mid INTEGER,
|
||||
up_name TEXT,
|
||||
pubdate INTEGER,
|
||||
cover TEXT,
|
||||
score REAL,
|
||||
channel TEXT,
|
||||
first_seen INTEGER NOT NULL,
|
||||
notified_at INTEGER,
|
||||
decision TEXT,
|
||||
decided_at INTEGER
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE seen_aid (
|
||||
aid INTEGER PRIMARY KEY,
|
||||
first_seen INTEGER NOT NULL,
|
||||
source TEXT
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE notify (
|
||||
aid INTEGER NOT NULL,
|
||||
channel TEXT NOT NULL,
|
||||
batch_id TEXT,
|
||||
sent_at INTEGER NOT NULL,
|
||||
ok INTEGER NOT NULL,
|
||||
err TEXT,
|
||||
PRIMARY KEY (aid, channel)
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE fav_action (
|
||||
token TEXT PRIMARY KEY,
|
||||
aid INTEGER NOT NULL,
|
||||
folder_id INTEGER NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
expires_at INTEGER NOT NULL,
|
||||
done_at INTEGER,
|
||||
position INTEGER,
|
||||
err TEXT
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE kv (
|
||||
k TEXT PRIMARY KEY,
|
||||
v TEXT
|
||||
)
|
||||
""",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class MigrationError(RuntimeError):
|
||||
"""迁移失败,原事务已回滚。"""
|
||||
|
||||
|
||||
def _schema_version_table(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_version ("
|
||||
"version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL)"
|
||||
)
|
||||
|
||||
|
||||
def current_version(conn: sqlite3.Connection) -> int:
|
||||
_schema_version_table(conn)
|
||||
row = conn.execute("SELECT MAX(version) FROM schema_version").fetchone()
|
||||
return int(row[0] or 0) if row else 0
|
||||
|
||||
|
||||
def _user_tables(conn: sqlite3.Connection) -> list[str]:
|
||||
rows = conn.execute(
|
||||
"SELECT name FROM sqlite_master "
|
||||
"WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name != 'schema_version'"
|
||||
).fetchall()
|
||||
return [str(row[0]) for row in rows]
|
||||
|
||||
|
||||
def backup_database(
|
||||
conn: sqlite3.Connection,
|
||||
backup_dir: str | Path,
|
||||
*,
|
||||
version: int,
|
||||
now: float | None = None,
|
||||
) -> Path | None:
|
||||
"""用 SQLite backup API 生成一致性备份;内存库返回 None。"""
|
||||
try:
|
||||
row = conn.execute("PRAGMA database_list").fetchone()
|
||||
except sqlite3.Error:
|
||||
return None
|
||||
if not row or not row[2] or str(row[2]) == ":memory:":
|
||||
return None
|
||||
|
||||
target_dir = Path(backup_dir)
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
stamp = time.strftime("%Y%m%d%H%M%S", time.localtime(now))
|
||||
target = target_dir / f"app-pre-v{version}-{stamp}.db"
|
||||
suffix = 1
|
||||
while target.exists():
|
||||
target = target_dir / f"app-pre-v{version}-{stamp}-{suffix}.db"
|
||||
suffix += 1
|
||||
|
||||
dst = sqlite3.connect(target)
|
||||
try:
|
||||
conn.backup(dst)
|
||||
finally:
|
||||
dst.close()
|
||||
return target
|
||||
|
||||
|
||||
def prune_backups(
|
||||
backup_dir: str | Path, *, keep_days: int, now: float | None = None
|
||||
) -> list[Path]:
|
||||
"""删除超过保留期的迁移备份。"""
|
||||
target_dir = Path(backup_dir)
|
||||
if not target_dir.is_dir():
|
||||
return []
|
||||
cutoff = (time.time() if now is None else now) - keep_days * 86400
|
||||
removed: list[Path] = []
|
||||
for path in target_dir.glob("app-pre-v*.db"):
|
||||
try:
|
||||
if path.stat().st_mtime < cutoff:
|
||||
path.unlink()
|
||||
removed.append(path)
|
||||
except OSError:
|
||||
continue
|
||||
return removed
|
||||
|
||||
|
||||
def _validate_migrations(migrations: Sequence[Migration]) -> None:
|
||||
versions = [m.version for m in migrations]
|
||||
if versions != sorted(versions):
|
||||
raise ValueError("迁移版本必须按升序排列")
|
||||
if len(set(versions)) != len(versions):
|
||||
raise ValueError("迁移版本不能重复")
|
||||
if versions and versions[0] < 1:
|
||||
raise ValueError("迁移版本必须从 1 开始")
|
||||
|
||||
|
||||
def run_migrations(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
backup_dir: str | Path | None = None,
|
||||
backup_keep_days: int = 14,
|
||||
now: float | None = None,
|
||||
migrations: Sequence[Migration] = MIGRATIONS,
|
||||
logger=None,
|
||||
) -> list[int]:
|
||||
"""应用所有待执行迁移,返回本次应用的版本列表。"""
|
||||
_validate_migrations(migrations)
|
||||
_schema_version_table(conn)
|
||||
version = current_version(conn)
|
||||
pending = [m for m in migrations if m.version > version]
|
||||
if not pending:
|
||||
return []
|
||||
|
||||
if backup_dir is not None and _user_tables(conn):
|
||||
backup = backup_database(conn, backup_dir, version=version, now=now)
|
||||
prune_backups(backup_dir, keep_days=backup_keep_days)
|
||||
if logger is not None and backup is not None:
|
||||
logger.info("db.backup", path=str(backup), version=version)
|
||||
|
||||
applied: list[int] = []
|
||||
for migration in pending:
|
||||
try:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
for statement in migration.statements:
|
||||
conn.execute(statement)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_version(version, applied_at) VALUES (?, ?)",
|
||||
(migration.version, int(time.time() if now is None else now)),
|
||||
)
|
||||
conn.execute("COMMIT")
|
||||
except Exception as exc: # noqa: BLE001 任何错误都必须回滚
|
||||
try:
|
||||
conn.execute("ROLLBACK")
|
||||
except sqlite3.Error:
|
||||
pass
|
||||
raise MigrationError(
|
||||
f"迁移 v{migration.version} ({migration.name}) 失败,已回滚: {exc}"
|
||||
) from exc
|
||||
applied.append(migration.version)
|
||||
if logger is not None:
|
||||
logger.info("db.migrated", version=migration.version, name=migration.name)
|
||||
|
||||
return applied
|
||||
@@ -0,0 +1,441 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""P0 数据访问层:收藏夹、视频与黑名单。
|
||||
|
||||
业务状态迁移集中在这里,服务层只负责请求与编排。所有写操作经 ``Database``
|
||||
的单写队列串行执行。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
from .config import FolderConfig
|
||||
from .database import Database
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UpsertOutcome:
|
||||
aid: int
|
||||
status: str
|
||||
previous_folder_id: int | None
|
||||
moved_folder: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FolderRow:
|
||||
media_id: int
|
||||
year: int
|
||||
title: str
|
||||
media_count: int | None
|
||||
last_ids_at: int | None
|
||||
last_full_at: int | None
|
||||
enabled: bool
|
||||
|
||||
|
||||
class Repository:
|
||||
def __init__(self, db: Database, *, clock=time.time) -> None:
|
||||
self.db = db
|
||||
self._clock = clock
|
||||
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
|
||||
# ------------------------------------------------------------ folder
|
||||
|
||||
def ensure_folder(self, folder: FolderConfig, *, now: int | None = None) -> None:
|
||||
now = self._now() if now is None else int(now)
|
||||
def run(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO folder(media_id, year, title, media_count,
|
||||
last_ids_at, last_full_at, enabled)
|
||||
VALUES (?, ?, ?, NULL, NULL, NULL, 1)
|
||||
ON CONFLICT(media_id) DO UPDATE SET
|
||||
year = excluded.year,
|
||||
title = COALESCE(NULLIF(excluded.title, ''), folder.title),
|
||||
enabled = 1
|
||||
""",
|
||||
(folder.media_id, folder.year, folder.title),
|
||||
)
|
||||
self.db.write(run)
|
||||
|
||||
def disable_folders_except(self, media_ids: Iterable[int]) -> None:
|
||||
ids = [int(mid) for mid in media_ids]
|
||||
def run(conn: sqlite3.Connection) -> None:
|
||||
if ids:
|
||||
placeholders = ",".join("?" for _ in ids)
|
||||
conn.execute(
|
||||
f"UPDATE folder SET enabled = 0 WHERE media_id NOT IN ({placeholders})",
|
||||
ids,
|
||||
)
|
||||
else:
|
||||
conn.execute("UPDATE folder SET enabled = 0")
|
||||
self.db.write(run)
|
||||
|
||||
def finish_folder_sync(
|
||||
self,
|
||||
folder: FolderConfig,
|
||||
*,
|
||||
media_count: int,
|
||||
full_refresh: bool,
|
||||
now: int | None = None,
|
||||
) -> None:
|
||||
now = self._now() if now is None else int(now)
|
||||
def run(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO folder(media_id, year, title, media_count,
|
||||
last_ids_at, last_full_at, enabled)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 1)
|
||||
ON CONFLICT(media_id) DO UPDATE SET
|
||||
year = excluded.year,
|
||||
title = COALESCE(NULLIF(excluded.title, ''), folder.title),
|
||||
media_count = excluded.media_count,
|
||||
last_ids_at = excluded.last_ids_at,
|
||||
last_full_at = CASE
|
||||
WHEN ? THEN excluded.last_full_at
|
||||
ELSE folder.last_full_at
|
||||
END,
|
||||
enabled = 1
|
||||
""",
|
||||
(
|
||||
folder.media_id, folder.year, folder.title, int(media_count),
|
||||
now, now if full_refresh else None, 1 if full_refresh else 0,
|
||||
),
|
||||
)
|
||||
self.db.write(run)
|
||||
|
||||
def get_folder(self, media_id: int) -> FolderRow | None:
|
||||
def run(conn: sqlite3.Connection) -> FolderRow | None:
|
||||
row = conn.execute(
|
||||
"SELECT media_id, year, title, media_count, last_ids_at, "
|
||||
"last_full_at, enabled FROM folder WHERE media_id = ?",
|
||||
(int(media_id),),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return FolderRow(
|
||||
media_id=int(row["media_id"]),
|
||||
year=int(row["year"]),
|
||||
title=str(row["title"] or ""),
|
||||
media_count=row["media_count"],
|
||||
last_ids_at=row["last_ids_at"],
|
||||
last_full_at=row["last_full_at"],
|
||||
enabled=bool(row["enabled"]),
|
||||
)
|
||||
return self.db.read(run)
|
||||
|
||||
def list_folders(self) -> list[FolderRow]:
|
||||
def run(conn: sqlite3.Connection) -> list[FolderRow]:
|
||||
rows = conn.execute(
|
||||
"SELECT media_id, year, title, media_count, last_ids_at, "
|
||||
"last_full_at, enabled FROM folder ORDER BY year, media_id"
|
||||
).fetchall()
|
||||
return [
|
||||
FolderRow(
|
||||
media_id=int(row["media_id"]),
|
||||
year=int(row["year"]),
|
||||
title=str(row["title"] or ""),
|
||||
media_count=row["media_count"],
|
||||
last_ids_at=row["last_ids_at"],
|
||||
last_full_at=row["last_full_at"],
|
||||
enabled=bool(row["enabled"]),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
return self.db.read(run)
|
||||
|
||||
def folder_active_aids(self, media_id: int) -> set[int]:
|
||||
def run(conn: sqlite3.Connection) -> set[int]:
|
||||
rows = conn.execute(
|
||||
"SELECT aid FROM folder_resource WHERE media_id = ? AND active = 1",
|
||||
(int(media_id),),
|
||||
).fetchall()
|
||||
return {int(row["aid"]) for row in rows}
|
||||
return self.db.read(run)
|
||||
|
||||
def touch_folder_resources(self, media_id: int, *, now: int | None = None) -> None:
|
||||
now = self._now() if now is None else int(now)
|
||||
self.db.write(lambda conn: conn.execute(
|
||||
"UPDATE folder_resource SET last_seen = ? WHERE media_id = ? AND active = 1",
|
||||
(now, int(media_id)),
|
||||
))
|
||||
|
||||
# ------------------------------------------------------------ video
|
||||
|
||||
def upsert_from_fav(
|
||||
self,
|
||||
resource,
|
||||
folder_id: int,
|
||||
*,
|
||||
was_added: bool,
|
||||
now: int | None = None,
|
||||
) -> UpsertOutcome:
|
||||
now = self._now() if now is None else int(now)
|
||||
folder_id = int(folder_id)
|
||||
invalid = bool(getattr(resource, "invalid", False))
|
||||
|
||||
def run(conn: sqlite3.Connection) -> UpsertOutcome:
|
||||
previous = conn.execute(
|
||||
"SELECT status, folder_id FROM video WHERE aid = ?", (resource.aid,)
|
||||
).fetchone()
|
||||
previous_status = str(previous["status"]) if previous else None
|
||||
previous_folder = (
|
||||
int(previous["folder_id"])
|
||||
if previous and previous["folder_id"] is not None else None
|
||||
)
|
||||
|
||||
if previous_status is None:
|
||||
status = "removed" if invalid else "active"
|
||||
elif previous_status == "dropped":
|
||||
status = "active" if (was_added and not invalid) else "dropped"
|
||||
elif invalid:
|
||||
status = "removed"
|
||||
else:
|
||||
status = "active"
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate,
|
||||
duration, cover, folder_id, fav_time, status,
|
||||
source, first_seen, last_sync, attr)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(aid) DO UPDATE SET
|
||||
bvid = COALESCE(excluded.bvid, video.bvid),
|
||||
title = CASE
|
||||
WHEN excluded.status = 'removed' AND video.title != ''
|
||||
THEN video.title
|
||||
WHEN excluded.title != '' THEN excluded.title
|
||||
ELSE video.title
|
||||
END,
|
||||
up_mid = COALESCE(excluded.up_mid, video.up_mid),
|
||||
up_name = COALESCE(excluded.up_name, video.up_name),
|
||||
pubdate = CASE
|
||||
WHEN excluded.pubdate > 0 THEN excluded.pubdate
|
||||
ELSE video.pubdate
|
||||
END,
|
||||
duration = COALESCE(excluded.duration, video.duration),
|
||||
cover = COALESCE(excluded.cover, video.cover),
|
||||
folder_id = excluded.folder_id,
|
||||
fav_time = COALESCE(excluded.fav_time, video.fav_time),
|
||||
status = excluded.status,
|
||||
source = COALESCE(video.source, excluded.source),
|
||||
last_sync = excluded.last_sync,
|
||||
attr = excluded.attr
|
||||
""",
|
||||
(
|
||||
resource.aid,
|
||||
resource.bvid,
|
||||
resource.title or "(无标题)",
|
||||
resource.up_mid,
|
||||
resource.up_name,
|
||||
int(resource.pubdate or 0),
|
||||
resource.duration,
|
||||
resource.cover,
|
||||
folder_id,
|
||||
resource.fav_time,
|
||||
status,
|
||||
f"fav:{folder_id}",
|
||||
now,
|
||||
now,
|
||||
resource.attr,
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO folder_resource(media_id, aid, first_seen, active, last_seen)
|
||||
VALUES (?, ?, ?, 1, ?)
|
||||
ON CONFLICT(media_id, aid) DO UPDATE SET
|
||||
active = 1,
|
||||
last_seen = excluded.last_seen
|
||||
""",
|
||||
(folder_id, resource.aid, now, now),
|
||||
)
|
||||
return UpsertOutcome(
|
||||
aid=resource.aid,
|
||||
status=status,
|
||||
previous_folder_id=previous_folder,
|
||||
moved_folder=previous_folder is not None and previous_folder != folder_id,
|
||||
)
|
||||
return self.db.write(run)
|
||||
|
||||
def mark_missing_resource(
|
||||
self,
|
||||
aid: int,
|
||||
*,
|
||||
folder_id: int,
|
||||
bvid: str | None = None,
|
||||
now: int | None = None,
|
||||
) -> str:
|
||||
"""处理 infos 静默缺失:写入/更新为 removed,保留历史。"""
|
||||
now = self._now() if now is None else int(now)
|
||||
aid = int(aid)
|
||||
folder_id = int(folder_id)
|
||||
|
||||
def run(conn: sqlite3.Connection) -> str:
|
||||
row = conn.execute("SELECT status FROM video WHERE aid = ?", (aid,)).fetchone()
|
||||
previous_status = str(row["status"]) if row else None
|
||||
status = "dropped" if previous_status == "dropped" else "removed"
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate,
|
||||
duration, cover, folder_id, fav_time, status,
|
||||
source, first_seen, last_sync, attr)
|
||||
VALUES (?, ?, '(资源已失效)', NULL, NULL, 0, NULL, NULL, ?, NULL,
|
||||
?, ?, ?, ?, 9)
|
||||
ON CONFLICT(aid) DO UPDATE SET
|
||||
bvid = COALESCE(video.bvid, excluded.bvid),
|
||||
status = ?,
|
||||
last_sync = excluded.last_sync,
|
||||
attr = 9
|
||||
""",
|
||||
(aid, bvid, folder_id, status, f"fav:{folder_id}", now, now, status),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO folder_resource(media_id, aid, first_seen, active, last_seen)
|
||||
VALUES (?, ?, ?, 1, ?)
|
||||
ON CONFLICT(media_id, aid) DO UPDATE SET
|
||||
active = 1,
|
||||
last_seen = excluded.last_seen
|
||||
""",
|
||||
(folder_id, aid, now, now),
|
||||
)
|
||||
return status
|
||||
return self.db.write(run)
|
||||
|
||||
def mark_resource_removed(
|
||||
self, media_id: int, aid: int, *, now: int | None = None
|
||||
) -> str | None:
|
||||
"""资源从收藏夹移除:active -> unfaved,其他(dropped/removed)保持不变。"""
|
||||
now = self._now() if now is None else int(now)
|
||||
media_id, aid = int(media_id), int(aid)
|
||||
|
||||
def run(conn: sqlite3.Connection) -> str | None:
|
||||
conn.execute(
|
||||
"UPDATE folder_resource SET active = 0, last_seen = ? "
|
||||
"WHERE media_id = ? AND aid = ?",
|
||||
(now, media_id, aid),
|
||||
)
|
||||
other = conn.execute(
|
||||
"""
|
||||
SELECT 1
|
||||
FROM folder_resource fr
|
||||
JOIN folder f ON f.media_id = fr.media_id
|
||||
WHERE fr.aid = ? AND fr.active = 1 AND f.enabled = 1
|
||||
LIMIT 1
|
||||
""",
|
||||
(aid,),
|
||||
).fetchone()
|
||||
if other is None:
|
||||
conn.execute(
|
||||
"UPDATE video SET status = 'unfaved', last_sync = ? "
|
||||
"WHERE aid = ? AND status IN ('active', 'unfaved')",
|
||||
(now, aid),
|
||||
)
|
||||
row = conn.execute("SELECT status FROM video WHERE aid = ?", (aid,)).fetchone()
|
||||
return str(row["status"]) if row else None
|
||||
return self.db.write(run)
|
||||
|
||||
def set_video_status(self, aid: int, status: str, *, now: int | None = None) -> None:
|
||||
now = self._now() if now is None else int(now)
|
||||
self.db.write(lambda conn: conn.execute(
|
||||
"UPDATE video SET status = ?, last_sync = ? WHERE aid = ?",
|
||||
(status, now, int(aid)),
|
||||
))
|
||||
|
||||
def get_video(self, aid: int) -> dict[str, Any] | None:
|
||||
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
|
||||
row = conn.execute("SELECT * FROM video WHERE aid = ?", (int(aid),)).fetchone()
|
||||
return dict(row) if row else None
|
||||
return self.db.read(run)
|
||||
|
||||
def list_videos(self) -> list[dict[str, Any]]:
|
||||
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
|
||||
rows = conn.execute("SELECT * FROM video ORDER BY aid").fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
return self.db.read(run)
|
||||
|
||||
def folder_resources(self, media_id: int) -> list[dict[str, Any]]:
|
||||
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM folder_resource WHERE media_id = ? ORDER BY aid",
|
||||
(int(media_id),),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
return self.db.read(run)
|
||||
|
||||
# ------------------------------------------------------------ blacklist
|
||||
|
||||
def seed_blacklist(self, mids: Iterable[int], *, now: int | None = None) -> int:
|
||||
now = self._now() if now is None else int(now)
|
||||
def run(conn: sqlite3.Connection) -> int:
|
||||
inserted = 0
|
||||
for mid in mids:
|
||||
cur = conn.execute(
|
||||
"INSERT OR IGNORE INTO up_blacklist(mid, name, reason, created_at) "
|
||||
"VALUES (?, NULL, 'seed', ?)",
|
||||
(int(mid), now),
|
||||
)
|
||||
inserted += max(cur.rowcount, 0)
|
||||
return inserted
|
||||
return self.db.write(run)
|
||||
|
||||
def add_blacklist(
|
||||
self,
|
||||
mid: int,
|
||||
*,
|
||||
name: str | None = None,
|
||||
reason: str | None = None,
|
||||
now: int | None = None,
|
||||
) -> None:
|
||||
now = self._now() if now is None else int(now)
|
||||
self.db.write(lambda conn: conn.execute(
|
||||
"""
|
||||
INSERT INTO up_blacklist(mid, name, reason, created_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(mid) DO UPDATE SET
|
||||
name = COALESCE(excluded.name, up_blacklist.name),
|
||||
reason = COALESCE(excluded.reason, up_blacklist.reason)
|
||||
""",
|
||||
(int(mid), name, reason, now),
|
||||
))
|
||||
|
||||
def is_blacklisted(self, mid: int) -> bool:
|
||||
def run(conn: sqlite3.Connection) -> bool:
|
||||
row = conn.execute(
|
||||
"SELECT 1 FROM up_blacklist WHERE mid = ?", (int(mid),)
|
||||
).fetchone()
|
||||
return row is not None
|
||||
return self.db.read(run)
|
||||
|
||||
def blacklist(self) -> list[dict[str, Any]]:
|
||||
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
|
||||
rows = conn.execute(
|
||||
"SELECT mid, name, reason, created_at FROM up_blacklist ORDER BY created_at, mid"
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
return self.db.read(run)
|
||||
|
||||
# ------------------------------------------------------------ summary
|
||||
|
||||
def counts(self) -> dict[str, Any]:
|
||||
def run(conn: sqlite3.Connection) -> dict[str, Any]:
|
||||
status_rows = conn.execute(
|
||||
"SELECT status, COUNT(*) AS n FROM video GROUP BY status"
|
||||
).fetchall()
|
||||
by_status = {str(row["status"]): int(row["n"]) for row in status_rows}
|
||||
folder_count = conn.execute("SELECT COUNT(*) FROM folder").fetchone()[0]
|
||||
blacklist_count = conn.execute("SELECT COUNT(*) FROM up_blacklist").fetchone()[0]
|
||||
candidate_count = conn.execute("SELECT COUNT(*) FROM candidate").fetchone()[0]
|
||||
return {
|
||||
"video_total": sum(by_status.values()),
|
||||
"video_by_status": by_status,
|
||||
"folders": int(folder_count),
|
||||
"blacklist": int(blacklist_count),
|
||||
"candidates": int(candidate_count),
|
||||
}
|
||||
return self.db.read(run)
|
||||
@@ -0,0 +1,115 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""单进程内置调度器。
|
||||
|
||||
P0 只注册收藏夹同步任务;P1/P3 会在同一调度器上追加快照、粉丝和发现任务。
|
||||
调度只负责“什么时候运行”,具体任务保持幂等,并在自己的服务层处理重试。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Sequence
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScheduledTask:
|
||||
name: str
|
||||
interval: float
|
||||
fn: Callable[[], Any]
|
||||
next_run: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskResult:
|
||||
name: str
|
||||
started_at: float
|
||||
finished_at: float
|
||||
ok: bool
|
||||
error: str | None = None
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"started_at": int(self.started_at),
|
||||
"finished_at": int(self.finished_at),
|
||||
"ok": self.ok,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
|
||||
class Scheduler:
|
||||
def __init__(
|
||||
self,
|
||||
tasks: Sequence[ScheduledTask],
|
||||
*,
|
||||
clock=time.time,
|
||||
sleeper=time.sleep,
|
||||
logger=None,
|
||||
) -> None:
|
||||
if not tasks:
|
||||
raise ValueError("至少需要一个调度任务")
|
||||
for task in tasks:
|
||||
if task.interval <= 0:
|
||||
raise ValueError(f"任务 {task.name} 的 interval 必须大于 0")
|
||||
self.tasks = list(tasks)
|
||||
self._clock = clock
|
||||
self._sleeper = sleeper
|
||||
self.logger = logger
|
||||
|
||||
def _log(self, level: str, event: str, **fields) -> None:
|
||||
if self.logger is not None:
|
||||
getattr(self.logger, level)(event, **fields)
|
||||
|
||||
def run_due(self, *, now: float | None = None) -> list[TaskResult]:
|
||||
current = self._clock() if now is None else now
|
||||
results: list[TaskResult] = []
|
||||
for task in self.tasks:
|
||||
if task.next_run > current:
|
||||
continue
|
||||
started = current
|
||||
error: str | None = None
|
||||
try:
|
||||
task.fn()
|
||||
except Exception as exc: # noqa: BLE001 单任务失败不终止调度
|
||||
error = str(exc)
|
||||
self._log("error", "scheduler.task_error", task=task.name, error=error)
|
||||
else:
|
||||
self._log("info", "scheduler.task_ok", task=task.name)
|
||||
finished = self._clock()
|
||||
task.next_run = started + task.interval
|
||||
result = TaskResult(
|
||||
name=task.name,
|
||||
started_at=started,
|
||||
finished_at=finished,
|
||||
ok=error is None,
|
||||
error=error,
|
||||
)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
def run_forever(
|
||||
self,
|
||||
*,
|
||||
stop_event=None,
|
||||
poll_seconds: float = 30.0,
|
||||
max_iterations: int | None = None,
|
||||
) -> list[TaskResult]:
|
||||
"""阻塞运行;``stop_event`` 或 ``max_iterations`` 用于可控退出/测试。"""
|
||||
if poll_seconds <= 0:
|
||||
raise ValueError("poll_seconds 必须大于 0")
|
||||
all_results: list[TaskResult] = []
|
||||
iterations = 0
|
||||
while True:
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
break
|
||||
all_results.extend(self.run_due())
|
||||
iterations += 1
|
||||
if max_iterations is not None and iterations >= max_iterations:
|
||||
break
|
||||
if stop_event is not None:
|
||||
if stop_event.wait(poll_seconds):
|
||||
break
|
||||
else:
|
||||
self._sleeper(poll_seconds)
|
||||
return all_results
|
||||
@@ -0,0 +1,2 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""业务服务层。"""
|
||||
@@ -0,0 +1,278 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""收藏夹同步(库的真相源)。
|
||||
|
||||
每次运行:
|
||||
|
||||
1. 读取每个配置年度夹的全部 aid;
|
||||
2. 与本地 ``folder_resource`` 做差集;
|
||||
3. 新增资源按 200 条分块拉取 ``infos`` 并入库;
|
||||
4. 从收藏夹移除的资源标记 ``unfaved``(不删除历史);
|
||||
5. 每 ``full_folder_refresh_hours`` 小时全量刷新一次元数据并识别失效资源;
|
||||
6. 容量接近上限时预警,但绝不自动换夹。
|
||||
|
||||
黑名单只在发现流程生效,本服务不读取 ``up_blacklist``。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from ..bilibili.client import BilibiliClient
|
||||
from ..config import AppConfig, ConfigStore, FolderConfig
|
||||
from ..repository import Repository
|
||||
|
||||
CAPACITY_WARN = 950
|
||||
|
||||
|
||||
@dataclass
|
||||
class FolderSyncResult:
|
||||
media_id: int
|
||||
year: int
|
||||
title: str
|
||||
remote_count: int = 0
|
||||
added: int = 0
|
||||
removed: int = 0
|
||||
refreshed: int = 0
|
||||
missing: int = 0
|
||||
full_refresh: bool = False
|
||||
warning: str | None = None
|
||||
error: str | None = None
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"media_id": self.media_id,
|
||||
"year": self.year,
|
||||
"title": self.title,
|
||||
"remote_count": self.remote_count,
|
||||
"added": self.added,
|
||||
"removed": self.removed,
|
||||
"refreshed": self.refreshed,
|
||||
"missing": self.missing,
|
||||
"full_refresh": self.full_refresh,
|
||||
"warning": self.warning,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncSummary:
|
||||
started_at: int
|
||||
finished_at: int = 0
|
||||
folders: list[FolderSyncResult] = field(default_factory=list)
|
||||
blacklist_seeded: int = 0
|
||||
|
||||
@property
|
||||
def added(self) -> int:
|
||||
return sum(f.added for f in self.folders)
|
||||
|
||||
@property
|
||||
def removed(self) -> int:
|
||||
return sum(f.removed for f in self.folders)
|
||||
|
||||
@property
|
||||
def refreshed(self) -> int:
|
||||
return sum(f.refreshed for f in self.folders)
|
||||
|
||||
@property
|
||||
def missing(self) -> int:
|
||||
return sum(f.missing for f in self.folders)
|
||||
|
||||
@property
|
||||
def errors(self) -> list[str]:
|
||||
return [f"folder {f.media_id}: {f.error}" for f in self.folders if f.error]
|
||||
|
||||
@property
|
||||
def warnings(self) -> list[str]:
|
||||
return [f"folder {f.media_id}: {f.warning}" for f in self.folders if f.warning]
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return not self.errors
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"elapsed": max(0, self.finished_at - self.started_at),
|
||||
"added": self.added,
|
||||
"removed": self.removed,
|
||||
"refreshed": self.refreshed,
|
||||
"missing": self.missing,
|
||||
"blacklist_seeded": self.blacklist_seeded,
|
||||
"folders": [f.as_dict() for f in self.folders],
|
||||
"errors": self.errors,
|
||||
"warnings": self.warnings,
|
||||
"ok": self.ok,
|
||||
}
|
||||
|
||||
|
||||
class FolderSyncService:
|
||||
def __init__(
|
||||
self,
|
||||
config_store: ConfigStore,
|
||||
repository: Repository,
|
||||
client: BilibiliClient,
|
||||
*,
|
||||
clock=time.time,
|
||||
logger=None,
|
||||
capacity_warn: int = CAPACITY_WARN,
|
||||
) -> None:
|
||||
self.config_store = config_store
|
||||
self.repo = repository
|
||||
self.client = client
|
||||
self._clock = clock
|
||||
self.logger = logger
|
||||
self.capacity_warn = capacity_warn
|
||||
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
|
||||
def _log(self, level: str, event: str, **fields) -> None:
|
||||
if self.logger is not None:
|
||||
getattr(self.logger, level)(event, **fields)
|
||||
|
||||
def run(self, *, full_refresh: bool = False) -> SyncSummary:
|
||||
cfg = self.config_store.get()
|
||||
now = self._now()
|
||||
summary = SyncSummary(started_at=now)
|
||||
self._log(
|
||||
"info", "folder_sync.start", folders=len(cfg.folders), full=full_refresh
|
||||
)
|
||||
|
||||
for folder in cfg.folders:
|
||||
self.repo.ensure_folder(folder, now=now)
|
||||
self.repo.disable_folders_except(f.media_id for f in cfg.folders)
|
||||
if cfg.up_blacklist_seed:
|
||||
summary.blacklist_seeded = self.repo.seed_blacklist(
|
||||
cfg.up_blacklist_seed, now=now
|
||||
)
|
||||
|
||||
for folder in cfg.folders:
|
||||
try:
|
||||
result = self._sync_folder(cfg, folder, now=now, force_full=full_refresh)
|
||||
except Exception as exc: # noqa: BLE001 单个夹失败不阻塞其它夹
|
||||
result = FolderSyncResult(
|
||||
media_id=folder.media_id,
|
||||
year=folder.year,
|
||||
title=folder.title,
|
||||
error=str(exc),
|
||||
)
|
||||
self._log(
|
||||
"error",
|
||||
"folder_sync.folder_error",
|
||||
media_id=folder.media_id,
|
||||
error=str(exc),
|
||||
)
|
||||
else:
|
||||
self._log("info", "folder_sync.folder", **result.as_dict())
|
||||
if result.warning:
|
||||
self._log(
|
||||
"warning",
|
||||
"folder_sync.capacity",
|
||||
media_id=folder.media_id,
|
||||
warning=result.warning,
|
||||
)
|
||||
summary.folders.append(result)
|
||||
|
||||
summary.finished_at = self._now()
|
||||
self._log(
|
||||
"info" if summary.ok else "error",
|
||||
"folder_sync.done",
|
||||
added=summary.added,
|
||||
removed=summary.removed,
|
||||
refreshed=summary.refreshed,
|
||||
missing=summary.missing,
|
||||
errors=len(summary.errors),
|
||||
elapsed=max(0, summary.finished_at - summary.started_at),
|
||||
)
|
||||
return summary
|
||||
|
||||
def _sync_folder(
|
||||
self,
|
||||
cfg: AppConfig,
|
||||
folder: FolderConfig,
|
||||
*,
|
||||
now: int,
|
||||
force_full: bool,
|
||||
) -> FolderSyncResult:
|
||||
row = self.repo.get_folder(folder.media_id)
|
||||
full_due = (
|
||||
force_full
|
||||
or row is None
|
||||
or row.last_full_at is None
|
||||
or now - int(row.last_full_at)
|
||||
>= cfg.schedule.full_folder_refresh_hours * 3600
|
||||
)
|
||||
|
||||
resources = self.client.fav_ids(folder.media_id)
|
||||
remote_map = {item.aid: item for item in resources if item.type == 2}
|
||||
remote = set(remote_map)
|
||||
local = self.repo.folder_active_aids(folder.media_id)
|
||||
added = remote - local
|
||||
removed = local - remote
|
||||
|
||||
result = FolderSyncResult(
|
||||
media_id=folder.media_id,
|
||||
year=folder.year,
|
||||
title=folder.title or (row.title if row else ""),
|
||||
remote_count=len(remote),
|
||||
added=len(added),
|
||||
removed=len(removed),
|
||||
full_refresh=full_due,
|
||||
)
|
||||
|
||||
targets = set(added)
|
||||
if full_due:
|
||||
targets |= remote
|
||||
|
||||
if targets:
|
||||
infos = self.client.fav_infos(
|
||||
folder.media_id,
|
||||
sorted(targets),
|
||||
folder_mid=cfg.bilibili.mid,
|
||||
)
|
||||
for aid, resource in infos.items.items():
|
||||
if aid not in remote:
|
||||
continue
|
||||
outcome = self.repo.upsert_from_fav(
|
||||
resource, folder.media_id, was_added=aid in added, now=now
|
||||
)
|
||||
if outcome.moved_folder:
|
||||
self._log(
|
||||
"warning",
|
||||
"folder_sync.multi_folder",
|
||||
aid=aid,
|
||||
previous_folder_id=outcome.previous_folder_id,
|
||||
folder_id=folder.media_id,
|
||||
)
|
||||
for aid in infos.missing:
|
||||
source = remote_map.get(aid)
|
||||
self.repo.mark_missing_resource(
|
||||
aid,
|
||||
folder_id=folder.media_id,
|
||||
bvid=source.bvid if source else None,
|
||||
now=now,
|
||||
)
|
||||
result.refreshed = len(infos.items)
|
||||
result.missing = len(infos.missing)
|
||||
|
||||
for aid in removed:
|
||||
self.repo.mark_resource_removed(folder.media_id, aid, now=now)
|
||||
|
||||
self.repo.touch_folder_resources(folder.media_id, now=now)
|
||||
self.repo.finish_folder_sync(
|
||||
folder,
|
||||
media_count=len(remote),
|
||||
full_refresh=full_due,
|
||||
now=now,
|
||||
)
|
||||
|
||||
if len(remote) >= self.capacity_warn:
|
||||
result.warning = (
|
||||
f"收藏夹「{result.title or folder.media_id}」已有 {len(remote)} 条,"
|
||||
f"达到预警线 {self.capacity_warn};请人工准备下一年度收藏夹并更新配置,"
|
||||
f"系统不会自动换夹"
|
||||
)
|
||||
return result
|
||||
Reference in new issue
Block a user