feat(p0): add favorite folder sync and sqlite data foundation

Implement P0 of the reaction dashboard: config loading with hot reload, SQLite migrations with backups, single-writer queue, read-only Bilibili fav client, folder import/diff sync, invalid-resource handling, blacklist seed, scheduler, and JSON redacted logging. Includes offline unit and transport-level contract tests; all existing and new tests pass.
This commit is contained in:
kairee committed 2026-10-31 18:08:33 +08:00
commit e2cdf997af
39 files changed
+8575

No files matched your search

+2
View File
@@ -0,0 +1,2 @@
# -*- coding: utf-8 -*-
"""Reaction video dashboard backend package."""
+2
View File
@@ -0,0 +1,2 @@
# -*- coding: utf-8 -*-
"""B 站接口访问层。"""
+143
View File
@@ -0,0 +1,143 @@
# -*- coding: utf-8 -*-
"""B 站 Cookie 与请求头工具。
从 ``favsort.py`` 抽取,供现有排序工具和看板共用,避免出现两套不一致的
身份 Cookie / 设备指纹处理逻辑。
"""
from __future__ import annotations
import random
import urllib.parse
import uuid
IDENTITY_KEYS = {
"sessdata",
"bili_jct",
"dedeuserid",
"dedeuserid__ckmd5",
"bili_ticket",
"bili_ticket_expires",
"sid",
"b_lsid",
}
DROP_STALE = {"sid", "b_lsid"} # 会话级字段,刷新时应丢弃旧值
def strip_identity(cookie: str) -> str:
"""剥离账号身份类 cookie,保留 buvid3 / buvid4 / _uuid 等设备指纹。
实测结论:
- ids:完全无 cookie 也能读(连私密收藏夹都返回全量);
- infos:不需要账号身份,但缺设备指纹 cookie 会被 HTTP 412 拦;
- sort:必须带完整 cookie + csrf,不能走这里。
"""
kept = []
for part in (cookie or "").split(";"):
part = part.strip()
if not part:
continue
key = part.split("=", 1)[0].strip().lower()
if key in IDENTITY_KEYS:
continue
kept.append(part)
return "; ".join(kept)
def make_headers(cfg: dict, auth: bool = False) -> dict[str, str]:
"""构造 B 站 Web 请求头;``auth=False`` 时去掉账号身份 cookie。"""
return {
"accept": "*/*",
"accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
"cookie": cfg["cookie"] if auth else strip_identity(cfg["cookie"]),
"dnt": "1",
"origin": "https://www.bilibili.com",
"referer": "https://www.bilibili.com/",
"sec-ch-ua": '"Google Chrome";v="155", "Chromium";v="155", "Not(A:Brand";v="24"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"Windows"',
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-site",
"user-agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36"
),
}
def parse_cookie(cookie: str) -> list[tuple[str, str]]:
"""把 cookie 字符串解析成 [(key, value)],保留原顺序。"""
pairs = []
for part in (cookie or "").split(";"):
part = part.strip()
if not part or "=" not in part:
continue
k, v = part.split("=", 1)
pairs.append((k.strip(), v.strip()))
return pairs
def cookie_value(cookie: str, name: str) -> str:
name = name.lower()
for k, v in parse_cookie(cookie):
if k.lower() == name:
return v
return ""
def cookies_from_set_cookie(headers) -> dict[str, str]:
"""从 Set-Cookie 头里取 name=value(忽略 Path/Domain/Expires 等属性)。"""
pairs = {}
for h in headers or []:
first = h.split(";", 1)[0].strip()
if "=" not in first:
continue
k, v = first.split("=", 1)
pairs[k.strip()] = v.strip()
return pairs
def merge_cookie(old: str, new_pairs: dict[str, str]) -> str:
"""只替换身份类 cookie,其它字段(设备指纹等)原样保留。
新值里没有的身份字段不会被删除(例如服务端没下发新的 sid 时保留旧值),
只有 DROP_STALE 里的会话字段会在刷新时清掉。
"""
new_l = {
k.lower(): (k, v)
for k, v in new_pairs.items()
if k.lower() in IDENTITY_KEYS
}
out, seen = [], set()
for k, v in parse_cookie(old):
kl = k.lower()
if kl in new_l:
nk, nv = new_l[kl]
out.append((nk, nv))
seen.add(kl)
elif kl in DROP_STALE:
continue
else:
out.append((k, v))
seen.add(kl)
for kl, (k, v) in new_l.items():
if kl not in seen:
out.append((k, v))
return "; ".join(f"{k}={v}" for k, v in out)
def gen_uuid() -> str:
"""生成 B 站风格的 _uuid(前端本来就是本地生成的,服务端不下发)。"""
suffix = "".join(random.choices("0123456789ABCDEF", k=7))
return f"{str(uuid.uuid4()).upper()}{suffix}infoc"
def sessdata_expiry(cookie: str) -> int | None:
"""SESSDATA 形如 <值>%2C<过期时间戳>%2C<校验>,取中间的时间戳。"""
value = urllib.parse.unquote(cookie_value(cookie, "SESSDATA"))
parts = value.split(",")
if len(parts) >= 2 and parts[1].isdigit():
return int(parts[1])
return None
+362
View File
@@ -0,0 +1,362 @@
# -*- coding: utf-8 -*-
"""只读 B 站 Web API 客户端(P0 范围)。
P0 只需要收藏夹读取能力:``ids`` / ``infos`` / ``folder/created/list-all``。
所有网络访问都经过 ``Transport`` 边界,测试可直接替换该边界,而不需要访问
真实 B 站。写操作(``deal`` / ``sort``)不在本模块实现。
"""
from __future__ import annotations
import json
import random
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from typing import Any, Mapping, Protocol, Sequence
from .auth import make_headers
from .models import FavId, FavResource, FolderInfo, InfosResult
BASE = "https://api.bilibili.com"
DEFAULT_CHUNK = 200
MAX_CHUNK = 400
class BilibiliError(RuntimeError):
"""B 站访问层基础异常。"""
class TransportError(BilibiliError):
"""网络层错误,可重试。"""
retryable = True
class BilibiliHTTPError(BilibiliError):
def __init__(self, status: int, message: str = "") -> None:
self.status = status
self.message = message
self.retryable = status >= 500 or status == 429
super().__init__(f"HTTP {status}: {message}".rstrip())
def __str__(self) -> str:
return f"HTTP {self.status}: {self.message}".rstrip(": ")
class BilibiliAPIError(BilibiliError):
def __init__(self, code: int, message: str = "") -> None:
self.code = code
self.message = message
super().__init__(f"B 站接口业务错误 code={code}: {message}".rstrip(": "))
def __str__(self) -> str:
return f"B 站接口业务错误 code={self.code}: {self.message}".rstrip(": ")
class BilibiliProtocolError(BilibiliError):
"""响应结构与已知契约不一致。"""
@dataclass(frozen=True)
class HTTPResponse:
status: int
headers: Mapping[str, str]
body: bytes
def json(self) -> Any:
try:
return json.loads(self.body.decode("utf-8", "replace"))
except json.JSONDecodeError as exc:
raise BilibiliProtocolError(f"响应不是合法 JSON: {exc}") from exc
class Transport(Protocol):
def request(
self,
method: str,
url: str,
headers: Mapping[str, str],
data: bytes | None = None,
timeout: float = 30,
) -> HTTPResponse: ...
class UrllibTransport:
"""基于 ``urllib.request`` 的默认传输实现。"""
def request(
self,
method: str,
url: str,
headers: Mapping[str, str],
data: bytes | None = None,
timeout: float = 30,
) -> HTTPResponse:
req = urllib.request.Request(
url, data=data, headers=dict(headers), method=method
)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
return HTTPResponse(
status=int(resp.status),
headers=dict(resp.headers.items()),
body=resp.read(),
)
except urllib.error.HTTPError as exc:
try:
body = exc.read()
except Exception: # noqa: BLE001
body = b""
return HTTPResponse(
status=int(exc.code),
headers=dict(exc.headers.items()) if exc.headers else {},
body=body,
)
except (urllib.error.URLError, TimeoutError, OSError) as exc:
raise TransportError(str(exc)) from exc
class Pacer:
"""请求间随机等待;第一个请求不等待。"""
def __init__(
self,
min_delay: float = 1.5,
max_delay: float = 2.5,
*,
sleep=time.sleep,
rng: random.Random | None = None,
) -> None:
if min_delay < 0 or max_delay < min_delay:
raise ValueError("非法的请求间隔")
self.min_delay = min_delay
self.max_delay = max_delay
self._sleep = sleep
self._rng = rng or random.Random()
self._first = True
def wait(self) -> float:
if self._first:
self._first = False
return 0.0
delay = self._rng.uniform(self.min_delay, self.max_delay)
if delay > 0:
self._sleep(delay)
return delay
def _chunks(seq: Sequence[int], size: int):
for index in range(0, len(seq), size):
yield seq[index:index + size]
class BilibiliClient:
"""只读收藏夹 API 客户端。"""
def __init__(
self,
*,
cookie: str = "",
csrf: str = "",
mid: int = 0,
transport: Transport | None = None,
pacer: Pacer | None = None,
sleeper=time.sleep,
retries: int = 3,
backoff: float = 1.0,
timeout: float = 30,
logger=None,
) -> None:
if retries < 1:
raise ValueError("retries 必须 >= 1")
self.cookie = cookie or ""
self.csrf = csrf or ""
self.mid = mid
self.transport = transport or UrllibTransport()
self.pacer = pacer or Pacer(sleep=sleeper)
self._sleeper = sleeper
self.retries = retries
self.backoff = backoff
self.timeout = timeout
self.logger = logger
# ------------------------------------------------------------ core
def _headers(self, auth: bool) -> dict[str, str]:
return make_headers({"cookie": self.cookie}, auth=auth)
def _request_json(self, url: str, *, auth: bool) -> dict[str, Any]:
self.pacer.wait()
resp = self.transport.request(
"GET", url, self._headers(auth), data=None, timeout=self.timeout
)
if resp.status >= 400:
raise BilibiliHTTPError(resp.status, resp.body.decode("utf-8", "replace")[:200])
payload = resp.json()
if not isinstance(payload, dict):
raise BilibiliProtocolError("接口响应顶层不是 JSON 对象")
return payload
def _request_with_retry(self, url: str, *, auth: bool) -> dict[str, Any]:
last: BilibiliError | None = None
for attempt in range(self.retries):
try:
return self._request_json(url, auth=auth)
except (TransportError, BilibiliHTTPError) as exc:
last = exc
if not exc.retryable or attempt >= self.retries - 1:
raise
delay = self.backoff * (2 ** attempt)
if self.logger is not None:
self.logger.warning(
"bili.retry",
attempt=attempt + 1,
delay=delay,
status=getattr(exc, "status", None),
)
if delay > 0:
self._sleeper(delay)
raise last or TransportError("未知网络错误")
@staticmethod
def _api_error(payload: Mapping[str, Any]) -> BilibiliAPIError:
code = payload.get("code")
if isinstance(code, bool) or not isinstance(code, int):
raise BilibiliProtocolError("接口响应缺少整数 code")
return BilibiliAPIError(code, str(payload.get("message") or ""))
def get_json(self, url: str, *, auth: bool = False) -> dict[str, Any]:
"""GET JSON;匿名失败时自动带完整 Cookie 重试一次(与 favsort 一致)。"""
if auth:
payload = self._request_with_retry(url, auth=True)
if payload.get("code") != 0:
raise self._api_error(payload)
return payload
anon_error: BilibiliError | None = None
try:
payload = self._request_with_retry(url, auth=False)
if payload.get("code") == 0:
return payload
anon_error = self._api_error(payload)
except BilibiliError as exc:
anon_error = exc
if self.logger is not None and anon_error is not None:
self.logger.debug("bili.anon_fallback", error=str(anon_error))
payload = self._request_with_retry(url, auth=True)
if payload.get("code") != 0:
raise self._api_error(payload)
return payload
# ------------------------------------------------------------ fav
def fav_ids(self, media_id: int) -> list[FavId]:
url = (
f"{BASE}/x/v3/fav/resource/ids?media_id={int(media_id)}&platform=web"
)
payload = self.get_json(url)
data = payload.get("data")
if data is None:
data = []
if not isinstance(data, list):
raise BilibiliProtocolError("fav/resource/ids data 不是数组")
out: list[FavId] = []
for index, item in enumerate(data):
if not isinstance(item, Mapping):
raise BilibiliProtocolError(f"fav/resource/ids data[{index}] 不是对象")
aid = item.get("id")
if isinstance(aid, bool) or not isinstance(aid, int):
raise BilibiliProtocolError(f"fav/resource/ids data[{index}].id 不是整数")
raw_type = item.get("type", 2)
if isinstance(raw_type, bool) or not isinstance(raw_type, int):
raise BilibiliProtocolError(f"fav/resource/ids data[{index}].type 不是整数")
bvid = item.get("bvid") or item.get("bv_id")
if not isinstance(bvid, str) or not bvid:
bvid = None
out.append(FavId(aid=aid, type=raw_type, bvid=bvid))
return out
def fav_infos(
self,
media_id: int,
aids: Sequence[int],
*,
folder_mid: int | None = None,
chunk_size: int = DEFAULT_CHUNK,
) -> InfosResult:
if not 1 <= chunk_size <= MAX_CHUNK:
raise ValueError(f"chunk_size 必须在 1..{MAX_CHUNK} 之间")
requested: list[int] = []
seen: set[int] = set()
for raw in aids:
aid = int(raw)
if aid not in seen:
seen.add(aid)
requested.append(aid)
if not requested:
return InfosResult(requested=(), items={}, missing=())
fmid = self.mid if folder_mid is None else int(folder_mid)
items: dict[int, FavResource] = {}
for chunk in _chunks(requested, chunk_size):
resources = ",".join(f"{aid}:2" for aid in chunk)
url = (
f"{BASE}/x/v3/fav/resource/infos"
f"?resources={urllib.parse.quote(resources)}"
f"&platform=web&folder_mid={fmid}&folder_id={int(media_id)}"
)
payload = self.get_json(url)
data = payload.get("data")
if data is None:
data = []
if not isinstance(data, list):
raise BilibiliProtocolError("fav/resource/infos data 不是数组")
chunk_set = set(chunk)
for item in data:
if not isinstance(item, Mapping):
raise BilibiliProtocolError("fav/resource/infos item 不是对象")
try:
parsed = FavResource.from_api(item)
except (TypeError, ValueError) as exc:
raise BilibiliProtocolError(f"fav/resource/infos 解析失败: {exc}") from exc
if parsed.aid not in chunk_set:
raise BilibiliProtocolError(
f"fav/resource/infos 返回了未请求的 aid={parsed.aid}"
)
items[parsed.aid] = parsed
missing = tuple(aid for aid in requested if aid not in items)
return InfosResult(requested=tuple(requested), items=items, missing=missing)
def fav_folders(self, up_mid: int | None = None) -> list[FolderInfo]:
mid = self.mid if up_mid is None else int(up_mid)
url = f"{BASE}/x/v3/fav/folder/created/list-all?up_mid={mid}&jsonp=jsonp"
payload = self.get_json(url, auth=True)
data = payload.get("data") or {}
if not isinstance(data, Mapping):
raise BilibiliProtocolError("fav/folder/created/list-all data 不是对象")
rows = data.get("list")
if rows is None:
return []
if not isinstance(rows, list):
raise BilibiliProtocolError("fav/folder/created/list-all data.list 不是数组")
out: list[FolderInfo] = []
for index, row in enumerate(rows):
if not isinstance(row, Mapping):
raise BilibiliProtocolError(f"folder list[{index}] 不是对象")
media_id = row.get("id")
if isinstance(media_id, bool) or not isinstance(media_id, int):
raise BilibiliProtocolError(f"folder list[{index}].id 不是整数")
count = row.get("media_count", 0)
if isinstance(count, bool) or not isinstance(count, int):
count = 0
out.append(FolderInfo(
media_id=media_id,
title=str(row.get("title") or ""),
media_count=count,
))
return out
+87
View File
@@ -0,0 +1,87 @@
# -*- coding: utf-8 -*-
"""B 站收藏夹接口的返回模型。"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Mapping
@dataclass(frozen=True)
class FavId:
"""``fav/resource/ids`` 的一条资源。"""
aid: int
type: int
bvid: str | None = None
@dataclass(frozen=True)
class FavResource:
"""``fav/resource/infos`` 的一条视频元数据。"""
aid: int
bvid: str | None
title: str
up_mid: int | None
up_name: str | None
pubdate: int
duration: int | None
cover: str | None
attr: int
fav_time: int | None
@property
def invalid(self) -> bool:
# 实测 attr=9 表示"已失效视频"。
return self.attr == 9
@classmethod
def from_api(cls, item: Mapping[str, Any]) -> "FavResource":
aid = item.get("id")
if isinstance(aid, bool) or not isinstance(aid, int):
raise ValueError("infos item.id 不是整数")
upper = item.get("upper") or {}
if not isinstance(upper, Mapping):
upper = {}
up_mid = upper.get("mid")
if isinstance(up_mid, bool) or not isinstance(up_mid, int):
up_mid = None
raw_duration = item.get("duration")
duration = raw_duration if isinstance(raw_duration, int) and not isinstance(
raw_duration, bool
) else None
raw_attr = item.get("attr", 0)
attr = raw_attr if isinstance(raw_attr, int) and not isinstance(raw_attr, bool) else 0
fav_time = item.get("fav_time")
if isinstance(fav_time, bool) or not isinstance(fav_time, int) or fav_time <= 0:
fav_time = None
bvid = item.get("bvid") or item.get("bv_id")
if not isinstance(bvid, str) or not bvid:
bvid = None
return cls(
aid=aid,
bvid=bvid,
title=str(item.get("title") or ""),
up_mid=up_mid,
up_name=str(upper.get("name") or "") or None,
pubdate=int(item.get("pubtime") or item.get("ctime") or 0),
duration=duration,
cover=str(item.get("cover") or "") or None,
attr=attr,
fav_time=fav_time,
)
@dataclass(frozen=True)
class FolderInfo:
media_id: int
title: str
media_count: int
@dataclass(frozen=True)
class InfosResult:
requested: tuple[int, ...]
items: dict[int, FavResource]
missing: tuple[int, ...]
+217
View File
@@ -0,0 +1,217 @@
# -*- coding: utf-8 -*-
"""P0 命令行入口。
python app.py init # 建库 / 迁移 / 写入黑名单种子(离线)
python app.py sync # 收藏夹只读同步(首次导入或增量)
python app.py run # 按 folder_sync_minutes 常驻周期同步(--once 只跑一次)
python app.py status # 查看库内状态(离线)
P2 起 ``python app.py`` 将在此基础上启动本地看板服务。
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Sequence
from .bilibili.client import BilibiliClient
from .config import ConfigError, ConfigStore
from .database import Database
from .logging_json import JsonLogger
from .repository import Repository
from .scheduler import ScheduledTask, Scheduler
from .services.folder_sync import FolderSyncService
ROOT = Path(__file__).resolve().parent.parent
DEFAULT_CONFIG = ROOT / "config.json"
def _build_parser() -> argparse.ArgumentParser:
common = argparse.ArgumentParser(add_help=False)
common.add_argument(
"--config",
default=str(DEFAULT_CONFIG),
help="配置文件路径(默认项目根目录 config.json)",
)
parser = argparse.ArgumentParser(
prog="app.py",
description="咻咻满 Reaction 数据看板 —— P0 数据底座",
)
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("init", parents=[common], help="建库、迁移并写入黑名单种子")
sync = sub.add_parser("sync", parents=[common], help="同步已配置的年度收藏夹")
sync.add_argument(
"--full", action="store_true", help="强制全量刷新元数据(忽略 6 小时周期)"
)
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行收藏夹同步")
run.add_argument(
"--once", action="store_true", help="只执行一次到期任务后退出"
)
sub.add_parser("status", parents=[common], help="查看数据库状态(离线)")
return parser
def _open_db(cfg, logger: JsonLogger) -> Database:
db = Database(
cfg.storage.database_path,
backup_dir=cfg.storage.backup_dir,
backup_keep_days=cfg.storage.backup_keep_days,
logger=logger,
)
db.initialize()
return db
def _print(payload) -> None:
print(json.dumps(payload, ensure_ascii=False, indent=2), flush=True)
def cmd_init(args, logger: JsonLogger) -> int:
store = ConfigStore(args.config, require_auth=False)
cfg = store.get()
db = _open_db(cfg, logger)
try:
repo = Repository(db)
for folder in cfg.folders:
repo.ensure_folder(folder)
seeded = repo.seed_blacklist(cfg.up_blacklist_seed)
_print({
"ok": True,
"schema_version": db.schema_version,
"database": str(cfg.storage.database_path),
"blacklist_seeded": seeded,
**repo.counts(),
})
finally:
db.close()
return 0
def cmd_sync(args, logger: JsonLogger) -> int:
store = ConfigStore(args.config, require_auth=True)
cfg = store.get()
db = _open_db(cfg, logger)
try:
repo = Repository(db)
client = BilibiliClient(
cookie=cfg.bilibili.cookie,
csrf=cfg.bilibili.csrf,
mid=cfg.bilibili.mid,
logger=logger,
)
service = FolderSyncService(store, repo, client, logger=logger)
summary = service.run(full_refresh=args.full)
payload = summary.as_dict()
payload["database"] = str(cfg.storage.database_path)
payload["counts"] = repo.counts()
_print(payload)
return 0 if summary.ok else 1
finally:
db.close()
def cmd_run(args, logger: JsonLogger) -> int:
store = ConfigStore(args.config, require_auth=True)
cfg = store.get()
db = _open_db(cfg, logger)
last: dict[str, object] = {}
try:
repo = Repository(db)
client = BilibiliClient(
cookie=cfg.bilibili.cookie,
csrf=cfg.bilibili.csrf,
mid=cfg.bilibili.mid,
logger=logger,
)
service = FolderSyncService(store, repo, client, logger=logger)
def run_sync():
summary = service.run()
last["summary"] = summary
logger.info("folder_sync.summary", **summary.as_dict())
return summary
scheduler = Scheduler(
[ScheduledTask(
"folder_sync",
float(cfg.schedule.folder_sync_minutes * 60),
run_sync,
)],
logger=logger,
)
if args.once:
results = scheduler.run_due()
_print([result.as_dict() for result in results])
else:
logger.info(
"scheduler.start",
interval_seconds=cfg.schedule.folder_sync_minutes * 60,
)
scheduler.run_forever()
summary = last.get("summary")
return 0 if summary is None or getattr(summary, "ok", True) else 1
finally:
db.close()
def cmd_status(args, logger: JsonLogger) -> int:
store = ConfigStore(args.config, require_auth=False)
cfg = store.get()
db = _open_db(cfg, logger)
try:
repo = Repository(db)
_print({
"ok": True,
"schema_version": db.schema_version,
"database": str(cfg.storage.database_path),
"configured_folders": [
{
"media_id": row.media_id,
"year": row.year,
"title": row.title,
"media_count": row.media_count,
"last_ids_at": row.last_ids_at,
"last_full_at": row.last_full_at,
"enabled": row.enabled,
}
for row in repo.list_folders()
],
**repo.counts(),
})
finally:
db.close()
return 0
def main(argv: Sequence[str] | None = None) -> int:
args = _build_parser().parse_args(list(argv) if argv is not None else None)
logger = JsonLogger()
try:
if args.command == "init":
return cmd_init(args, logger)
if args.command == "sync":
return cmd_sync(args, logger)
if args.command == "run":
return cmd_run(args, logger)
if args.command == "status":
return cmd_status(args, logger)
except ConfigError as exc:
logger.error("config.error", error=str(exc))
return 2
except KeyboardInterrupt:
logger.warning("app.interrupted")
return 130
except Exception as exc: # noqa: BLE001 CLI 边界统一记录
logger.error("app.error", error=str(exc))
return 2
logger.error("app.unknown_command", command=getattr(args, "command", None))
return 2
if __name__ == "__main__":
sys.exit(main())
+495
View File
@@ -0,0 +1,495 @@
# -*- coding: utf-8 -*-
"""运行期配置加载、校验与热加载。
支持两种写法:
* 新格式:``{"bilibili": {"mid": ..., "cookie": ...}, ...}``;
* 旧 ``favsort.py`` 格式:顶层 ``mid`` / ``cookie`` / ``csrf``。
旧的扁平字段仍然被读取,因此 ``bili-fav auth`` 写回的 config.json 无需迁移即可
被看板使用。敏感值只保存在配置对象中,日志输出必须先经过 ``logging_json`` 脱敏。
"""
from __future__ import annotations
import copy
import json
import os
import re
import threading
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Mapping
DEFAULT_CLASSIFY: dict[str, Any] = {
"require_title": "咻咻满",
"positive": [
"reaction", "re", "反应", "看", "听", "初听", "首次",
"点评", "锐评", "路人", "震惊", "双人",
],
"charge": ["充电", "加急", "专场"],
"negative": ["歌切", "切片", "纯享", "搬运", "完整版"],
"duration_boost": {"gt_30min": 1, "gt_60min": 2},
}
DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
"view": 0.30,
"like": 0.20,
"coin": 0.20,
"favorite": 0.15,
"share": 0.10,
"reply": 0.03,
"danmaku": 0.02,
}
DEFAULT_CONFIG: dict[str, Any] = {
"bilibili": {"mid": 0, "csrf": "", "cookie": ""},
"telegram": {"enabled": False, "bot_token": "", "chat_id": 0},
"web": {
"host": "127.0.0.1",
"port": 8080,
"auth_token": "",
"public_base_url": "",
},
"storage": {
"database_path": "data/app.db",
"backup_dir": "data/backup",
"backup_keep_days": 14,
},
"folders": [],
"track_fans": [],
"up_blacklist_seed": [],
"schedule": {
"discover_minutes": 60,
"folder_sync_minutes": 60,
"full_folder_refresh_hours": 6,
"snapshot_at": "00:01",
"candidate_ttl_days": 7,
},
"notify": {"min_score": 2, "max_items_per_message": 10},
"classify": DEFAULT_CLASSIFY,
"heat_weights": DEFAULT_HEAT_WEIGHTS,
}
# 环境变量只作为 CI / 临时调试的覆盖入口,不作为主路径。
ENV_OVERRIDES: dict[str, tuple[tuple[str, ...], type]] = {
"BILI_COOKIE": (("bilibili", "cookie"), str),
"BILI_CSRF": (("bilibili", "csrf"), str),
"BILI_MID": (("bilibili", "mid"), int),
"TG_BOT_TOKEN": (("telegram", "bot_token"), str),
"TG_CHAT_ID": (("telegram", "chat_id"), int),
"DASHBOARD_AUTH_TOKEN": (("web", "auth_token"), str),
"DATABASE_PATH": (("storage", "database_path"), str),
}
_SNAPSHOT_AT_RE = re.compile(r"^([01]\d|2[0-3]):[0-5]\d$")
_HEAT_METRICS = tuple(DEFAULT_HEAT_WEIGHTS)
class ConfigError(ValueError):
"""配置文件缺失、格式错误或字段非法。"""
@dataclass(frozen=True)
class BilibiliConfig:
mid: int
csrf: str
cookie: str
@dataclass(frozen=True)
class TelegramConfig:
enabled: bool
bot_token: str
chat_id: int
@dataclass(frozen=True)
class WebConfig:
host: str
port: int
auth_token: str
public_base_url: str
@dataclass(frozen=True)
class StorageConfig:
database_path: Path
backup_dir: Path
backup_keep_days: int
@dataclass(frozen=True)
class FolderConfig:
year: int
media_id: int
title: str = ""
@dataclass(frozen=True)
class ScheduleConfig:
discover_minutes: int
folder_sync_minutes: int
full_folder_refresh_hours: int
snapshot_at: str
candidate_ttl_days: int
@dataclass(frozen=True)
class NotifyConfig:
min_score: int
max_items_per_message: int
@dataclass(frozen=True)
class AppConfig:
path: Path
bilibili: BilibiliConfig
telegram: TelegramConfig
web: WebConfig
storage: StorageConfig
folders: tuple[FolderConfig, ...]
track_fans: tuple[int, ...]
up_blacklist_seed: tuple[int, ...]
schedule: ScheduleConfig
notify: NotifyConfig
classify: Mapping[str, Any]
heat_weights: Mapping[str, float]
raw: Mapping[str, Any] = field(repr=False, compare=False)
def folder_for_year(self, year: int) -> FolderConfig | None:
for folder in self.folders:
if folder.year == year:
return folder
return None
def latest_folder(self) -> FolderConfig | None:
if not self.folders:
return None
return max(self.folders, key=lambda f: f.year)
def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> dict[str, Any]:
out = copy.deepcopy(dict(base))
for key, value in override.items():
if isinstance(value, Mapping) and isinstance(out.get(key), Mapping):
out[key] = _deep_merge(out[key], value)
else:
out[key] = copy.deepcopy(value)
return out
def _get_path(data: Mapping[str, Any], path: tuple[str, ...]) -> Any:
cur: Any = data
for key in path:
if not isinstance(cur, Mapping) or key not in cur:
return None
cur = cur[key]
return cur
def _set_path(data: dict[str, Any], path: tuple[str, ...], value: Any) -> None:
cur = data
for key in path[:-1]:
cur = cur.setdefault(key, {})
cur[path[-1]] = value
def _as_int(value: Any, name: str, *, minimum: int | None = None,
maximum: int | None = None) -> int:
if isinstance(value, bool) or not isinstance(value, int):
raise ConfigError(f"{name} 必须是整数")
if minimum is not None and value < minimum:
raise ConfigError(f"{name} 不能小于 {minimum}")
if maximum is not None and value > maximum:
raise ConfigError(f"{name} 不能大于 {maximum}")
return value
def _as_str(value: Any, name: str) -> str:
if not isinstance(value, str):
raise ConfigError(f"{name} 必须是字符串")
return value
def _as_int_list(value: Any, name: str) -> tuple[int, ...]:
if not isinstance(value, list):
raise ConfigError(f"{name} 必须是数组")
out: list[int] = []
for item in value:
mid = _as_int(item, f"{name}[]", minimum=1)
if mid not in out:
out.append(mid)
return tuple(out)
def _as_str_list(value: Any, name: str) -> list[str]:
if not isinstance(value, list):
raise ConfigError(f"{name} 必须是数组")
out = []
for item in value:
if not isinstance(item, str) or not item:
raise ConfigError(f"{name}[] 必须是非空字符串")
out.append(item)
return out
def _legacy_to_current(raw: Mapping[str, Any]) -> dict[str, Any]:
"""把 favsort 的扁平字段映射到新格式,不修改调用方对象。"""
data = copy.deepcopy(dict(raw))
if "bilibili" not in data:
data["bilibili"] = {
"mid": raw.get("mid", 0),
"csrf": raw.get("csrf", ""),
"cookie": raw.get("cookie", ""),
}
return data
def _validate_folders(value: Any) -> tuple[FolderConfig, ...]:
if not isinstance(value, list):
raise ConfigError("folders 必须是数组")
folders: list[FolderConfig] = []
seen_ids: set[int] = set()
seen_years: set[int] = set()
for index, item in enumerate(value):
name = f"folders[{index}]"
if not isinstance(item, Mapping):
raise ConfigError(f"{name} 必须是对象")
year = _as_int(item.get("year"), f"{name}.year", minimum=1900, maximum=9999)
media_id = _as_int(item.get("media_id"), f"{name}.media_id", minimum=1)
title = _as_str(item.get("title", "") or "", f"{name}.title")
if media_id in seen_ids:
raise ConfigError(f"{name}.media_id 重复: {media_id}")
if year in seen_years:
raise ConfigError(f"{name}.year 重复: {year}")
seen_ids.add(media_id)
seen_years.add(year)
folders.append(FolderConfig(year=year, media_id=media_id, title=title))
return tuple(folders)
def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
b = data["bilibili"]
if not isinstance(b, Mapping):
raise ConfigError("bilibili 必须是对象")
bilibili = BilibiliConfig(
mid=_as_int(b.get("mid", 0), "bilibili.mid", minimum=0),
csrf=_as_str(b.get("csrf", "") or "", "bilibili.csrf"),
cookie=_as_str(b.get("cookie", "") or "", "bilibili.cookie"),
)
tg = data["telegram"]
if not isinstance(tg, Mapping):
raise ConfigError("telegram 必须是对象")
telegram = TelegramConfig(
enabled=bool(tg.get("enabled", False)),
bot_token=_as_str(tg.get("bot_token", "") or "", "telegram.bot_token"),
chat_id=_as_int(tg.get("chat_id", 0), "telegram.chat_id"),
)
web_raw = data["web"]
if not isinstance(web_raw, Mapping):
raise ConfigError("web 必须是对象")
web = WebConfig(
host=_as_str(web_raw.get("host", "127.0.0.1"), "web.host"),
port=_as_int(web_raw.get("port", 8080), "web.port", minimum=1, maximum=65535),
auth_token=_as_str(web_raw.get("auth_token", "") or "", "web.auth_token"),
public_base_url=_as_str(
web_raw.get("public_base_url", "") or "", "web.public_base_url"
),
)
storage_raw = data["storage"]
if not isinstance(storage_raw, Mapping):
raise ConfigError("storage 必须是对象")
base_dir = path.parent
database_path = Path(_as_str(
storage_raw.get("database_path", "data/app.db") or "data/app.db",
"storage.database_path",
))
backup_dir = Path(_as_str(
storage_raw.get("backup_dir", "data/backup") or "data/backup",
"storage.backup_dir",
))
if not database_path.is_absolute():
database_path = base_dir / database_path
if not backup_dir.is_absolute():
backup_dir = base_dir / backup_dir
storage = StorageConfig(
database_path=database_path,
backup_dir=backup_dir,
backup_keep_days=_as_int(
storage_raw.get("backup_keep_days", 14),
"storage.backup_keep_days",
minimum=1,
),
)
sched_raw = data["schedule"]
if not isinstance(sched_raw, Mapping):
raise ConfigError("schedule 必须是对象")
snapshot_at = _as_str(sched_raw.get("snapshot_at", "00:01"), "schedule.snapshot_at")
if not _SNAPSHOT_AT_RE.match(snapshot_at):
raise ConfigError("schedule.snapshot_at 必须是 HH:MM(24 小时制)")
schedule = ScheduleConfig(
discover_minutes=_as_int(
sched_raw.get("discover_minutes", 60), "schedule.discover_minutes", minimum=1
),
folder_sync_minutes=_as_int(
sched_raw.get("folder_sync_minutes", 60),
"schedule.folder_sync_minutes",
minimum=1,
),
full_folder_refresh_hours=_as_int(
sched_raw.get("full_folder_refresh_hours", 6),
"schedule.full_folder_refresh_hours",
minimum=1,
),
snapshot_at=snapshot_at,
candidate_ttl_days=_as_int(
sched_raw.get("candidate_ttl_days", 7),
"schedule.candidate_ttl_days",
minimum=1,
),
)
notify_raw = data["notify"]
if not isinstance(notify_raw, Mapping):
raise ConfigError("notify 必须是对象")
notify = NotifyConfig(
min_score=_as_int(notify_raw.get("min_score", 2), "notify.min_score"),
max_items_per_message=_as_int(
notify_raw.get("max_items_per_message", 10),
"notify.max_items_per_message",
minimum=1,
),
)
classify_raw = data["classify"]
if not isinstance(classify_raw, Mapping):
raise ConfigError("classify 必须是对象")
classify = copy.deepcopy(dict(classify_raw))
classify["require_title"] = _as_str(
classify.get("require_title", "咻咻满"), "classify.require_title"
)
if not classify["require_title"]:
raise ConfigError("classify.require_title 不能为空")
for key in ("positive", "charge", "negative"):
classify[key] = _as_str_list(classify.get(key, []), f"classify.{key}")
boost = classify.get("duration_boost", {})
if not isinstance(boost, Mapping):
raise ConfigError("classify.duration_boost 必须是对象")
classify["duration_boost"] = {
"gt_30min": _as_int(boost.get("gt_30min", 1), "classify.duration_boost.gt_30min"),
"gt_60min": _as_int(boost.get("gt_60min", 2), "classify.duration_boost.gt_60min"),
}
heat_raw = data["heat_weights"]
if not isinstance(heat_raw, Mapping):
raise ConfigError("heat_weights 必须是对象")
heat: dict[str, float] = {}
for metric in _HEAT_METRICS:
value = heat_raw.get(metric, DEFAULT_HEAT_WEIGHTS[metric])
if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0:
raise ConfigError(f"heat_weights.{metric} 必须是非负数字")
heat[metric] = float(value)
return AppConfig(
path=path,
bilibili=bilibili,
telegram=telegram,
web=web,
storage=storage,
folders=_validate_folders(data["folders"]),
track_fans=_as_int_list(data["track_fans"], "track_fans"),
up_blacklist_seed=_as_int_list(data["up_blacklist_seed"], "up_blacklist_seed"),
schedule=schedule,
notify=notify,
classify=classify,
heat_weights=heat,
raw=data,
)
def load_config(
path: str | os.PathLike[str],
*,
env: Mapping[str, str] | None = None,
require_auth: bool = False,
) -> AppConfig:
"""读取并校验配置。``env`` 默认为 ``os.environ``。"""
config_path = Path(path)
if not config_path.exists():
raise ConfigError(f"缺少配置文件: {config_path}")
try:
with open(config_path, encoding="utf-8") as fh:
raw = json.load(fh)
except json.JSONDecodeError as exc:
raise ConfigError(f"配置文件不是合法 JSON: {config_path}: {exc}") from exc
except OSError as exc:
raise ConfigError(f"无法读取配置文件: {config_path}: {exc}") from exc
if not isinstance(raw, Mapping):
raise ConfigError("配置文件顶层必须是 JSON 对象")
data = _deep_merge(DEFAULT_CONFIG, _legacy_to_current(raw))
env_map = os.environ if env is None else env
for env_name, (target, caster) in ENV_OVERRIDES.items():
if env_name not in env_map:
continue
value: Any = env_map[env_name]
if caster is int:
try:
value = int(value)
except (TypeError, ValueError) as exc:
raise ConfigError(f"环境变量 {env_name} 必须是整数") from exc
_set_path(data, target, value)
cfg = _build_config(data, config_path)
if require_auth and (not cfg.bilibili.cookie or not cfg.bilibili.csrf):
raise ConfigError(
"config.json 中的 cookie / csrf 不能为空(可先运行 bili-fav auth)"
)
return cfg
class ConfigStore:
"""按文件 mtime 热加载配置;坏配置不会覆盖上一份可用配置。"""
def __init__(
self,
path: str | os.PathLike[str],
*,
env: Mapping[str, str] | None = None,
require_auth: bool = False,
) -> None:
self.path = Path(path)
self._env = env
self._require_auth = require_auth
self._lock = threading.RLock()
self._config = load_config(path, env=env, require_auth=require_auth)
self._mtime_ns = self._read_mtime_ns()
self.last_error: str | None = None
def _read_mtime_ns(self) -> int | None:
try:
return self.path.stat().st_mtime_ns
except OSError:
return None
def get(self) -> AppConfig:
with self._lock:
mtime_ns = self._read_mtime_ns()
if mtime_ns is None or mtime_ns == self._mtime_ns:
return self._config
self._mtime_ns = mtime_ns
try:
self._config = load_config(
self.path, env=self._env, require_auth=self._require_auth
)
except ConfigError as exc:
self.last_error = str(exc)
return self._config
self.last_error = None
return self._config
+175
View File
@@ -0,0 +1,175 @@
# -*- coding: utf-8 -*-
"""SQLite 连接、迁移入口与单写队列。
所有写操作经单个后台线程串行执行,避免并发任务出现 ``database is locked``;
读操作使用短生命周期连接。网络请求绝不在写事务内执行。
"""
from __future__ import annotations
import queue
import sqlite3
import threading
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable, TypeVar
from .migrations import current_version, run_migrations
T = TypeVar("T")
class DatabaseClosed(RuntimeError):
pass
@dataclass
class _Job:
fn: Callable[[sqlite3.Connection], Any]
done: threading.Event = field(default_factory=threading.Event)
result: Any = None
error: BaseException | None = None
def _configure(conn: sqlite3.Connection) -> sqlite3.Connection:
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys = ON")
conn.execute("PRAGMA busy_timeout = 5000")
conn.execute("PRAGMA synchronous = NORMAL")
conn.execute("PRAGMA journal_mode = WAL")
return conn
class Database:
"""线程安全的 SQLite 访问入口。"""
def __init__(
self,
path: str | Path,
*,
backup_dir: str | Path | None = None,
backup_keep_days: int = 14,
logger=None,
) -> None:
self.path = Path(path)
self.backup_dir = Path(backup_dir) if backup_dir is not None else None
self.backup_keep_days = backup_keep_days
self.logger = logger
self.schema_version: int | None = None
self._queue: queue.Queue[_Job | None] = queue.Queue()
self._thread: threading.Thread | None = None
self._ready = threading.Event()
self._writer_error: BaseException | None = None
self._closed = False
# ------------------------------------------------------------ setup
def _connect(self) -> sqlite3.Connection:
# isolation_level=None:单条 DML 立即提交;显式事务由 transaction() 管理。
return _configure(
sqlite3.connect(self.path, timeout=5.0, isolation_level=None)
)
def initialize(self) -> list[int]:
"""创建目录、执行迁移并启动单写线程。"""
self.path.parent.mkdir(parents=True, exist_ok=True)
conn = self._connect()
try:
applied = run_migrations(
conn,
backup_dir=self.backup_dir,
backup_keep_days=self.backup_keep_days,
logger=self.logger,
)
self.schema_version = current_version(conn)
finally:
conn.close()
self._start_writer()
if self.logger is not None:
self.logger.info(
"db.ready", path=str(self.path), schema_version=self.schema_version
)
return applied
def _start_writer(self) -> None:
if self._thread is not None:
return
self._thread = threading.Thread(target=self._writer_loop, name="sqlite-writer", daemon=True)
self._thread.start()
self._ready.wait(timeout=10)
if self._writer_error is not None:
raise RuntimeError(f"SQLite 写线程启动失败: {self._writer_error}") from self._writer_error
def _writer_loop(self) -> None:
conn: sqlite3.Connection | None = None
try:
conn = self._connect()
except BaseException as exc: # noqa: BLE001
self._writer_error = exc
self._ready.set()
return
self._ready.set()
try:
while True:
job = self._queue.get()
if job is None:
break
try:
job.result = job.fn(conn)
except BaseException as exc: # noqa: BLE001 原样返回给调用方
job.error = exc
finally:
job.done.set()
finally:
conn.close()
# ------------------------------------------------------------ access
def write(self, fn: Callable[[sqlite3.Connection], T]) -> T:
"""把函数交给单写线程执行,并同步等待结果。"""
if self._closed:
raise DatabaseClosed("数据库已关闭")
if self._thread is None:
raise RuntimeError("请先调用 initialize()")
job = _Job(fn=fn)
self._queue.put(job)
job.done.wait()
if job.error is not None:
raise job.error
return job.result
def transaction(self, fn: Callable[[sqlite3.Connection], T]) -> T:
"""在单写线程内执行一个显式事务。"""
def run(conn: sqlite3.Connection) -> T:
conn.execute("BEGIN IMMEDIATE")
try:
result = fn(conn)
conn.execute("COMMIT")
return result
except BaseException:
try:
conn.execute("ROLLBACK")
except sqlite3.Error:
pass
raise
return self.write(run)
def read(self, fn: Callable[[sqlite3.Connection], T]) -> T:
"""使用独立的短生命周期连接执行只读查询。"""
if self._closed:
raise DatabaseClosed("数据库已关闭")
conn = self._connect()
try:
return fn(conn)
finally:
conn.close()
def close(self) -> None:
if self._closed:
return
self._closed = True
if self._thread is not None:
self._queue.put(None)
self._thread.join(timeout=10)
self._thread = None
+101
View File
@@ -0,0 +1,101 @@
# -*- coding: utf-8 -*-
"""脱敏 JSON 行日志。
日志只写 stdout,敏感字段(cookie / csrf / token / secret 等)在序列化前统一
替换为 ``***``;即使调用方误把异常文本或 Cookie 塞进普通字段,也会在字符串
层面再脱敏一次。
"""
from __future__ import annotations
import json
import re
import sys
import threading
from datetime import datetime, timezone
from typing import Any, IO, Mapping
REDACTED = "***"
_SENSITIVE_KEY_PARTS = (
"cookie",
"csrf",
"token",
"secret",
"password",
"authorization",
"sessdata",
"bili_jct",
)
_STRING_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
(
re.compile(
r"(?i)\b(SESSDATA|bili_jct|buvid3|buvid4|DedeUserID__ckMd5|DedeUserID)"
r"\s*=\s*([^;\s]+)"
),
r"\1=" + REDACTED,
),
(re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+"), "Bearer " + REDACTED),
(re.compile(r"\b\d{6,}:[A-Za-z0-9_-]{25,}\b"), REDACTED),
)
def _is_sensitive_key(key: str) -> bool:
normalized = key.lower().replace("-", "_")
return any(part in normalized for part in _SENSITIVE_KEY_PARTS)
def _redact_string(value: str) -> str:
for pattern, replacement in _STRING_PATTERNS:
value = pattern.sub(replacement, value)
return value
def redact(value: Any, *, key: str | None = None) -> Any:
"""递归脱敏。敏感键的整个值替换为 ``***``。"""
if key is not None and _is_sensitive_key(key):
return REDACTED
if isinstance(value, str):
return _redact_string(value)
if isinstance(value, Mapping):
return {k: redact(v, key=str(k)) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return [redact(item) for item in value]
return value
def _now_iso() -> str:
return datetime.now(timezone.utc).isoformat(timespec="seconds")
class JsonLogger:
"""线程安全的 JSON 行 logger。"""
def __init__(self, stream: IO[str] | None = None) -> None:
self.stream = stream if stream is not None else sys.stdout
self._lock = threading.Lock()
def emit(self, level: str, event: str, **fields: Any) -> dict[str, Any]:
payload: dict[str, Any] = {
"ts": _now_iso(),
"level": level,
"event": event,
}
payload.update(redact(fields))
line = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
with self._lock:
print(line, file=self.stream, flush=True)
return payload
def debug(self, event: str, **fields: Any) -> dict[str, Any]:
return self.emit("debug", event, **fields)
def info(self, event: str, **fields: Any) -> dict[str, Any]:
return self.emit("info", event, **fields)
def warning(self, event: str, **fields: Any) -> dict[str, Any]:
return self.emit("warning", event, **fields)
def error(self, event: str, **fields: Any) -> dict[str, Any]:
return self.emit("error", event, **fields)
+305
View File
@@ -0,0 +1,305 @@
# -*- coding: utf-8 -*-
"""SQLite schema 迁移。
规则(DEVELOPMENT.md §6.3):
* 使用 ``schema_version(version, applied_at)`` 记录已应用版本;
* 每个版本包在单一事务里,失败整体回滚;
* 只向前追加,不允许删库重建;
* 升级已有数据库前自动创建 SQLite 备份。
"""
from __future__ import annotations
import sqlite3
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Sequence
@dataclass(frozen=True)
class Migration:
version: int
name: str
statements: tuple[str, ...]
MIGRATIONS: tuple[Migration, ...] = (
Migration(
version=1,
name="initial schema",
statements=(
"""
CREATE TABLE video (
aid INTEGER PRIMARY KEY,
bvid TEXT UNIQUE,
title TEXT NOT NULL,
up_mid INTEGER,
up_name TEXT,
pubdate INTEGER NOT NULL,
duration INTEGER,
cover TEXT,
folder_id INTEGER,
fav_time INTEGER,
status TEXT NOT NULL,
source TEXT,
first_seen INTEGER NOT NULL,
last_sync INTEGER,
attr INTEGER
)
""",
"CREATE INDEX idx_video_status ON video(status, pubdate DESC)",
"CREATE INDEX idx_video_up ON video(up_mid, pubdate DESC)",
"""
CREATE TABLE snapshot (
stat_date TEXT NOT NULL,
aid INTEGER NOT NULL,
view INTEGER,
like INTEGER,
favorite INTEGER,
coin INTEGER,
share INTEGER,
reply INTEGER,
danmaku INTEGER,
captured_at INTEGER NOT NULL,
ok INTEGER NOT NULL DEFAULT 1,
err TEXT,
src TEXT DEFAULT 'view',
PRIMARY KEY (stat_date, aid)
)
""",
"CREATE INDEX idx_snap_aid ON snapshot(aid, stat_date)",
"""
CREATE TABLE folder (
media_id INTEGER PRIMARY KEY,
year INTEGER NOT NULL,
title TEXT,
media_count INTEGER,
last_ids_at INTEGER,
last_full_at INTEGER,
enabled INTEGER NOT NULL DEFAULT 1
)
""",
"""
CREATE TABLE folder_resource (
media_id INTEGER NOT NULL,
aid INTEGER NOT NULL,
first_seen INTEGER NOT NULL,
active INTEGER NOT NULL DEFAULT 1,
last_seen INTEGER,
PRIMARY KEY (media_id, aid)
)
""",
"CREATE INDEX idx_folder_resource_aid ON folder_resource(aid, active)",
"""
CREATE TABLE up_blacklist (
mid INTEGER PRIMARY KEY,
name TEXT,
reason TEXT,
created_at INTEGER NOT NULL
)
""",
"""
CREATE TABLE up_stat (
stat_date TEXT NOT NULL,
mid INTEGER NOT NULL,
follower INTEGER,
following INTEGER,
captured_at INTEGER NOT NULL,
ok INTEGER NOT NULL DEFAULT 1,
PRIMARY KEY (stat_date, mid)
)
""",
"""
CREATE TABLE candidate (
aid INTEGER PRIMARY KEY,
bvid TEXT,
title TEXT,
up_mid INTEGER,
up_name TEXT,
pubdate INTEGER,
cover TEXT,
score REAL,
channel TEXT,
first_seen INTEGER NOT NULL,
notified_at INTEGER,
decision TEXT,
decided_at INTEGER
)
""",
"""
CREATE TABLE seen_aid (
aid INTEGER PRIMARY KEY,
first_seen INTEGER NOT NULL,
source TEXT
)
""",
"""
CREATE TABLE notify (
aid INTEGER NOT NULL,
channel TEXT NOT NULL,
batch_id TEXT,
sent_at INTEGER NOT NULL,
ok INTEGER NOT NULL,
err TEXT,
PRIMARY KEY (aid, channel)
)
""",
"""
CREATE TABLE fav_action (
token TEXT PRIMARY KEY,
aid INTEGER NOT NULL,
folder_id INTEGER NOT NULL,
state TEXT NOT NULL,
created_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL,
done_at INTEGER,
position INTEGER,
err TEXT
)
""",
"""
CREATE TABLE kv (
k TEXT PRIMARY KEY,
v TEXT
)
""",
),
),
)
class MigrationError(RuntimeError):
"""迁移失败,原事务已回滚。"""
def _schema_version_table(conn: sqlite3.Connection) -> None:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_version ("
"version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL)"
)
def current_version(conn: sqlite3.Connection) -> int:
_schema_version_table(conn)
row = conn.execute("SELECT MAX(version) FROM schema_version").fetchone()
return int(row[0] or 0) if row else 0
def _user_tables(conn: sqlite3.Connection) -> list[str]:
rows = conn.execute(
"SELECT name FROM sqlite_master "
"WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name != 'schema_version'"
).fetchall()
return [str(row[0]) for row in rows]
def backup_database(
conn: sqlite3.Connection,
backup_dir: str | Path,
*,
version: int,
now: float | None = None,
) -> Path | None:
"""用 SQLite backup API 生成一致性备份;内存库返回 None。"""
try:
row = conn.execute("PRAGMA database_list").fetchone()
except sqlite3.Error:
return None
if not row or not row[2] or str(row[2]) == ":memory:":
return None
target_dir = Path(backup_dir)
target_dir.mkdir(parents=True, exist_ok=True)
stamp = time.strftime("%Y%m%d%H%M%S", time.localtime(now))
target = target_dir / f"app-pre-v{version}-{stamp}.db"
suffix = 1
while target.exists():
target = target_dir / f"app-pre-v{version}-{stamp}-{suffix}.db"
suffix += 1
dst = sqlite3.connect(target)
try:
conn.backup(dst)
finally:
dst.close()
return target
def prune_backups(
backup_dir: str | Path, *, keep_days: int, now: float | None = None
) -> list[Path]:
"""删除超过保留期的迁移备份。"""
target_dir = Path(backup_dir)
if not target_dir.is_dir():
return []
cutoff = (time.time() if now is None else now) - keep_days * 86400
removed: list[Path] = []
for path in target_dir.glob("app-pre-v*.db"):
try:
if path.stat().st_mtime < cutoff:
path.unlink()
removed.append(path)
except OSError:
continue
return removed
def _validate_migrations(migrations: Sequence[Migration]) -> None:
versions = [m.version for m in migrations]
if versions != sorted(versions):
raise ValueError("迁移版本必须按升序排列")
if len(set(versions)) != len(versions):
raise ValueError("迁移版本不能重复")
if versions and versions[0] < 1:
raise ValueError("迁移版本必须从 1 开始")
def run_migrations(
conn: sqlite3.Connection,
*,
backup_dir: str | Path | None = None,
backup_keep_days: int = 14,
now: float | None = None,
migrations: Sequence[Migration] = MIGRATIONS,
logger=None,
) -> list[int]:
"""应用所有待执行迁移,返回本次应用的版本列表。"""
_validate_migrations(migrations)
_schema_version_table(conn)
version = current_version(conn)
pending = [m for m in migrations if m.version > version]
if not pending:
return []
if backup_dir is not None and _user_tables(conn):
backup = backup_database(conn, backup_dir, version=version, now=now)
prune_backups(backup_dir, keep_days=backup_keep_days)
if logger is not None and backup is not None:
logger.info("db.backup", path=str(backup), version=version)
applied: list[int] = []
for migration in pending:
try:
conn.execute("BEGIN IMMEDIATE")
for statement in migration.statements:
conn.execute(statement)
conn.execute(
"INSERT INTO schema_version(version, applied_at) VALUES (?, ?)",
(migration.version, int(time.time() if now is None else now)),
)
conn.execute("COMMIT")
except Exception as exc: # noqa: BLE001 任何错误都必须回滚
try:
conn.execute("ROLLBACK")
except sqlite3.Error:
pass
raise MigrationError(
f"迁移 v{migration.version} ({migration.name}) 失败,已回滚: {exc}"
) from exc
applied.append(migration.version)
if logger is not None:
logger.info("db.migrated", version=migration.version, name=migration.name)
return applied
+441
View File
@@ -0,0 +1,441 @@
# -*- coding: utf-8 -*-
"""P0 数据访问层:收藏夹、视频与黑名单。
业务状态迁移集中在这里,服务层只负责请求与编排。所有写操作经 ``Database``
的单写队列串行执行。
"""
from __future__ import annotations
import sqlite3
import time
from dataclasses import dataclass
from typing import Any, Iterable
from .config import FolderConfig
from .database import Database
@dataclass(frozen=True)
class UpsertOutcome:
aid: int
status: str
previous_folder_id: int | None
moved_folder: bool
@dataclass(frozen=True)
class FolderRow:
media_id: int
year: int
title: str
media_count: int | None
last_ids_at: int | None
last_full_at: int | None
enabled: bool
class Repository:
def __init__(self, db: Database, *, clock=time.time) -> None:
self.db = db
self._clock = clock
def _now(self) -> int:
return int(self._clock())
# ------------------------------------------------------------ folder
def ensure_folder(self, folder: FolderConfig, *, now: int | None = None) -> None:
now = self._now() if now is None else int(now)
def run(conn: sqlite3.Connection) -> None:
conn.execute(
"""
INSERT INTO folder(media_id, year, title, media_count,
last_ids_at, last_full_at, enabled)
VALUES (?, ?, ?, NULL, NULL, NULL, 1)
ON CONFLICT(media_id) DO UPDATE SET
year = excluded.year,
title = COALESCE(NULLIF(excluded.title, ''), folder.title),
enabled = 1
""",
(folder.media_id, folder.year, folder.title),
)
self.db.write(run)
def disable_folders_except(self, media_ids: Iterable[int]) -> None:
ids = [int(mid) for mid in media_ids]
def run(conn: sqlite3.Connection) -> None:
if ids:
placeholders = ",".join("?" for _ in ids)
conn.execute(
f"UPDATE folder SET enabled = 0 WHERE media_id NOT IN ({placeholders})",
ids,
)
else:
conn.execute("UPDATE folder SET enabled = 0")
self.db.write(run)
def finish_folder_sync(
self,
folder: FolderConfig,
*,
media_count: int,
full_refresh: bool,
now: int | None = None,
) -> None:
now = self._now() if now is None else int(now)
def run(conn: sqlite3.Connection) -> None:
conn.execute(
"""
INSERT INTO folder(media_id, year, title, media_count,
last_ids_at, last_full_at, enabled)
VALUES (?, ?, ?, ?, ?, ?, 1)
ON CONFLICT(media_id) DO UPDATE SET
year = excluded.year,
title = COALESCE(NULLIF(excluded.title, ''), folder.title),
media_count = excluded.media_count,
last_ids_at = excluded.last_ids_at,
last_full_at = CASE
WHEN ? THEN excluded.last_full_at
ELSE folder.last_full_at
END,
enabled = 1
""",
(
folder.media_id, folder.year, folder.title, int(media_count),
now, now if full_refresh else None, 1 if full_refresh else 0,
),
)
self.db.write(run)
def get_folder(self, media_id: int) -> FolderRow | None:
def run(conn: sqlite3.Connection) -> FolderRow | None:
row = conn.execute(
"SELECT media_id, year, title, media_count, last_ids_at, "
"last_full_at, enabled FROM folder WHERE media_id = ?",
(int(media_id),),
).fetchone()
if row is None:
return None
return FolderRow(
media_id=int(row["media_id"]),
year=int(row["year"]),
title=str(row["title"] or ""),
media_count=row["media_count"],
last_ids_at=row["last_ids_at"],
last_full_at=row["last_full_at"],
enabled=bool(row["enabled"]),
)
return self.db.read(run)
def list_folders(self) -> list[FolderRow]:
def run(conn: sqlite3.Connection) -> list[FolderRow]:
rows = conn.execute(
"SELECT media_id, year, title, media_count, last_ids_at, "
"last_full_at, enabled FROM folder ORDER BY year, media_id"
).fetchall()
return [
FolderRow(
media_id=int(row["media_id"]),
year=int(row["year"]),
title=str(row["title"] or ""),
media_count=row["media_count"],
last_ids_at=row["last_ids_at"],
last_full_at=row["last_full_at"],
enabled=bool(row["enabled"]),
)
for row in rows
]
return self.db.read(run)
def folder_active_aids(self, media_id: int) -> set[int]:
def run(conn: sqlite3.Connection) -> set[int]:
rows = conn.execute(
"SELECT aid FROM folder_resource WHERE media_id = ? AND active = 1",
(int(media_id),),
).fetchall()
return {int(row["aid"]) for row in rows}
return self.db.read(run)
def touch_folder_resources(self, media_id: int, *, now: int | None = None) -> None:
now = self._now() if now is None else int(now)
self.db.write(lambda conn: conn.execute(
"UPDATE folder_resource SET last_seen = ? WHERE media_id = ? AND active = 1",
(now, int(media_id)),
))
# ------------------------------------------------------------ video
def upsert_from_fav(
self,
resource,
folder_id: int,
*,
was_added: bool,
now: int | None = None,
) -> UpsertOutcome:
now = self._now() if now is None else int(now)
folder_id = int(folder_id)
invalid = bool(getattr(resource, "invalid", False))
def run(conn: sqlite3.Connection) -> UpsertOutcome:
previous = conn.execute(
"SELECT status, folder_id FROM video WHERE aid = ?", (resource.aid,)
).fetchone()
previous_status = str(previous["status"]) if previous else None
previous_folder = (
int(previous["folder_id"])
if previous and previous["folder_id"] is not None else None
)
if previous_status is None:
status = "removed" if invalid else "active"
elif previous_status == "dropped":
status = "active" if (was_added and not invalid) else "dropped"
elif invalid:
status = "removed"
else:
status = "active"
conn.execute(
"""
INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate,
duration, cover, folder_id, fav_time, status,
source, first_seen, last_sync, attr)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(aid) DO UPDATE SET
bvid = COALESCE(excluded.bvid, video.bvid),
title = CASE
WHEN excluded.status = 'removed' AND video.title != ''
THEN video.title
WHEN excluded.title != '' THEN excluded.title
ELSE video.title
END,
up_mid = COALESCE(excluded.up_mid, video.up_mid),
up_name = COALESCE(excluded.up_name, video.up_name),
pubdate = CASE
WHEN excluded.pubdate > 0 THEN excluded.pubdate
ELSE video.pubdate
END,
duration = COALESCE(excluded.duration, video.duration),
cover = COALESCE(excluded.cover, video.cover),
folder_id = excluded.folder_id,
fav_time = COALESCE(excluded.fav_time, video.fav_time),
status = excluded.status,
source = COALESCE(video.source, excluded.source),
last_sync = excluded.last_sync,
attr = excluded.attr
""",
(
resource.aid,
resource.bvid,
resource.title or "(无标题)",
resource.up_mid,
resource.up_name,
int(resource.pubdate or 0),
resource.duration,
resource.cover,
folder_id,
resource.fav_time,
status,
f"fav:{folder_id}",
now,
now,
resource.attr,
),
)
conn.execute(
"""
INSERT INTO folder_resource(media_id, aid, first_seen, active, last_seen)
VALUES (?, ?, ?, 1, ?)
ON CONFLICT(media_id, aid) DO UPDATE SET
active = 1,
last_seen = excluded.last_seen
""",
(folder_id, resource.aid, now, now),
)
return UpsertOutcome(
aid=resource.aid,
status=status,
previous_folder_id=previous_folder,
moved_folder=previous_folder is not None and previous_folder != folder_id,
)
return self.db.write(run)
def mark_missing_resource(
self,
aid: int,
*,
folder_id: int,
bvid: str | None = None,
now: int | None = None,
) -> str:
"""处理 infos 静默缺失:写入/更新为 removed,保留历史。"""
now = self._now() if now is None else int(now)
aid = int(aid)
folder_id = int(folder_id)
def run(conn: sqlite3.Connection) -> str:
row = conn.execute("SELECT status FROM video WHERE aid = ?", (aid,)).fetchone()
previous_status = str(row["status"]) if row else None
status = "dropped" if previous_status == "dropped" else "removed"
conn.execute(
"""
INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate,
duration, cover, folder_id, fav_time, status,
source, first_seen, last_sync, attr)
VALUES (?, ?, '(资源已失效)', NULL, NULL, 0, NULL, NULL, ?, NULL,
?, ?, ?, ?, 9)
ON CONFLICT(aid) DO UPDATE SET
bvid = COALESCE(video.bvid, excluded.bvid),
status = ?,
last_sync = excluded.last_sync,
attr = 9
""",
(aid, bvid, folder_id, status, f"fav:{folder_id}", now, now, status),
)
conn.execute(
"""
INSERT INTO folder_resource(media_id, aid, first_seen, active, last_seen)
VALUES (?, ?, ?, 1, ?)
ON CONFLICT(media_id, aid) DO UPDATE SET
active = 1,
last_seen = excluded.last_seen
""",
(folder_id, aid, now, now),
)
return status
return self.db.write(run)
def mark_resource_removed(
self, media_id: int, aid: int, *, now: int | None = None
) -> str | None:
"""资源从收藏夹移除:active -> unfaved,其他(dropped/removed)保持不变。"""
now = self._now() if now is None else int(now)
media_id, aid = int(media_id), int(aid)
def run(conn: sqlite3.Connection) -> str | None:
conn.execute(
"UPDATE folder_resource SET active = 0, last_seen = ? "
"WHERE media_id = ? AND aid = ?",
(now, media_id, aid),
)
other = conn.execute(
"""
SELECT 1
FROM folder_resource fr
JOIN folder f ON f.media_id = fr.media_id
WHERE fr.aid = ? AND fr.active = 1 AND f.enabled = 1
LIMIT 1
""",
(aid,),
).fetchone()
if other is None:
conn.execute(
"UPDATE video SET status = 'unfaved', last_sync = ? "
"WHERE aid = ? AND status IN ('active', 'unfaved')",
(now, aid),
)
row = conn.execute("SELECT status FROM video WHERE aid = ?", (aid,)).fetchone()
return str(row["status"]) if row else None
return self.db.write(run)
def set_video_status(self, aid: int, status: str, *, now: int | None = None) -> None:
now = self._now() if now is None else int(now)
self.db.write(lambda conn: conn.execute(
"UPDATE video SET status = ?, last_sync = ? WHERE aid = ?",
(status, now, int(aid)),
))
def get_video(self, aid: int) -> dict[str, Any] | None:
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
row = conn.execute("SELECT * FROM video WHERE aid = ?", (int(aid),)).fetchone()
return dict(row) if row else None
return self.db.read(run)
def list_videos(self) -> list[dict[str, Any]]:
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
rows = conn.execute("SELECT * FROM video ORDER BY aid").fetchall()
return [dict(row) for row in rows]
return self.db.read(run)
def folder_resources(self, media_id: int) -> list[dict[str, Any]]:
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
rows = conn.execute(
"SELECT * FROM folder_resource WHERE media_id = ? ORDER BY aid",
(int(media_id),),
).fetchall()
return [dict(row) for row in rows]
return self.db.read(run)
# ------------------------------------------------------------ blacklist
def seed_blacklist(self, mids: Iterable[int], *, now: int | None = None) -> int:
now = self._now() if now is None else int(now)
def run(conn: sqlite3.Connection) -> int:
inserted = 0
for mid in mids:
cur = conn.execute(
"INSERT OR IGNORE INTO up_blacklist(mid, name, reason, created_at) "
"VALUES (?, NULL, 'seed', ?)",
(int(mid), now),
)
inserted += max(cur.rowcount, 0)
return inserted
return self.db.write(run)
def add_blacklist(
self,
mid: int,
*,
name: str | None = None,
reason: str | None = None,
now: int | None = None,
) -> None:
now = self._now() if now is None else int(now)
self.db.write(lambda conn: conn.execute(
"""
INSERT INTO up_blacklist(mid, name, reason, created_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(mid) DO UPDATE SET
name = COALESCE(excluded.name, up_blacklist.name),
reason = COALESCE(excluded.reason, up_blacklist.reason)
""",
(int(mid), name, reason, now),
))
def is_blacklisted(self, mid: int) -> bool:
def run(conn: sqlite3.Connection) -> bool:
row = conn.execute(
"SELECT 1 FROM up_blacklist WHERE mid = ?", (int(mid),)
).fetchone()
return row is not None
return self.db.read(run)
def blacklist(self) -> list[dict[str, Any]]:
def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
rows = conn.execute(
"SELECT mid, name, reason, created_at FROM up_blacklist ORDER BY created_at, mid"
).fetchall()
return [dict(row) for row in rows]
return self.db.read(run)
# ------------------------------------------------------------ summary
def counts(self) -> dict[str, Any]:
def run(conn: sqlite3.Connection) -> dict[str, Any]:
status_rows = conn.execute(
"SELECT status, COUNT(*) AS n FROM video GROUP BY status"
).fetchall()
by_status = {str(row["status"]): int(row["n"]) for row in status_rows}
folder_count = conn.execute("SELECT COUNT(*) FROM folder").fetchone()[0]
blacklist_count = conn.execute("SELECT COUNT(*) FROM up_blacklist").fetchone()[0]
candidate_count = conn.execute("SELECT COUNT(*) FROM candidate").fetchone()[0]
return {
"video_total": sum(by_status.values()),
"video_by_status": by_status,
"folders": int(folder_count),
"blacklist": int(blacklist_count),
"candidates": int(candidate_count),
}
return self.db.read(run)
+115
View File
@@ -0,0 +1,115 @@
# -*- coding: utf-8 -*-
"""单进程内置调度器。
P0 只注册收藏夹同步任务;P1/P3 会在同一调度器上追加快照、粉丝和发现任务。
调度只负责“什么时候运行”,具体任务保持幂等,并在自己的服务层处理重试。
"""
from __future__ import annotations
import time
from dataclasses import dataclass
from typing import Any, Callable, Sequence
@dataclass
class ScheduledTask:
name: str
interval: float
fn: Callable[[], Any]
next_run: float = 0.0
@dataclass
class TaskResult:
name: str
started_at: float
finished_at: float
ok: bool
error: str | None = None
def as_dict(self) -> dict[str, Any]:
return {
"name": self.name,
"started_at": int(self.started_at),
"finished_at": int(self.finished_at),
"ok": self.ok,
"error": self.error,
}
class Scheduler:
def __init__(
self,
tasks: Sequence[ScheduledTask],
*,
clock=time.time,
sleeper=time.sleep,
logger=None,
) -> None:
if not tasks:
raise ValueError("至少需要一个调度任务")
for task in tasks:
if task.interval <= 0:
raise ValueError(f"任务 {task.name} 的 interval 必须大于 0")
self.tasks = list(tasks)
self._clock = clock
self._sleeper = sleeper
self.logger = logger
def _log(self, level: str, event: str, **fields) -> None:
if self.logger is not None:
getattr(self.logger, level)(event, **fields)
def run_due(self, *, now: float | None = None) -> list[TaskResult]:
current = self._clock() if now is None else now
results: list[TaskResult] = []
for task in self.tasks:
if task.next_run > current:
continue
started = current
error: str | None = None
try:
task.fn()
except Exception as exc: # noqa: BLE001 单任务失败不终止调度
error = str(exc)
self._log("error", "scheduler.task_error", task=task.name, error=error)
else:
self._log("info", "scheduler.task_ok", task=task.name)
finished = self._clock()
task.next_run = started + task.interval
result = TaskResult(
name=task.name,
started_at=started,
finished_at=finished,
ok=error is None,
error=error,
)
results.append(result)
return results
def run_forever(
self,
*,
stop_event=None,
poll_seconds: float = 30.0,
max_iterations: int | None = None,
) -> list[TaskResult]:
"""阻塞运行;``stop_event`` 或 ``max_iterations`` 用于可控退出/测试。"""
if poll_seconds <= 0:
raise ValueError("poll_seconds 必须大于 0")
all_results: list[TaskResult] = []
iterations = 0
while True:
if stop_event is not None and stop_event.is_set():
break
all_results.extend(self.run_due())
iterations += 1
if max_iterations is not None and iterations >= max_iterations:
break
if stop_event is not None:
if stop_event.wait(poll_seconds):
break
else:
self._sleeper(poll_seconds)
return all_results
+2
View File
@@ -0,0 +1,2 @@
# -*- coding: utf-8 -*-
"""业务服务层。"""
+278
View File
@@ -0,0 +1,278 @@
# -*- coding: utf-8 -*-
"""收藏夹同步(库的真相源)。
每次运行:
1. 读取每个配置年度夹的全部 aid;
2. 与本地 ``folder_resource`` 做差集;
3. 新增资源按 200 条分块拉取 ``infos`` 并入库;
4. 从收藏夹移除的资源标记 ``unfaved``(不删除历史);
5. 每 ``full_folder_refresh_hours`` 小时全量刷新一次元数据并识别失效资源;
6. 容量接近上限时预警,但绝不自动换夹。
黑名单只在发现流程生效,本服务不读取 ``up_blacklist``。
"""
from __future__ import annotations
import time
from dataclasses import dataclass, field
from typing import Any
from ..bilibili.client import BilibiliClient
from ..config import AppConfig, ConfigStore, FolderConfig
from ..repository import Repository
CAPACITY_WARN = 950
@dataclass
class FolderSyncResult:
media_id: int
year: int
title: str
remote_count: int = 0
added: int = 0
removed: int = 0
refreshed: int = 0
missing: int = 0
full_refresh: bool = False
warning: str | None = None
error: str | None = None
def as_dict(self) -> dict[str, Any]:
return {
"media_id": self.media_id,
"year": self.year,
"title": self.title,
"remote_count": self.remote_count,
"added": self.added,
"removed": self.removed,
"refreshed": self.refreshed,
"missing": self.missing,
"full_refresh": self.full_refresh,
"warning": self.warning,
"error": self.error,
}
@dataclass
class SyncSummary:
started_at: int
finished_at: int = 0
folders: list[FolderSyncResult] = field(default_factory=list)
blacklist_seeded: int = 0
@property
def added(self) -> int:
return sum(f.added for f in self.folders)
@property
def removed(self) -> int:
return sum(f.removed for f in self.folders)
@property
def refreshed(self) -> int:
return sum(f.refreshed for f in self.folders)
@property
def missing(self) -> int:
return sum(f.missing for f in self.folders)
@property
def errors(self) -> list[str]:
return [f"folder {f.media_id}: {f.error}" for f in self.folders if f.error]
@property
def warnings(self) -> list[str]:
return [f"folder {f.media_id}: {f.warning}" for f in self.folders if f.warning]
@property
def ok(self) -> bool:
return not self.errors
def as_dict(self) -> dict[str, Any]:
return {
"started_at": self.started_at,
"finished_at": self.finished_at,
"elapsed": max(0, self.finished_at - self.started_at),
"added": self.added,
"removed": self.removed,
"refreshed": self.refreshed,
"missing": self.missing,
"blacklist_seeded": self.blacklist_seeded,
"folders": [f.as_dict() for f in self.folders],
"errors": self.errors,
"warnings": self.warnings,
"ok": self.ok,
}
class FolderSyncService:
def __init__(
self,
config_store: ConfigStore,
repository: Repository,
client: BilibiliClient,
*,
clock=time.time,
logger=None,
capacity_warn: int = CAPACITY_WARN,
) -> None:
self.config_store = config_store
self.repo = repository
self.client = client
self._clock = clock
self.logger = logger
self.capacity_warn = capacity_warn
def _now(self) -> int:
return int(self._clock())
def _log(self, level: str, event: str, **fields) -> None:
if self.logger is not None:
getattr(self.logger, level)(event, **fields)
def run(self, *, full_refresh: bool = False) -> SyncSummary:
cfg = self.config_store.get()
now = self._now()
summary = SyncSummary(started_at=now)
self._log(
"info", "folder_sync.start", folders=len(cfg.folders), full=full_refresh
)
for folder in cfg.folders:
self.repo.ensure_folder(folder, now=now)
self.repo.disable_folders_except(f.media_id for f in cfg.folders)
if cfg.up_blacklist_seed:
summary.blacklist_seeded = self.repo.seed_blacklist(
cfg.up_blacklist_seed, now=now
)
for folder in cfg.folders:
try:
result = self._sync_folder(cfg, folder, now=now, force_full=full_refresh)
except Exception as exc: # noqa: BLE001 单个夹失败不阻塞其它夹
result = FolderSyncResult(
media_id=folder.media_id,
year=folder.year,
title=folder.title,
error=str(exc),
)
self._log(
"error",
"folder_sync.folder_error",
media_id=folder.media_id,
error=str(exc),
)
else:
self._log("info", "folder_sync.folder", **result.as_dict())
if result.warning:
self._log(
"warning",
"folder_sync.capacity",
media_id=folder.media_id,
warning=result.warning,
)
summary.folders.append(result)
summary.finished_at = self._now()
self._log(
"info" if summary.ok else "error",
"folder_sync.done",
added=summary.added,
removed=summary.removed,
refreshed=summary.refreshed,
missing=summary.missing,
errors=len(summary.errors),
elapsed=max(0, summary.finished_at - summary.started_at),
)
return summary
def _sync_folder(
self,
cfg: AppConfig,
folder: FolderConfig,
*,
now: int,
force_full: bool,
) -> FolderSyncResult:
row = self.repo.get_folder(folder.media_id)
full_due = (
force_full
or row is None
or row.last_full_at is None
or now - int(row.last_full_at)
>= cfg.schedule.full_folder_refresh_hours * 3600
)
resources = self.client.fav_ids(folder.media_id)
remote_map = {item.aid: item for item in resources if item.type == 2}
remote = set(remote_map)
local = self.repo.folder_active_aids(folder.media_id)
added = remote - local
removed = local - remote
result = FolderSyncResult(
media_id=folder.media_id,
year=folder.year,
title=folder.title or (row.title if row else ""),
remote_count=len(remote),
added=len(added),
removed=len(removed),
full_refresh=full_due,
)
targets = set(added)
if full_due:
targets |= remote
if targets:
infos = self.client.fav_infos(
folder.media_id,
sorted(targets),
folder_mid=cfg.bilibili.mid,
)
for aid, resource in infos.items.items():
if aid not in remote:
continue
outcome = self.repo.upsert_from_fav(
resource, folder.media_id, was_added=aid in added, now=now
)
if outcome.moved_folder:
self._log(
"warning",
"folder_sync.multi_folder",
aid=aid,
previous_folder_id=outcome.previous_folder_id,
folder_id=folder.media_id,
)
for aid in infos.missing:
source = remote_map.get(aid)
self.repo.mark_missing_resource(
aid,
folder_id=folder.media_id,
bvid=source.bvid if source else None,
now=now,
)
result.refreshed = len(infos.items)
result.missing = len(infos.missing)
for aid in removed:
self.repo.mark_resource_removed(folder.media_id, aid, now=now)
self.repo.touch_folder_resources(folder.media_id, now=now)
self.repo.finish_folder_sync(
folder,
media_count=len(remote),
full_refresh=full_due,
now=now,
)
if len(remote) >= self.capacity_warn:
result.warning = (
f"收藏夹「{result.title or folder.media_id}」已有 {len(remote)} 条,"
f"达到预警线 {self.capacity_warn};请人工准备下一年度收藏夹并更新配置,"
f"系统不会自动换夹"
)
return result