kairee
e2cdf997af
Implement P0 of the reaction dashboard: config loading with hot reload, SQLite migrations with backups, single-writer queue, read-only Bilibili fav client, folder import/diff sync, invalid-resource handling, blacklist seed, scheduler, and JSON redacted logging. Includes offline unit and transport-level contract tests; all existing and new tests pass.
279 lines
8.7 KiB
Python
279 lines
8.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""收藏夹同步（库的真相源）。
|
|
|
|
每次运行：
|
|
|
|
1. 读取每个配置年度夹的全部 aid；
|
|
2. 与本地 ``folder_resource`` 做差集；
|
|
3. 新增资源按 200 条分块拉取 ``infos`` 并入库；
|
|
4. 从收藏夹移除的资源标记 ``unfaved``（不删除历史）；
|
|
5. 每 ``full_folder_refresh_hours`` 小时全量刷新一次元数据并识别失效资源；
|
|
6. 容量接近上限时预警，但绝不自动换夹。
|
|
|
|
黑名单只在发现流程生效，本服务不读取 ``up_blacklist``。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from ..bilibili.client import BilibiliClient
|
|
from ..config import AppConfig, ConfigStore, FolderConfig
|
|
from ..repository import Repository
|
|
|
|
CAPACITY_WARN = 950
|
|
|
|
|
|
@dataclass
|
|
class FolderSyncResult:
|
|
media_id: int
|
|
year: int
|
|
title: str
|
|
remote_count: int = 0
|
|
added: int = 0
|
|
removed: int = 0
|
|
refreshed: int = 0
|
|
missing: int = 0
|
|
full_refresh: bool = False
|
|
warning: str | None = None
|
|
error: str | None = None
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"media_id": self.media_id,
|
|
"year": self.year,
|
|
"title": self.title,
|
|
"remote_count": self.remote_count,
|
|
"added": self.added,
|
|
"removed": self.removed,
|
|
"refreshed": self.refreshed,
|
|
"missing": self.missing,
|
|
"full_refresh": self.full_refresh,
|
|
"warning": self.warning,
|
|
"error": self.error,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class SyncSummary:
|
|
started_at: int
|
|
finished_at: int = 0
|
|
folders: list[FolderSyncResult] = field(default_factory=list)
|
|
blacklist_seeded: int = 0
|
|
|
|
@property
|
|
def added(self) -> int:
|
|
return sum(f.added for f in self.folders)
|
|
|
|
@property
|
|
def removed(self) -> int:
|
|
return sum(f.removed for f in self.folders)
|
|
|
|
@property
|
|
def refreshed(self) -> int:
|
|
return sum(f.refreshed for f in self.folders)
|
|
|
|
@property
|
|
def missing(self) -> int:
|
|
return sum(f.missing for f in self.folders)
|
|
|
|
@property
|
|
def errors(self) -> list[str]:
|
|
return [f"folder {f.media_id}: {f.error}" for f in self.folders if f.error]
|
|
|
|
@property
|
|
def warnings(self) -> list[str]:
|
|
return [f"folder {f.media_id}: {f.warning}" for f in self.folders if f.warning]
|
|
|
|
@property
|
|
def ok(self) -> bool:
|
|
return not self.errors
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"started_at": self.started_at,
|
|
"finished_at": self.finished_at,
|
|
"elapsed": max(0, self.finished_at - self.started_at),
|
|
"added": self.added,
|
|
"removed": self.removed,
|
|
"refreshed": self.refreshed,
|
|
"missing": self.missing,
|
|
"blacklist_seeded": self.blacklist_seeded,
|
|
"folders": [f.as_dict() for f in self.folders],
|
|
"errors": self.errors,
|
|
"warnings": self.warnings,
|
|
"ok": self.ok,
|
|
}
|
|
|
|
|
|
class FolderSyncService:
|
|
def __init__(
|
|
self,
|
|
config_store: ConfigStore,
|
|
repository: Repository,
|
|
client: BilibiliClient,
|
|
*,
|
|
clock=time.time,
|
|
logger=None,
|
|
capacity_warn: int = CAPACITY_WARN,
|
|
) -> None:
|
|
self.config_store = config_store
|
|
self.repo = repository
|
|
self.client = client
|
|
self._clock = clock
|
|
self.logger = logger
|
|
self.capacity_warn = capacity_warn
|
|
|
|
def _now(self) -> int:
|
|
return int(self._clock())
|
|
|
|
def _log(self, level: str, event: str, **fields) -> None:
|
|
if self.logger is not None:
|
|
getattr(self.logger, level)(event, **fields)
|
|
|
|
def run(self, *, full_refresh: bool = False) -> SyncSummary:
|
|
cfg = self.config_store.get()
|
|
now = self._now()
|
|
summary = SyncSummary(started_at=now)
|
|
self._log(
|
|
"info", "folder_sync.start", folders=len(cfg.folders), full=full_refresh
|
|
)
|
|
|
|
for folder in cfg.folders:
|
|
self.repo.ensure_folder(folder, now=now)
|
|
self.repo.disable_folders_except(f.media_id for f in cfg.folders)
|
|
if cfg.up_blacklist_seed:
|
|
summary.blacklist_seeded = self.repo.seed_blacklist(
|
|
cfg.up_blacklist_seed, now=now
|
|
)
|
|
|
|
for folder in cfg.folders:
|
|
try:
|
|
result = self._sync_folder(cfg, folder, now=now, force_full=full_refresh)
|
|
except Exception as exc: # noqa: BLE001 单个夹失败不阻塞其它夹
|
|
result = FolderSyncResult(
|
|
media_id=folder.media_id,
|
|
year=folder.year,
|
|
title=folder.title,
|
|
error=str(exc),
|
|
)
|
|
self._log(
|
|
"error",
|
|
"folder_sync.folder_error",
|
|
media_id=folder.media_id,
|
|
error=str(exc),
|
|
)
|
|
else:
|
|
self._log("info", "folder_sync.folder", **result.as_dict())
|
|
if result.warning:
|
|
self._log(
|
|
"warning",
|
|
"folder_sync.capacity",
|
|
media_id=folder.media_id,
|
|
warning=result.warning,
|
|
)
|
|
summary.folders.append(result)
|
|
|
|
summary.finished_at = self._now()
|
|
self._log(
|
|
"info" if summary.ok else "error",
|
|
"folder_sync.done",
|
|
added=summary.added,
|
|
removed=summary.removed,
|
|
refreshed=summary.refreshed,
|
|
missing=summary.missing,
|
|
errors=len(summary.errors),
|
|
elapsed=max(0, summary.finished_at - summary.started_at),
|
|
)
|
|
return summary
|
|
|
|
def _sync_folder(
|
|
self,
|
|
cfg: AppConfig,
|
|
folder: FolderConfig,
|
|
*,
|
|
now: int,
|
|
force_full: bool,
|
|
) -> FolderSyncResult:
|
|
row = self.repo.get_folder(folder.media_id)
|
|
full_due = (
|
|
force_full
|
|
or row is None
|
|
or row.last_full_at is None
|
|
or now - int(row.last_full_at)
|
|
>= cfg.schedule.full_folder_refresh_hours * 3600
|
|
)
|
|
|
|
resources = self.client.fav_ids(folder.media_id)
|
|
remote_map = {item.aid: item for item in resources if item.type == 2}
|
|
remote = set(remote_map)
|
|
local = self.repo.folder_active_aids(folder.media_id)
|
|
added = remote - local
|
|
removed = local - remote
|
|
|
|
result = FolderSyncResult(
|
|
media_id=folder.media_id,
|
|
year=folder.year,
|
|
title=folder.title or (row.title if row else ""),
|
|
remote_count=len(remote),
|
|
added=len(added),
|
|
removed=len(removed),
|
|
full_refresh=full_due,
|
|
)
|
|
|
|
targets = set(added)
|
|
if full_due:
|
|
targets |= remote
|
|
|
|
if targets:
|
|
infos = self.client.fav_infos(
|
|
folder.media_id,
|
|
sorted(targets),
|
|
folder_mid=cfg.bilibili.mid,
|
|
)
|
|
for aid, resource in infos.items.items():
|
|
if aid not in remote:
|
|
continue
|
|
outcome = self.repo.upsert_from_fav(
|
|
resource, folder.media_id, was_added=aid in added, now=now
|
|
)
|
|
if outcome.moved_folder:
|
|
self._log(
|
|
"warning",
|
|
"folder_sync.multi_folder",
|
|
aid=aid,
|
|
previous_folder_id=outcome.previous_folder_id,
|
|
folder_id=folder.media_id,
|
|
)
|
|
for aid in infos.missing:
|
|
source = remote_map.get(aid)
|
|
self.repo.mark_missing_resource(
|
|
aid,
|
|
folder_id=folder.media_id,
|
|
bvid=source.bvid if source else None,
|
|
now=now,
|
|
)
|
|
result.refreshed = len(infos.items)
|
|
result.missing = len(infos.missing)
|
|
|
|
for aid in removed:
|
|
self.repo.mark_resource_removed(folder.media_id, aid, now=now)
|
|
|
|
self.repo.touch_folder_resources(folder.media_id, now=now)
|
|
self.repo.finish_folder_sync(
|
|
folder,
|
|
media_count=len(remote),
|
|
full_refresh=full_due,
|
|
now=now,
|
|
)
|
|
|
|
if len(remote) >= self.capacity_warn:
|
|
result.warning = (
|
|
f"收藏夹「{result.title or folder.media_id}」已有 {len(remote)} 条，"
|
|
f"达到预警线 {self.capacity_warn}；请人工准备下一年度收藏夹并更新配置，"
|
|
f"系统不会自动换夹"
|
|
)
|
|
return result
|