fix: close data integrity and hot-reload review findings

Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.

reaction_board/services/folder_sync.py

@@ -19,7 +19,6 @@ import time
 from dataclasses import dataclass, field
 from typing import Any
 
-from ..bilibili.client import BilibiliClient
 from ..config import AppConfig, ConfigStore, FolderConfig
 from ..repository import Repository
 
@@ -113,18 +112,29 @@ class FolderSyncService:
         self,
         config_store: ConfigStore,
         repository: Repository,
-        client: BilibiliClient,
+        client=None,
         *,
+        client_provider=None,
         clock=time.time,
         logger=None,
         capacity_warn: int = CAPACITY_WARN,
+        allow_empty_folders: bool = False,
     ) -> None:
+        if client is None and client_provider is None:
+            raise ValueError("必须提供 client 或 client_provider")
         self.config_store = config_store
         self.repo = repository
         self.client = client
+        self.client_provider = client_provider
         self._clock = clock
         self.logger = logger
         self.capacity_warn = capacity_warn
+        self.allow_empty_folders = allow_empty_folders
+
+    def _client(self):
+        if self.client_provider is not None:
+            return self.client_provider.get()
+        return self.client
 
     def _now(self) -> int:
         return int(self._clock())
@@ -135,6 +145,7 @@ class FolderSyncService:
 
     def run(self, *, full_refresh: bool = False) -> SyncSummary:
         cfg = self.config_store.get()
+        client = self._client()
         now = self._now()
         summary = SyncSummary(started_at=now)
         self._log(
@@ -151,7 +162,9 @@ class FolderSyncService:
 
         for folder in cfg.folders:
             try:
-                result = self._sync_folder(cfg, folder, now=now, force_full=full_refresh)
+                result = self._sync_folder(
+                    client, cfg, folder, now=now, force_full=full_refresh
+                )
             except Exception as exc:  # noqa: BLE001  单个夹失败不阻塞其它夹
                 result = FolderSyncResult(
                     media_id=folder.media_id,
@@ -191,6 +204,7 @@ class FolderSyncService:
 
     def _sync_folder(
         self,
+        client,
         cfg: AppConfig,
         folder: FolderConfig,
         *,
@@ -206,13 +220,20 @@ class FolderSyncService:
             >= cfg.schedule.full_folder_refresh_hours * 3600
         )
 
-        resources = self.client.fav_ids(folder.media_id)
+        resources = client.fav_ids(folder.media_id)
         remote_map = {item.aid: item for item in resources if item.type == 2}
         remote = set(remote_map)
         local = self.repo.folder_active_aids(folder.media_id)
         added = remote - local
         removed = local - remote
 
+        if not remote and local and not self.allow_empty_folders:
+            raise RuntimeError(
+                f"收藏夹 {folder.media_id} 远端返回 0 条，但本地仍有 {len(local)} 条；"
+                f"疑似接口异常，已跳过差集。确认收藏夹确实被清空后，"
+                f"可使用 --allow-empty-folder 重试"
+            )
+
         result = FolderSyncResult(
             media_id=folder.media_id,
             year=folder.year,
@@ -228,7 +249,7 @@ class FolderSyncService:
             targets |= remote
 
         if targets:
-            infos = self.client.fav_infos(
+            infos = client.fav_infos(
                 folder.media_id,
                 sorted(targets),
                 folder_mid=cfg.bilibili.read.mid,

reaction_board/services/snapshot.py

@@ -20,7 +20,6 @@ from typing import Any
 from ..bilibili.client import (
     VIDEO_UNAVAILABLE_CODES,
     BilibiliAPIError,
-    BilibiliClient,
     BilibiliError,
     BilibiliHTTPError,
 )
@@ -41,6 +40,7 @@ class SnapshotSummary:
     removed: int = 0
     fans_ok: int = 0
     fans_failed: int = 0
+    fans_skipped: int = 0
     auth_ok: bool | None = None
     auth_message: str = ""
     backup_path: str | None = None
@@ -63,6 +63,7 @@ class SnapshotSummary:
             "removed": self.removed,
             "fans_ok": self.fans_ok,
             "fans_failed": self.fans_failed,
+            "fans_skipped": self.fans_skipped,
             "auth_ok": self.auth_ok,
             "auth_message": self.auth_message,
             "backup_path": self.backup_path,
@@ -78,8 +79,9 @@ class SnapshotService:
         self,
         config_store: ConfigStore,
         repository: Repository,
-        client: BilibiliClient,
+        client=None,
         *,
+        client_provider=None,
         clock=time.time,
         sleeper=time.sleep,
         logger=None,
@@ -89,9 +91,12 @@ class SnapshotService:
     ) -> None:
         if retries < 1:
             raise ValueError("retries 必须 >= 1")
+        if client is None and client_provider is None:
+            raise ValueError("必须提供 client 或 client_provider")
         self.config_store = config_store
         self.repo = repository
         self.client = client
+        self.client_provider = client_provider
         self._clock = clock
         self._sleep = sleeper
         self.logger = logger
@@ -102,6 +107,11 @@ class SnapshotService:
     def _now(self) -> int:
         return int(self._clock())
 
+    def _client(self):
+        if self.client_provider is not None:
+            return self.client_provider.get()
+        return self.client
+
     def _log(self, level: str, event: str, **fields) -> None:
         if self.logger is not None:
             getattr(self.logger, level)(event, **fields)
@@ -110,6 +120,14 @@ class SnapshotService:
     def completed_key(stat_date: str) -> str:
         return f"snapshot:completed:{stat_date}"
 
+    @staticmethod
+    def backup_key(stat_date: str) -> str:
+        return f"snapshot:backup:{stat_date}"
+
+    @staticmethod
+    def backup_error_key(stat_date: str) -> str:
+        return f"snapshot:backup_error:{stat_date}"
+
     def run_if_needed(
         self, *, date: str | None = None, force: bool = False
     ) -> SnapshotSummary:
@@ -117,7 +135,14 @@ class SnapshotService:
         stat_date = date or shanghai_date(now)
         if not is_date(stat_date):
             raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
-        if not force and self.repo.kv_get(self.completed_key(stat_date)) == "1":
+        if force:
+            return self.run(date=stat_date, force=True)
+
+        completed = self.repo.kv_get(self.completed_key(stat_date)) == "1"
+        pending = self.repo.pending_snapshot_aids(stat_date)
+        if completed and not pending:
+            if self.repo.kv_get(self.backup_key(stat_date)) != "1":
+                return self.backup_only(stat_date)
             self._log("info", "snapshot.skip", stat_date=stat_date, reason="already_completed")
             return SnapshotSummary(
                 stat_date=stat_date,
@@ -126,10 +151,27 @@ class SnapshotService:
                 completed=True,
                 already_completed=True,
             )
-        return self.run(date=stat_date, force=force)
+        return self.run(date=stat_date, force=False)
+
+    def backup_only(self, stat_date: str) -> SnapshotSummary:
+        """采集已完成但备份缺失时，只重试备份，不请求任何外部接口。"""
+        now = self._now()
+        cfg = self.config_store.get()
+        summary = SnapshotSummary(
+            stat_date=stat_date,
+            started_at=now,
+            completed=True,
+            already_completed=True,
+        )
+        self._log("info", "snapshot.backup_retry", stat_date=stat_date)
+        self._run_backup(cfg, stat_date, summary)
+        summary.finished_at = self._now()
+        self._log("info", "snapshot.done", **summary.as_dict())
+        return summary
 
     def run(self, *, date: str | None = None, force: bool = False) -> SnapshotSummary:
         cfg = self.config_store.get()
+        client = self._client()
         now = self._now()
         stat_date = date or shanghai_date(now)
         if not is_date(stat_date):
@@ -137,8 +179,12 @@ class SnapshotService:
 
         summary = SnapshotSummary(stat_date=stat_date, started_at=now)
         self._log("info", "snapshot.start", stat_date=stat_date, force=force)
+        # 先落“未完成”，保证强制重采或中途中断后仍可续跑。
+        self.repo.kv_set(self.completed_key(stat_date), "0")
+        if self.repo.kv_get("snapshot:last_completed") == stat_date:
+            self.repo.kv_set("snapshot:last_completed", "")
 
-        summary.auth_ok, summary.auth_message = self._check_auth(cfg, now)
+        summary.auth_ok, summary.auth_message = self._check_auth(client, cfg, now)
 
         videos = self.repo.video_aids_for_snapshot()
         summary.total = len(videos)
@@ -148,7 +194,7 @@ class SnapshotService:
                 summary.skipped += 1
                 continue
 
-            stat, error, unavailable = self._fetch_view(aid)
+            stat, error, unavailable = self._fetch_view(client, aid)
             if unavailable:
                 summary.removed += 1
                 self.repo.set_video_status(aid, "removed", now=now)
@@ -193,7 +239,15 @@ class SnapshotService:
                 )
 
         for mid in cfg.track_fans:
-            stat, error = self._fetch_fan(mid)
+            existing_fan = self.repo.get_up_stat(stat_date, mid)
+            if (
+                not force
+                and existing_fan is not None
+                and int(existing_fan.get("ok") or 0) == 1
+            ):
+                summary.fans_skipped += 1
+                continue
+            stat, error = self._fetch_fan(client, mid)
             if stat is not None:
                 self.repo.upsert_up_stat(
                     stat_date,
@@ -228,6 +282,21 @@ class SnapshotService:
         elif self.repo.kv_get("snapshot:last_completed") == stat_date:
             self.repo.kv_set("snapshot:last_completed", "")
 
+        self._run_backup(cfg, stat_date, summary)
+
+        summary.finished_at = self._now()
+        self._log(
+            "info" if summary.completed else "warning",
+            "snapshot.done",
+            **summary.as_dict(),
+        )
+        return summary
+
+    # ------------------------------------------------------------ internals
+
+    def _run_backup(
+        self, cfg: AppConfig, stat_date: str, summary: SnapshotSummary
+    ) -> None:
         try:
             path = backup_daily(
                 self.repo.db,
@@ -240,28 +309,24 @@ class SnapshotService:
                 keep_days=cfg.storage.backup_keep_days,
             )
             summary.backup_path = str(path)
+            self.repo.kv_set(self.backup_key(stat_date), "1")
+            self.repo.kv_set(self.backup_error_key(stat_date), "")
         except Exception as exc:  # noqa: BLE001  备份失败不推翻已采集数据
             summary.backup_error = str(exc)
+            self.repo.kv_set(self.backup_key(stat_date), "0")
+            self.repo.kv_set(self.backup_error_key(stat_date), str(exc))
             self._log("error", "backup.failed", error=str(exc), stat_date=stat_date)
 
-        summary.finished_at = self._now()
-        self._log(
-            "info" if summary.completed else "warning",
-            "snapshot.done",
-            **summary.as_dict(),
-        )
-        return summary
-
-    # ------------------------------------------------------------ internals
-
-    def _check_auth(self, cfg: AppConfig, now: int) -> tuple[bool, str]:
+    def _check_auth(
+        self, client, cfg: AppConfig, now: int
+    ) -> tuple[bool, str]:
         if not cfg.bilibili.read.cookie:
             self.repo.kv_set("auth:last_check", str(now))
             self.repo.kv_set("auth:ok", "0")
             self.repo.kv_set("auth:message", "未配置 cookie")
             return False, "未配置 cookie"
         try:
-            state = self.client.check_login()
+            state = client.check_login()
         except BilibiliError as exc:
             message = str(exc)
             self.repo.kv_set("auth:last_check", str(now))
@@ -282,11 +347,11 @@ class SnapshotService:
             )
         return state.is_login, state.message
 
-    def _fetch_view(self, aid: int):
+    def _fetch_view(self, client, aid: int):
         last_error = ""
         for attempt in range(1, self.retries + 1):
             try:
-                return self.client.view(aid), None, False
+                return client.view(aid), None, False
             except BilibiliAPIError as exc:
                 last_error = str(exc)
                 if exc.code in VIDEO_UNAVAILABLE_CODES:
@@ -301,11 +366,11 @@ class SnapshotService:
                 self._sleep(self.backoff * (2 ** (attempt - 1)))
         return None, last_error or "未知错误", False
 
-    def _fetch_fan(self, mid: int):
+    def _fetch_fan(self, client, mid: int):
         last_error = ""
         for attempt in range(1, self.retries + 1):
             try:
-                return self.client.relation_stat(mid), None
+                return client.relation_stat(mid), None
             except BilibiliError as exc:
                 last_error = str(exc)
             if attempt < self.retries: