feat(up): capture follower snapshots at ingest and backfill history

reaction_board/cli.py

@@ -39,6 +39,7 @@ from .services.favorites import FavoriteActionService
 from .services.folder_sync import FolderSyncService
 from .services.snapshot import SnapshotService
 from .services.telegram_poller import TelegramPoller
+from .services.up_profile import UpProfileService
 from .timeutil import next_daily_run, parse_hhmm, shanghai_date
 
 ROOT = Path(__file__).resolve().parent.parent
@@ -120,6 +121,12 @@ def _build_parser() -> argparse.ArgumentParser:
         "--preview", action="store_true",
         help="以预览模式启动（要求数据库含 preview:seeded=1 标记）",
     )
+    backfill = sub.add_parser(
+        "backfill-followers", parents=[common],
+        help="为历史视频回填 UP 粉丝数快照（只补空值）",
+    )
+    backfill.add_argument("--limit", type=int, default=None, help="最多处理多少个 UP")
+    backfill.add_argument("--dry-run", action="store_true", help="只统计待回填数量，不发请求")
     sub.add_parser("status", parents=[common], help="查看数据库状态（离线）")
     return parser
 
@@ -226,6 +233,33 @@ def cmd_sync(args, logger: JsonLogger) -> int:
         db.close()
 
 
+def cmd_backfill_followers(args, logger: JsonLogger) -> int:
+    store = ConfigStore(args.config, require_auth=False)
+    cfg = store.get()
+    db = _open_db(cfg, logger)
+    try:
+        repo = Repository(db)
+        client = _client(
+            cfg.bilibili.read, logger, pacer=Pacer(1.0, 1.6), retries=2
+        )
+        service = UpProfileService(repo, lambda: client, logger=logger)
+        started = int(time.time())
+        result = service.backfill(limit=args.limit, dry_run=args.dry_run)
+        payload = result.as_dict()
+        payload["dry_run"] = bool(args.dry_run)
+        payload["coverage"] = repo.up_follower_coverage()
+        payload["database"] = str(cfg.storage.database_path)
+        _record_task(
+            repo, "backfill_followers", ok=result.failed == 0,
+            started_at=started, finished_at=int(time.time()), payload=payload,
+            error="；".join(result.errors) if result.errors else None,
+        )
+        _print(payload)
+        return 0 if result.failed == 0 else 1
+    finally:
+        db.close()
+
+
 def cmd_snapshot(args, logger: JsonLogger) -> int:
     store = ConfigStore(args.config, require_auth=False)
     cfg = store.get()
@@ -673,6 +707,8 @@ def main(argv: Sequence[str] | None = None) -> int:
             return cmd_sync(args, logger)
         if args.command == "snapshot":
             return cmd_snapshot(args, logger)
+        if args.command == "backfill-followers":
+            return cmd_backfill_followers(args, logger)
         if args.command == "discover":
             return cmd_discover(args, logger)
         if args.command == "candidates":

reaction_board/repository.py

@@ -410,12 +410,14 @@ class Repository:
         following: int | None = None,
         now: int | None = None,
         backfilled: bool = False,
+        live_since: int | None = None,
     ) -> int:
         """写入 UP 粉丝快照，并补齐该 UP 尚未记录粉丝数的视频。
 
         ``follower_first`` 只在首次插入时写入；``follower_latest`` 每次刷新都会
-        更新。视频上的 ``up_follower`` 是入库时快照，已有值不会被覆盖。返回本次
-        补齐的视频数。
+        更新。视频上的 ``up_follower`` 是入库时快照，已有值不会被覆盖。传入
+        ``live_since`` 时，只有 ``first_seen >= live_since`` 的视频视为实时采集，
+        更早的视频标记为回填。返回本次补齐的视频数。
         """
         now = self._now() if now is None else int(now)
         mid = int(mid)
@@ -436,14 +438,29 @@ class Repository:
                 """,
                 (mid, name, follower, following_value, now, follower, now),
             )
-            cursor = conn.execute(
-                """
-                UPDATE video
-                SET up_follower = ?, up_follower_at = ?, up_follower_backfilled = ?
-                WHERE up_mid = ? AND up_follower IS NULL
-                """,
-                (follower, now, backfill_flag, mid),
-            )
+            if live_since is None:
+                cursor = conn.execute(
+                    """
+                    UPDATE video
+                    SET up_follower = ?, up_follower_at = ?, up_follower_backfilled = ?
+                    WHERE up_mid = ? AND up_follower IS NULL
+                      AND status IN ('active', 'unfaved')
+                    """,
+                    (follower, now, backfill_flag, mid),
+                )
+            else:
+                cursor = conn.execute(
+                    """
+                    UPDATE video
+                    SET up_follower = ?, up_follower_at = ?,
+                        up_follower_backfilled = CASE
+                          WHEN first_seen >= ? THEN 0 ELSE 1
+                        END
+                    WHERE up_mid = ? AND up_follower IS NULL
+                      AND status IN ('active', 'unfaved')
+                    """,
+                    (follower, now, int(live_since), mid),
+                )
             return max(cursor.rowcount, 0)
 
         return self.db.write(run)

reaction_board/services/folder_sync.py

@@ -21,6 +21,7 @@ from typing import Any
 
 from ..config import AppConfig, ConfigStore, FolderConfig
 from ..repository import Repository
+from .up_profile import UpProfileService
 
 CAPACITY_WARN = 950
 
@@ -61,6 +62,8 @@ class SyncSummary:
     finished_at: int = 0
     folders: list[FolderSyncResult] = field(default_factory=list)
     blacklist_seeded: int = 0
+    followers_updated: int = 0
+    followers_failed: int = 0
 
     @property
     def added(self) -> int:
@@ -100,6 +103,8 @@ class SyncSummary:
             "refreshed": self.refreshed,
             "missing": self.missing,
             "blacklist_seeded": self.blacklist_seeded,
+            "followers_updated": self.followers_updated,
+            "followers_failed": self.followers_failed,
             "folders": [f.as_dict() for f in self.folders],
             "errors": self.errors,
             "warnings": self.warnings,
@@ -119,6 +124,8 @@ class FolderSyncService:
         logger=None,
         capacity_warn: int = CAPACITY_WARN,
         allow_empty_folders: bool = False,
+        up_profile_service: UpProfileService | None = None,
+        capture_followers: bool = True,
     ) -> None:
         if client is None and client_provider is None:
             raise ValueError("必须提供 client 或 client_provider")
@@ -130,6 +137,10 @@ class FolderSyncService:
         self.logger = logger
         self.capacity_warn = capacity_warn
         self.allow_empty_folders = allow_empty_folders
+        self.capture_followers = capture_followers
+        self.up_profile = up_profile_service or UpProfileService(
+            repository, self._client, clock=clock, logger=logger
+        )
 
     def _client(self):
         if self.client_provider is not None:
@@ -189,6 +200,26 @@ class FolderSyncService:
                     )
             summary.folders.append(result)
 
+        if self.capture_followers:
+            try:
+                missing = self.repo.up_mids_missing_follower()
+                if missing:
+                    capture = self.up_profile.capture(
+                        [int(row["up_mid"]) for row in missing],
+                        names={
+                            int(row["up_mid"]): str(row.get("up_name") or "")
+                            for row in missing
+                        },
+                        live_since=summary.started_at - 3600,
+                    )
+                    summary.followers_updated = capture.updated_videos
+                    summary.followers_failed = capture.failed
+            except Exception as exc:  # noqa: BLE001  粉丝采集失败不影响收藏夹同步
+                summary.followers_failed += 1
+                self._log(
+                    "warning", "folder_sync.followers_failed", error=str(exc)
+                )
+
         summary.finished_at = self._now()
         self._log(
             "info" if summary.ok else "error",
@@ -200,7 +231,7 @@ class FolderSyncService:
             errors=len(summary.errors),
             elapsed=max(0, summary.finished_at - summary.started_at),
         )
-        if summary.added or summary.removed or summary.refreshed:
+        if summary.added or summary.removed or summary.refreshed or summary.followers_updated:
             self.repo.bump_cache_version("videos")
         return summary
 

reaction_board/services/up_profile.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+"""UP 粉丝快照采集。
+
+视频的播放、互动记在 bvid 上，而 UP 的粉丝数是视频之外的受众基数。为了让以后
+能计算“播放 / 入库时粉丝数”，本服务在视频入库时记录一次粉丝快照，并支持对历史
+数据做一次性回填。
+
+``up_profile.follower_first`` 永远保留首次采集值；``video.up_follower`` 保留每个
+视频入库时的值，不会被后续刷新覆盖。
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable, Iterable, Mapping
+
+from ..repository import Repository
+
+
+@dataclass
+class CaptureResult:
+    requested: int = 0
+    updated_videos: int = 0
+    failed: int = 0
+    errors: list[str] = field(default_factory=list)
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "requested": self.requested,
+            "updated_videos": self.updated_videos,
+            "failed": self.failed,
+            "errors": self.errors,
+        }
+
+
+class UpProfileService:
+    def __init__(
+        self,
+        repository: Repository,
+        client_provider: Callable[[], Any],
+        *,
+        clock=time.time,
+        logger=None,
+    ) -> None:
+        self.repo = repository
+        self.client_provider = client_provider
+        self._clock = clock
+        self.logger = logger
+
+    def _now(self) -> int:
+        return int(self._clock())
+
+    def _log(self, level: str, event: str, **fields) -> None:
+        if self.logger is not None:
+            getattr(self.logger, level)(event, **fields)
+
+    def capture(
+        self,
+        mids: Iterable[int],
+        *,
+        names: Mapping[int, str] | None = None,
+        live_since: int | None = None,
+        backfilled: bool = False,
+    ) -> CaptureResult:
+        """采集一组 UP 的当前粉丝数，并补齐其视频的空快照。
+
+        同一个 mid 在一次调用里只请求一次；单个失败不会中断其余 UP。
+        """
+        ordered: list[int] = []
+        seen: set[int] = set()
+        for mid in mids:
+            if mid is None or isinstance(mid, bool):
+                continue
+            try:
+                value = int(mid)
+            except (TypeError, ValueError):
+                continue
+            if value <= 0 or value in seen:
+                continue
+            seen.add(value)
+            ordered.append(value)
+
+        result = CaptureResult(requested=len(ordered))
+        if not ordered:
+            return result
+        now = self._now()
+        names = names or {}
+        for mid in ordered:
+            try:
+                fan = self.client_provider().relation_stat(mid)
+                updated = self.repo.apply_up_follower(
+                    mid=mid,
+                    name=names.get(mid) or getattr(fan, "uname", None),
+                    follower=int(fan.follower),
+                    following=int(getattr(fan, "following", 0) or 0),
+                    now=now,
+                    backfilled=backfilled,
+                    live_since=live_since,
+                )
+            except Exception as exc:  # noqa: BLE001  单个 UP 失败不阻塞同步
+                result.failed += 1
+                result.errors.append(f"{mid}: {exc}")
+                self._log(
+                    "warning", "up_profile.capture_failed", mid=mid, error=str(exc)
+                )
+            else:
+                result.updated_videos += updated
+                self._log(
+                    "info", "up_profile.captured",
+                    mid=mid, follower=int(fan.follower), updated=updated,
+                )
+        return result
+
+    def backfill(
+        self, *, limit: int | None = None, dry_run: bool = False
+    ) -> CaptureResult:
+        """为历史视频回填粉丝快照；已有值的视频不会被覆盖。"""
+        rows = self.repo.up_mids_missing_follower()
+        if limit is not None:
+            rows = rows[: max(0, int(limit))]
+        if dry_run:
+            return CaptureResult(requested=len(rows))
+        names = {
+            int(row["up_mid"]): str(row.get("up_name") or "")
+            for row in rows
+        }
+        return self.capture(
+            [int(row["up_mid"]) for row in rows],
+            names=names,
+            backfilled=True,
+        )