feat(db): add video pages and up follower snapshot schema

reaction_board/analytics.py

@@ -35,6 +35,9 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
 
 MIN_VIEW_FOR_INTERACTION = 100
 MIN_VIEW_FOR_GROWTH_RATE = 200
+MIN_EFFICIENCY_DURATION = 600
+DEFAULT_MATURITY_DAYS = 14
+DEFAULT_SHRINK_MINUTES = 120.0
 _IQR_TO_SIGMA = 1.349
 
 
@@ -83,6 +86,128 @@ def _metric_value(row: Mapping[str, object], metric: str) -> float:
         return 0.0
 
 
+def content_hours(duration_seconds: object) -> float | None:
+    """把视频总时长（秒）换算为内容小时；缺失或非正数返回 ``None``。
+
+    多 P 视频的 ``duration`` 已是各 P 之和，因此这里不需要再乘 P 数。
+    """
+    if duration_seconds is None or isinstance(duration_seconds, bool):
+        return None
+    try:
+        seconds = float(duration_seconds)
+    except (TypeError, ValueError):
+        return None
+    if seconds <= 0:
+        return None
+    return seconds / 3600.0
+
+
+def views_per_hour(view: object, duration_seconds: object) -> float | None:
+    """播放/内容小时；时长缺失或非正数时返回 ``None``，不做除零。"""
+    hours = content_hours(duration_seconds)
+    if hours is None:
+        return None
+    return _metric_value({"view": view}, "view") / hours
+
+
+def interactions_per_hour(
+    metrics: Mapping[str, object] | None,
+    duration_seconds: object,
+    *,
+    interaction_metrics: Sequence[str] = ("like", "coin", "favorite"),
+) -> float | None:
+    """互动/内容小时；统计缺失或时长无效时返回 ``None``。"""
+    hours = content_hours(duration_seconds)
+    if hours is None or metrics is None:
+        return None
+    interactions = sum(_metric_value(metrics, metric) for metric in interaction_metrics)
+    return interactions / hours
+
+
+def age_days(pubdate: object, reference: int | float) -> int | None:
+    """发布到参考时点（通常为最新数据日）的整天数；无效发布时间返回 ``None``。"""
+    if pubdate is None or isinstance(pubdate, bool):
+        return None
+    try:
+        published = int(pubdate)
+        now = int(reference)
+    except (TypeError, ValueError):
+        return None
+    if published <= 0:
+        return None
+    return max(0, (now - published) // 86400)
+
+
+def is_mature(
+    pubdate: object,
+    reference: int | float,
+    maturity_days: int = DEFAULT_MATURITY_DAYS,
+) -> bool:
+    """是否已达到成熟窗口；发布时间缺失视为未成熟。"""
+    days = age_days(pubdate, reference)
+    return days is not None and days >= max(0, int(maturity_days))
+
+
+def median_of(values: Iterable[object]) -> float | None:
+    """忽略无效值的中位数；没有有效值返回 ``None``。"""
+    numbers: list[float] = []
+    for value in values:
+        if value is None or isinstance(value, bool):
+            continue
+        try:
+            numbers.append(float(value))
+        except (TypeError, ValueError):
+            continue
+    if not numbers:
+        return None
+    return float(_stat_median(numbers))
+
+
+def effective_shrink_minutes(
+    per_up_minutes: Iterable[object],
+    fallback: float = DEFAULT_SHRINK_MINUTES,
+) -> float:
+    """自适应收缩强度：各 UP 总内容分钟的中位数，缺失时回退固定值。"""
+    positives: list[float] = []
+    for minutes in per_up_minutes:
+        if minutes is None or isinstance(minutes, bool):
+            continue
+        try:
+            value = float(minutes)
+        except (TypeError, ValueError):
+            continue
+        if value > 0:
+            positives.append(value)
+    value = median_of(positives)
+    if value is None or value <= 0:
+        return float(fallback)
+    return value
+
+
+def eb_rate(
+    total: object,
+    exposure: object,
+    prior_rate: float,
+    prior_strength: float,
+) -> float | None:
+    """Gamma–Poisson 经验贝叶斯收缩后的速率。
+
+    ``exposure`` 与 ``prior_strength`` 必须使用同一单位（本项目中为分钟），
+    返回单位与 ``prior_rate`` 一致（每分钟）。
+    """
+    if prior_strength < 0:
+        raise ValueError("prior_strength 不能为负")
+    try:
+        total_value = max(0.0, float(total))
+        exposure_value = max(0.0, float(exposure))
+        prior = max(0.0, float(prior_rate))
+    except (TypeError, ValueError):
+        return None
+    if prior_strength <= 0:
+        return None if exposure_value <= 0 else total_value / exposure_value
+    return (total_value + prior_strength * prior) / (exposure_value + prior_strength)
+
+
 def normalized_weights(weights: Mapping[str, float] | None = None) -> dict[str, float]:
     source = weights or DEFAULT_HEAT_WEIGHTS
     result: dict[str, float] = {}

reaction_board/bilibili/models.py

@@ -30,6 +30,7 @@ class FavResource:
     cover: str | None
     attr: int
     fav_time: int | None
+    pages: int | None = None
 
     @property
     def invalid(self) -> bool:
@@ -51,6 +52,12 @@ class FavResource:
         duration = raw_duration if isinstance(raw_duration, int) and not isinstance(
             raw_duration, bool
         ) else None
+        raw_pages = item.get("page")
+        pages = raw_pages if (
+            isinstance(raw_pages, int)
+            and not isinstance(raw_pages, bool)
+            and raw_pages > 0
+        ) else None
         raw_attr = item.get("attr", 0)
         attr = raw_attr if isinstance(raw_attr, int) and not isinstance(raw_attr, bool) else 0
         fav_time = item.get("fav_time")
@@ -70,6 +77,7 @@ class FavResource:
             cover=str(item.get("cover") or "") or None,
             attr=attr,
             fav_time=fav_time,
+            pages=pages,
         )
 
 

reaction_board/config.py

@@ -38,6 +38,13 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
     "danmaku": 0.02,
 }
 
+DEFAULT_ANALYTICS: dict[str, Any] = {
+    "min_duration_seconds": 600,
+    "maturity_days": 14,
+    "min_view_for_efficiency": 0,
+    "shrink_minutes": None,
+}
+
 DEFAULT_CONFIG: dict[str, Any] = {
     "bilibili": {
         "read": {"mid": 0, "csrf": "", "cookie": ""},
@@ -73,6 +80,7 @@ DEFAULT_CONFIG: dict[str, Any] = {
     "notify": {"min_score": 2, "max_items_per_message": 10},
     "classify": DEFAULT_CLASSIFY,
     "heat_weights": DEFAULT_HEAT_WEIGHTS,
+    "analytics": DEFAULT_ANALYTICS,
 }
 
 # 环境变量只作为 CI / 临时调试的覆盖入口，不作为主路径。
@@ -161,6 +169,14 @@ class DiscoverConfig:
     window_days: int
 
 
+@dataclass(frozen=True)
+class AnalyticsConfig:
+    min_duration_seconds: int
+    maturity_days: int
+    min_view_for_efficiency: int
+    shrink_minutes: int | None
+
+
 @dataclass(frozen=True)
 class AppConfig:
     path: Path
@@ -176,6 +192,7 @@ class AppConfig:
     discover: DiscoverConfig
     classify: Mapping[str, Any]
     heat_weights: Mapping[str, float]
+    analytics: AnalyticsConfig
     raw: Mapping[str, Any] = field(repr=False, compare=False)
 
     def folder_for_year(self, year: int) -> FolderConfig | None:
@@ -431,6 +448,33 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
             raise ConfigError(f"heat_weights.{metric} 必须是非负数字")
         heat[metric] = float(value)
 
+    analytics_raw = data["analytics"]
+    if not isinstance(analytics_raw, Mapping):
+        raise ConfigError("analytics 必须是对象")
+    shrink_raw = analytics_raw.get("shrink_minutes")
+    analytics = AnalyticsConfig(
+        min_duration_seconds=_as_int(
+            analytics_raw.get("min_duration_seconds", 600),
+            "analytics.min_duration_seconds",
+            minimum=0,
+        ),
+        maturity_days=_as_int(
+            analytics_raw.get("maturity_days", 14),
+            "analytics.maturity_days",
+            minimum=0,
+        ),
+        min_view_for_efficiency=_as_int(
+            analytics_raw.get("min_view_for_efficiency", 0),
+            "analytics.min_view_for_efficiency",
+            minimum=0,
+        ),
+        shrink_minutes=(
+            None if shrink_raw is None else _as_int(
+                shrink_raw, "analytics.shrink_minutes", minimum=1
+            )
+        ),
+    )
+
     return AppConfig(
         path=path,
         bilibili=bilibili,
@@ -445,6 +489,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
         discover=discover,
         classify=classify,
         heat_weights=heat,
+        analytics=analytics,
         raw=data,
     )
 

reaction_board/migrations.py

@@ -186,6 +186,27 @@ MIGRATIONS: tuple[Migration, ...] = (
             "ALTER TABLE candidate ADD COLUMN review INTEGER",
         ),
     ),
+    Migration(
+        version=4,
+        name="video pages and up follower snapshot",
+        statements=(
+            "ALTER TABLE video ADD COLUMN pages INTEGER",
+            "ALTER TABLE video ADD COLUMN up_follower INTEGER",
+            "ALTER TABLE video ADD COLUMN up_follower_at INTEGER",
+            "ALTER TABLE video ADD COLUMN up_follower_backfilled INTEGER NOT NULL DEFAULT 0",
+            """
+            CREATE TABLE up_profile (
+              mid               INTEGER PRIMARY KEY,
+              name              TEXT,
+              follower_first    INTEGER,
+              following_first   INTEGER,
+              first_captured_at INTEGER,
+              follower_latest   INTEGER,
+              last_refreshed    INTEGER
+            )
+            """,
+        ),
+    ),
 )
 
 

reaction_board/repository.py

@@ -229,9 +229,9 @@ class Repository:
             conn.execute(
                 """
                 INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate,
-                                  duration, cover, folder_id, fav_time, status,
+                                  duration, pages, cover, folder_id, fav_time, status,
                                   source, first_seen, last_sync, attr)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                 ON CONFLICT(aid) DO UPDATE SET
                   bvid = COALESCE(excluded.bvid, video.bvid),
                   title = CASE
@@ -248,6 +248,7 @@ class Repository:
                     ELSE video.pubdate
                   END,
                   duration = COALESCE(excluded.duration, video.duration),
+                  pages = COALESCE(excluded.pages, video.pages),
                   cover = COALESCE(excluded.cover, video.cover),
                   folder_id = CASE
                     WHEN video.folder_id IS NULL THEN excluded.folder_id
@@ -267,6 +268,7 @@ class Repository:
                     resource.up_name,
                     int(resource.pubdate or 0),
                     resource.duration,
+                    resource.pages,
                     resource.cover,
                     folder_id,
                     resource.fav_time,
@@ -397,6 +399,105 @@ class Repository:
             return [dict(row) for row in rows]
         return self.db.read(run)
 
+    # ------------------------------------------------------- up profile
+
+    def apply_up_follower(
+        self,
+        *,
+        mid: int,
+        name: str | None,
+        follower: int,
+        following: int | None = None,
+        now: int | None = None,
+        backfilled: bool = False,
+    ) -> int:
+        """写入 UP 粉丝快照，并补齐该 UP 尚未记录粉丝数的视频。
+
+        ``follower_first`` 只在首次插入时写入；``follower_latest`` 每次刷新都会
+        更新。视频上的 ``up_follower`` 是入库时快照，已有值不会被覆盖。返回本次
+        补齐的视频数。
+        """
+        now = self._now() if now is None else int(now)
+        mid = int(mid)
+        follower = max(0, int(follower))
+        following_value = None if following is None else max(0, int(following))
+        backfill_flag = 1 if backfilled else 0
+
+        def run(conn: sqlite3.Connection) -> int:
+            conn.execute(
+                """
+                INSERT INTO up_profile(mid, name, follower_first, following_first,
+                                       first_captured_at, follower_latest, last_refreshed)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(mid) DO UPDATE SET
+                  name = COALESCE(NULLIF(excluded.name, ''), up_profile.name),
+                  follower_latest = excluded.follower_latest,
+                  last_refreshed = excluded.last_refreshed
+                """,
+                (mid, name, follower, following_value, now, follower, now),
+            )
+            cursor = conn.execute(
+                """
+                UPDATE video
+                SET up_follower = ?, up_follower_at = ?, up_follower_backfilled = ?
+                WHERE up_mid = ? AND up_follower IS NULL
+                """,
+                (follower, now, backfill_flag, mid),
+            )
+            return max(cursor.rowcount, 0)
+
+        return self.db.write(run)
+
+    def get_up_profile(self, mid: int) -> dict[str, Any] | None:
+        def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
+            row = conn.execute(
+                "SELECT * FROM up_profile WHERE mid = ?", (int(mid),)
+            ).fetchone()
+            return dict(row) if row else None
+        return self.db.read(run)
+
+    def up_mids_missing_follower(self) -> list[dict[str, Any]]:
+        """追踪中但仍有视频缺少入库粉丝数的 UP。"""
+        def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
+            rows = conn.execute(
+                """
+                SELECT up_mid, COALESCE(MAX(up_name), '') AS up_name,
+                       COUNT(*) AS missing
+                FROM video
+                WHERE status IN ('active', 'unfaved')
+                  AND up_mid IS NOT NULL
+                  AND up_follower IS NULL
+                GROUP BY up_mid
+                ORDER BY up_mid
+                """
+            ).fetchall()
+            return [dict(row) for row in rows]
+        return self.db.read(run)
+
+    def up_follower_coverage(self) -> dict[str, int]:
+        """返回追踪视频的粉丝快照覆盖情况。"""
+        def run(conn: sqlite3.Connection) -> dict[str, int]:
+            row = conn.execute(
+                """
+                SELECT COUNT(*) AS total,
+                       SUM(CASE WHEN up_follower IS NOT NULL THEN 1 ELSE 0 END) AS covered,
+                       COUNT(DISTINCT CASE WHEN up_follower IS NOT NULL
+                                           THEN up_mid END) AS ups_covered
+                FROM video
+                WHERE status IN ('active', 'unfaved')
+                  AND up_mid IS NOT NULL
+                """
+            ).fetchone()
+            total = int(row["total"] or 0)
+            covered = int(row["covered"] or 0)
+            return {
+                "total": total,
+                "covered": covered,
+                "missing": max(0, total - covered),
+                "ups_covered": int(row["ups_covered"] or 0),
+            }
+        return self.db.read(run)
+
     def folder_resources(self, media_id: int) -> list[dict[str, Any]]:
         def run(conn: sqlite3.Connection) -> list[dict[str, Any]]:
             rows = conn.execute(