From 4942d1d161a15a7c34026d2c0f922d3d4ca4b699 Mon Sep 17 00:00:00 2001 From: kaireewu Date: Sun, 1 Nov 2026 11:14:36 +0800 Subject: [PATCH] feat(db): add video pages and up follower snapshot schema --- reaction_board/analytics.py | 125 ++++++++++++++++++++++++++++++ reaction_board/bilibili/models.py | 8 ++ reaction_board/config.py | 45 +++++++++++ reaction_board/migrations.py | 21 +++++ reaction_board/repository.py | 105 ++++++++++++++++++++++++- tests/test_analytics.py | 79 ++++++++++++++++++- tests/test_bilibili_client.py | 19 +++++ tests/test_config.py | 45 ++++++++++- tests/test_migrations.py | 30 +++++-- tests/test_repository.py | 57 +++++++++++++- 10 files changed, 521 insertions(+), 13 deletions(-) diff --git a/reaction_board/analytics.py b/reaction_board/analytics.py index b680cb6..73566ed 100644 --- a/reaction_board/analytics.py +++ b/reaction_board/analytics.py @@ -35,6 +35,9 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = { MIN_VIEW_FOR_INTERACTION = 100 MIN_VIEW_FOR_GROWTH_RATE = 200 +MIN_EFFICIENCY_DURATION = 600 +DEFAULT_MATURITY_DAYS = 14 +DEFAULT_SHRINK_MINUTES = 120.0 _IQR_TO_SIGMA = 1.349 @@ -83,6 +86,128 @@ def _metric_value(row: Mapping[str, object], metric: str) -> float: return 0.0 +def content_hours(duration_seconds: object) -> float | None: + """把视频总时长（秒）换算为内容小时；缺失或非正数返回 ``None``。 + + 多 P 视频的 ``duration`` 已是各 P 之和，因此这里不需要再乘 P 数。 + """ + if duration_seconds is None or isinstance(duration_seconds, bool): + return None + try: + seconds = float(duration_seconds) + except (TypeError, ValueError): + return None + if seconds <= 0: + return None + return seconds / 3600.0 + + +def views_per_hour(view: object, duration_seconds: object) -> float | None: + """播放/内容小时；时长缺失或非正数时返回 ``None``，不做除零。""" + hours = content_hours(duration_seconds) + if hours is None: + return None + return _metric_value({"view": view}, "view") / hours + + +def interactions_per_hour( + metrics: Mapping[str, object] | None, + duration_seconds: object, + *, + interaction_metrics: Sequence[str] = ("like", "coin", "favorite"), +) -> float | None: + """互动/内容小时；统计缺失或时长无效时返回 ``None``。""" + hours = content_hours(duration_seconds) + if hours is None or metrics is None: + return None + interactions = sum(_metric_value(metrics, metric) for metric in interaction_metrics) + return interactions / hours + + +def age_days(pubdate: object, reference: int | float) -> int | None: + """发布到参考时点（通常为最新数据日）的整天数；无效发布时间返回 ``None``。""" + if pubdate is None or isinstance(pubdate, bool): + return None + try: + published = int(pubdate) + now = int(reference) + except (TypeError, ValueError): + return None + if published <= 0: + return None + return max(0, (now - published) // 86400) + + +def is_mature( + pubdate: object, + reference: int | float, + maturity_days: int = DEFAULT_MATURITY_DAYS, +) -> bool: + """是否已达到成熟窗口；发布时间缺失视为未成熟。""" + days = age_days(pubdate, reference) + return days is not None and days >= max(0, int(maturity_days)) + + +def median_of(values: Iterable[object]) -> float | None: + """忽略无效值的中位数；没有有效值返回 ``None``。""" + numbers: list[float] = [] + for value in values: + if value is None or isinstance(value, bool): + continue + try: + numbers.append(float(value)) + except (TypeError, ValueError): + continue + if not numbers: + return None + return float(_stat_median(numbers)) + + +def effective_shrink_minutes( + per_up_minutes: Iterable[object], + fallback: float = DEFAULT_SHRINK_MINUTES, +) -> float: + """自适应收缩强度：各 UP 总内容分钟的中位数，缺失时回退固定值。""" + positives: list[float] = [] + for minutes in per_up_minutes: + if minutes is None or isinstance(minutes, bool): + continue + try: + value = float(minutes) + except (TypeError, ValueError): + continue + if value > 0: + positives.append(value) + value = median_of(positives) + if value is None or value <= 0: + return float(fallback) + return value + + +def eb_rate( + total: object, + exposure: object, + prior_rate: float, + prior_strength: float, +) -> float | None: + """Gamma–Poisson 经验贝叶斯收缩后的速率。 + + ``exposure`` 与 ``prior_strength`` 必须使用同一单位（本项目中为分钟）， + 返回单位与 ``prior_rate`` 一致（每分钟）。 + """ + if prior_strength < 0: + raise ValueError("prior_strength 不能为负") + try: + total_value = max(0.0, float(total)) + exposure_value = max(0.0, float(exposure)) + prior = max(0.0, float(prior_rate)) + except (TypeError, ValueError): + return None + if prior_strength <= 0: + return None if exposure_value <= 0 else total_value / exposure_value + return (total_value + prior_strength * prior) / (exposure_value + prior_strength) + + def normalized_weights(weights: Mapping[str, float] | None = None) -> dict[str, float]: source = weights or DEFAULT_HEAT_WEIGHTS result: dict[str, float] = {} diff --git a/reaction_board/bilibili/models.py b/reaction_board/bilibili/models.py index ec56f5f..6530b01 100644 --- a/reaction_board/bilibili/models.py +++ b/reaction_board/bilibili/models.py @@ -30,6 +30,7 @@ class FavResource: cover: str | None attr: int fav_time: int | None + pages: int | None = None @property def invalid(self) -> bool: @@ -51,6 +52,12 @@ class FavResource: duration = raw_duration if isinstance(raw_duration, int) and not isinstance( raw_duration, bool ) else None + raw_pages = item.get("page") + pages = raw_pages if ( + isinstance(raw_pages, int) + and not isinstance(raw_pages, bool) + and raw_pages > 0 + ) else None raw_attr = item.get("attr", 0) attr = raw_attr if isinstance(raw_attr, int) and not isinstance(raw_attr, bool) else 0 fav_time = item.get("fav_time") @@ -70,6 +77,7 @@ class FavResource: cover=str(item.get("cover") or "") or None, attr=attr, fav_time=fav_time, + pages=pages, ) diff --git a/reaction_board/config.py b/reaction_board/config.py index 89acd26..7ed7fd7 100644 --- a/reaction_board/config.py +++ b/reaction_board/config.py @@ -38,6 +38,13 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = { "danmaku": 0.02, } +DEFAULT_ANALYTICS: dict[str, Any] = { + "min_duration_seconds": 600, + "maturity_days": 14, + "min_view_for_efficiency": 0, + "shrink_minutes": None, +} + DEFAULT_CONFIG: dict[str, Any] = { "bilibili": { "read": {"mid": 0, "csrf": "", "cookie": ""}, @@ -73,6 +80,7 @@ DEFAULT_CONFIG: dict[str, Any] = { "notify": {"min_score": 2, "max_items_per_message": 10}, "classify": DEFAULT_CLASSIFY, "heat_weights": DEFAULT_HEAT_WEIGHTS, + "analytics": DEFAULT_ANALYTICS, } # 环境变量只作为 CI / 临时调试的覆盖入口，不作为主路径。 @@ -161,6 +169,14 @@ class DiscoverConfig: window_days: int +@dataclass(frozen=True) +class AnalyticsConfig: + min_duration_seconds: int + maturity_days: int + min_view_for_efficiency: int + shrink_minutes: int | None + + @dataclass(frozen=True) class AppConfig: path: Path @@ -176,6 +192,7 @@ class AppConfig: discover: DiscoverConfig classify: Mapping[str, Any] heat_weights: Mapping[str, float] + analytics: AnalyticsConfig raw: Mapping[str, Any] = field(repr=False, compare=False) def folder_for_year(self, year: int) -> FolderConfig | None: @@ -431,6 +448,33 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig: raise ConfigError(f"heat_weights.{metric} 必须是非负数字") heat[metric] = float(value) + analytics_raw = data["analytics"] + if not isinstance(analytics_raw, Mapping): + raise ConfigError("analytics 必须是对象") + shrink_raw = analytics_raw.get("shrink_minutes") + analytics = AnalyticsConfig( + min_duration_seconds=_as_int( + analytics_raw.get("min_duration_seconds", 600), + "analytics.min_duration_seconds", + minimum=0, + ), + maturity_days=_as_int( + analytics_raw.get("maturity_days", 14), + "analytics.maturity_days", + minimum=0, + ), + min_view_for_efficiency=_as_int( + analytics_raw.get("min_view_for_efficiency", 0), + "analytics.min_view_for_efficiency", + minimum=0, + ), + shrink_minutes=( + None if shrink_raw is None else _as_int( + shrink_raw, "analytics.shrink_minutes", minimum=1 + ) + ), + ) + return AppConfig( path=path, bilibili=bilibili, @@ -445,6 +489,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig: discover=discover, classify=classify, heat_weights=heat, + analytics=analytics, raw=data, ) diff --git a/reaction_board/migrations.py b/reaction_board/migrations.py index 8f304d4..a2c0a7d 100644 --- a/reaction_board/migrations.py +++ b/reaction_board/migrations.py @@ -186,6 +186,27 @@ MIGRATIONS: tuple[Migration, ...] = ( "ALTER TABLE candidate ADD COLUMN review INTEGER", ), ), + Migration( + version=4, + name="video pages and up follower snapshot", + statements=( + "ALTER TABLE video ADD COLUMN pages INTEGER", + "ALTER TABLE video ADD COLUMN up_follower INTEGER", + "ALTER TABLE video ADD COLUMN up_follower_at INTEGER", + "ALTER TABLE video ADD COLUMN up_follower_backfilled INTEGER NOT NULL DEFAULT 0", + """ + CREATE TABLE up_profile ( + mid INTEGER PRIMARY KEY, + name TEXT, + follower_first INTEGER, + following_first INTEGER, + first_captured_at INTEGER, + follower_latest INTEGER, + last_refreshed INTEGER + ) + """, + ), + ), ) diff --git a/reaction_board/repository.py b/reaction_board/repository.py index b7a6dab..ffb451a 100644 --- a/reaction_board/repository.py +++ b/reaction_board/repository.py @@ -229,9 +229,9 @@ class Repository: conn.execute( """ INSERT INTO video(aid, bvid, title, up_mid, up_name, pubdate, - duration, cover, folder_id, fav_time, status, + duration, pages, cover, folder_id, fav_time, status, source, first_seen, last_sync, attr) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(aid) DO UPDATE SET bvid = COALESCE(excluded.bvid, video.bvid), title = CASE @@ -248,6 +248,7 @@ class Repository: ELSE video.pubdate END, duration = COALESCE(excluded.duration, video.duration), + pages = COALESCE(excluded.pages, video.pages), cover = COALESCE(excluded.cover, video.cover), folder_id = CASE WHEN video.folder_id IS NULL THEN excluded.folder_id @@ -267,6 +268,7 @@ class Repository: resource.up_name, int(resource.pubdate or 0), resource.duration, + resource.pages, resource.cover, folder_id, resource.fav_time, @@ -397,6 +399,105 @@ class Repository: return [dict(row) for row in rows] return self.db.read(run) + # ------------------------------------------------------- up profile + + def apply_up_follower( + self, + *, + mid: int, + name: str | None, + follower: int, + following: int | None = None, + now: int | None = None, + backfilled: bool = False, + ) -> int: + """写入 UP 粉丝快照，并补齐该 UP 尚未记录粉丝数的视频。 + + ``follower_first`` 只在首次插入时写入；``follower_latest`` 每次刷新都会 + 更新。视频上的 ``up_follower`` 是入库时快照，已有值不会被覆盖。返回本次 + 补齐的视频数。 + """ + now = self._now() if now is None else int(now) + mid = int(mid) + follower = max(0, int(follower)) + following_value = None if following is None else max(0, int(following)) + backfill_flag = 1 if backfilled else 0 + + def run(conn: sqlite3.Connection) -> int: + conn.execute( + """ + INSERT INTO up_profile(mid, name, follower_first, following_first, + first_captured_at, follower_latest, last_refreshed) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(mid) DO UPDATE SET + name = COALESCE(NULLIF(excluded.name, ''), up_profile.name), + follower_latest = excluded.follower_latest, + last_refreshed = excluded.last_refreshed + """, + (mid, name, follower, following_value, now, follower, now), + ) + cursor = conn.execute( + """ + UPDATE video + SET up_follower = ?, up_follower_at = ?, up_follower_backfilled = ? + WHERE up_mid = ? AND up_follower IS NULL + """, + (follower, now, backfill_flag, mid), + ) + return max(cursor.rowcount, 0) + + return self.db.write(run) + + def get_up_profile(self, mid: int) -> dict[str, Any] | None: + def run(conn: sqlite3.Connection) -> dict[str, Any] | None: + row = conn.execute( + "SELECT * FROM up_profile WHERE mid = ?", (int(mid),) + ).fetchone() + return dict(row) if row else None + return self.db.read(run) + + def up_mids_missing_follower(self) -> list[dict[str, Any]]: + """追踪中但仍有视频缺少入库粉丝数的 UP。""" + def run(conn: sqlite3.Connection) -> list[dict[str, Any]]: + rows = conn.execute( + """ + SELECT up_mid, COALESCE(MAX(up_name), '') AS up_name, + COUNT(*) AS missing + FROM video + WHERE status IN ('active', 'unfaved') + AND up_mid IS NOT NULL + AND up_follower IS NULL + GROUP BY up_mid + ORDER BY up_mid + """ + ).fetchall() + return [dict(row) for row in rows] + return self.db.read(run) + + def up_follower_coverage(self) -> dict[str, int]: + """返回追踪视频的粉丝快照覆盖情况。""" + def run(conn: sqlite3.Connection) -> dict[str, int]: + row = conn.execute( + """ + SELECT COUNT(*) AS total, + SUM(CASE WHEN up_follower IS NOT NULL THEN 1 ELSE 0 END) AS covered, + COUNT(DISTINCT CASE WHEN up_follower IS NOT NULL + THEN up_mid END) AS ups_covered + FROM video + WHERE status IN ('active', 'unfaved') + AND up_mid IS NOT NULL + """ + ).fetchone() + total = int(row["total"] or 0) + covered = int(row["covered"] or 0) + return { + "total": total, + "covered": covered, + "missing": max(0, total - covered), + "ups_covered": int(row["ups_covered"] or 0), + } + return self.db.read(run) + def folder_resources(self, media_id: int) -> list[dict[str, Any]]: def run(conn: sqlite3.Connection) -> list[dict[str, Any]]: rows = conn.execute( diff --git a/tests/test_analytics.py b/tests/test_analytics.py index 4295a90..18135e7 100644 --- a/tests/test_analytics.py +++ b/tests/test_analytics.py @@ -10,15 +10,23 @@ sys.stdout.reconfigure(encoding="utf-8") from reaction_board.analytics import ( # noqa: E402 DEFAULT_HEAT_WEIGHTS, METRICS, + age_days, compute_growth, + content_hours, + eb_rate, + effective_shrink_minutes, heat_scores, + interactions_per_hour, interaction_rate, is_loopback, + is_mature, + median_of, normalize_cover, normalized_weights, percentile, robust_center_scale, score_percentiles, + views_per_hour, ) @@ -157,6 +165,72 @@ def test_normalize_cover_and_hosts(): assert not is_loopback("192.168.1.2") +def test_content_efficiency_helpers(): + assert content_hours(3600) == 1.0 + assert content_hours(7200) == 2.0 + assert content_hours(0) is None + assert content_hours(-1) is None + assert content_hours(None) is None + assert content_hours(True) is None + assert content_hours("bad") is None + + assert views_per_hour(1000, 3600) == 1000.0 + assert views_per_hour(1000, 7200) == 500.0 + assert views_per_hour(None, 3600) == 0.0 + assert views_per_hour(1000, None) is None + assert views_per_hour(1000, 0) is None + + metrics = {"like": 30, "coin": 10, "favorite": 20, "view": 999} + assert interactions_per_hour(metrics, 3600) == 60.0 + assert interactions_per_hour(metrics, 7200) == 30.0 + assert interactions_per_hour(None, 3600) is None + assert interactions_per_hour(metrics, None) is None + # 单项缺失按 0 处理，不抛异常 + assert interactions_per_hour({"like": None, "coin": "bad"}, 3600) == 0.0 + + +def test_age_and_maturity(): + reference = 1_800_000_000 + assert age_days(reference - 86400 * 10, reference) == 10 + assert age_days(reference, reference) == 0 + # 未来时间不产生负年龄 + assert age_days(reference + 86400, reference) == 0 + assert age_days(None, reference) is None + assert age_days(0, reference) is None + assert age_days("bad", reference) is None + + assert is_mature(reference - 86400 * 14, reference, 14) is True + assert is_mature(reference - 86400 * 13, reference, 14) is False + assert is_mature(None, reference, 14) is False + assert is_mature(reference - 86400 * 14, reference, 0) is True + + +def test_shrinkage_helpers(): + assert median_of([]) is None + assert median_of([None, "bad", True]) is None + assert median_of([1, 3, 2]) == 2.0 + + # 自适应 κ 取各 UP 总分钟的中位数 + assert effective_shrink_minutes([10, 30, 20]) == 20.0 + assert effective_shrink_minutes([0, -5, None]) == 120.0 + assert effective_shrink_minutes([]) == 120.0 + assert effective_shrink_minutes([], fallback=60) == 60.0 + + # κ 越大越靠近先验；exposure 为 0 时直接返回先验 + assert abs(eb_rate(0, 0, 0.5, 100) - 0.5) < 1e-12 + assert eb_rate(100, 100, 1.0, 0) == 1.0 + assert eb_rate(100, 0, 1.0, 0) is None + # (100 + 100*0.5) / (100 + 100) = 0.75 + assert abs(eb_rate(100, 100, 0.5, 100) - 0.75) < 1e-12 + assert eb_rate(None, 100, 0.5, 100) is None + try: + eb_rate(1, 1, 0.5, -1) + except ValueError: + pass + else: + raise AssertionError("负 prior_strength 必须报错") + + def main_test(): test_percentile_and_robust_scale() test_normalized_weights() @@ -166,7 +240,10 @@ def main_test(): test_interaction_rate_thresholds() test_compute_growth() test_normalize_cover_and_hosts() - print("OK: analytics 热度/增长/互动率/封面测试全部通过") + test_content_efficiency_helpers() + test_age_and_maturity() + test_shrinkage_helpers() + print("OK: analytics 热度/增长/互动率/时长效率/收缩测试全部通过") if __name__ == "__main__": diff --git a/tests/test_bilibili_client.py b/tests/test_bilibili_client.py index 63e42ab..3b293a6 100644 --- a/tests/test_bilibili_client.py +++ b/tests/test_bilibili_client.py @@ -216,6 +216,25 @@ def test_infos_invalid_attr_and_empty(): assert len(transport.calls) == 1 +def test_infos_parses_page_count(): + payload = {"code": 0, "message": "0", "ttl": 1, "data": [VALID_INFO, INVALID_INFO]} + client, _ = make_client(QueueTransport([http_json(payload)])) + result = client.fav_infos( + 3700023351, [VALID_INFO["id"], INVALID_INFO["id"]] + ) + assert result.items[VALID_INFO["id"]].pages == 2 + assert result.items[INVALID_INFO["id"]].pages == 1 + + # page 字段缺失或非法时保持 None，不把 0 当分 P 数 + broken = json.loads(json.dumps(VALID_INFO)) + broken["id"] = 999 + broken["page"] = 0 + client, _ = make_client(QueueTransport([http_json({ + "code": 0, "message": "0", "ttl": 1, "data": [broken], + })])) + assert client.fav_infos(3700023351, [999]).items[999].pages is None + + def test_infos_unexpected_aid_is_protocol_error(): transport = QueueTransport([http_json({ "code": 0, "message": "0", "ttl": 1, "data": [VALID_INFO], diff --git a/tests/test_config.py b/tests/test_config.py index f883f4d..c986f9b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -126,12 +126,55 @@ def test_invalid_values(): raise AssertionError("非法环境变量必须拒绝") +def test_analytics_defaults_and_validation(): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "config.json" + write(path, base_config()) + cfg = load_config(path, env={}) + assert cfg.analytics.min_duration_seconds == 600 + assert cfg.analytics.maturity_days == 14 + assert cfg.analytics.min_view_for_efficiency == 0 + assert cfg.analytics.shrink_minutes is None + + data = base_config() + data["analytics"] = { + "min_duration_seconds": 300, + "maturity_days": 7, + "min_view_for_efficiency": 100, + "shrink_minutes": 90, + } + write(path, data) + cfg = load_config(path, env={}) + assert cfg.analytics.min_duration_seconds == 300 + assert cfg.analytics.maturity_days == 7 + assert cfg.analytics.min_view_for_efficiency == 100 + assert cfg.analytics.shrink_minutes == 90 + + for bad, hint in ( + ({"min_duration_seconds": -1}, "min_duration_seconds"), + ({"maturity_days": -1}, "maturity_days"), + ({"min_view_for_efficiency": -1}, "min_view_for_efficiency"), + ({"shrink_minutes": 0}, "shrink_minutes"), + ({"shrink_minutes": "auto"}, "shrink_minutes"), + ): + data = base_config() + data["analytics"] = bad + write(path, data) + try: + load_config(path, env={}) + except ConfigError as exc: + assert hint in str(exc), (bad, str(exc)) + else: + raise AssertionError(f"非法 analytics 必须拒绝: {bad}") + + def main(): test_dual_accounts_and_env_override() test_flat_and_missing_account_rejected() test_validation_require_read_auth_and_hot_reload() test_invalid_values() - print("OK: 双账号配置加载、拒绝 flat 格式、ENV 覆盖与热加载测试全部通过") + test_analytics_defaults_and_validation() + print("OK: 双账号配置加载、拒绝 flat 格式、ENV 覆盖、热加载与 analytics 测试全部通过") if __name__ == "__main__": diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 44aaaed..311a93b 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -21,7 +21,7 @@ from reaction_board.migrations import ( # noqa: E402 EXPECTED_TABLES = { "video", "snapshot", "folder", "folder_resource", "up_blacklist", - "up_stat", "candidate", "seen_aid", "notify", "fav_action", "kv", + "up_stat", "up_profile", "candidate", "seen_aid", "notify", "fav_action", "kv", } @@ -37,8 +37,8 @@ def test_fresh_and_idempotent(): conn = sqlite3.connect(Path(tmp) / "app.db") try: applied = run_migrations(conn) - assert applied == [1, 2, 3] - assert current_version(conn) == 3 + assert applied == [1, 2, 3, 4] + assert current_version(conn) == 4 assert EXPECTED_TABLES <= table_names(conn) assert "schema_version" in table_names(conn) columns = { @@ -49,10 +49,24 @@ def test_fresh_and_idempotent(): row[1] for row in conn.execute("PRAGMA table_info(candidate)") } assert {"duration", "play", "like", "favorite", "danmaku", "review"} <= candidate_columns + video_columns = { + row[1] for row in conn.execute("PRAGMA table_info(video)") + } + assert { + "pages", "duration", "up_follower", "up_follower_at", + "up_follower_backfilled", + } <= video_columns + profile_columns = { + row[1] for row in conn.execute("PRAGMA table_info(up_profile)") + } + assert { + "mid", "follower_first", "following_first", "first_captured_at", + "follower_latest", "last_refreshed", + } <= profile_columns # 再跑一次不应重复应用 assert run_migrations(conn) == [] row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone() - assert row[0] == 3 + assert row[0] == 4 finally: conn.close() @@ -67,15 +81,15 @@ def test_backup_before_upgrade(): conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')") conn.commit() - v4 = Migration(4, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",)) + v5 = Migration(5, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",)) applied = run_migrations( conn, backup_dir=backup_dir, now=1_800_000_000, - migrations=MIGRATIONS + (v4,), + migrations=MIGRATIONS + (v5,), ) - assert applied == [4] - backups = list(backup_dir.glob("app-pre-v3-*.db")) + assert applied == [5] + backups = list(backup_dir.glob("app-pre-v4-*.db")) assert len(backups) == 1 check = sqlite3.connect(backups[0]) try: diff --git a/tests/test_repository.py b/tests/test_repository.py index 98cade6..fb4a5b0 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -18,7 +18,8 @@ F1 = FolderConfig(year=2025, media_id=4451053851, title="2025") F2 = FolderConfig(year=2026, media_id=3700023351, title="2026") -def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=None): +def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=None, + pages=2): return FavResource( aid=aid, bvid=bvid or f"BV{aid}", @@ -30,6 +31,7 @@ def resource(aid, *, attr=0, up_mid=100, title="视频", bvid=None, fav_time=Non cover="http://example/cover.jpg", attr=attr, fav_time=fav_time, + pages=pages, ) @@ -54,6 +56,7 @@ def test_upsert_and_folder_resource(): assert video["folder_id"] == F1.media_id assert video["source"] == f"fav:{F1.media_id}" assert video["first_seen"] == 1_800_000_000 + assert video["pages"] == 2 assert repo.folder_active_aids(F1.media_id) == {1} # 幂等：重复导入不新增行、不改 first_seen @@ -293,6 +296,57 @@ def test_pending_snapshot_aids(): db.close() +def test_up_follower_snapshot_and_backfill(): + with tempfile.TemporaryDirectory() as tmp: + db, repo = setup(tmp) + try: + repo.upsert_from_fav(resource(1, up_mid=100), F1.media_id, was_added=True) + repo.upsert_from_fav(resource(2, up_mid=100), F1.media_id, was_added=True) + repo.upsert_from_fav(resource(3, up_mid=200), F1.media_id, was_added=True) + missing = {row["up_mid"] for row in repo.up_mids_missing_follower()} + assert missing == {100, 200} + + now = 1_800_000_000 + assert repo.apply_up_follower( + mid=100, name="UP", follower=1234, following=5, now=now + ) == 2 + first = repo.get_video(1) + assert first["up_follower"] == 1234 + assert first["up_follower_at"] == now + assert first["up_follower_backfilled"] == 0 + profile = repo.get_up_profile(100) + assert profile["follower_first"] == 1234 + assert profile["follower_latest"] == 1234 + assert { + row["up_mid"] for row in repo.up_mids_missing_follower() + } == {200} + + # 同一 UP 后来新增的视频记录当下的粉丝快照，不覆盖旧视频 + repo.upsert_from_fav(resource(4, up_mid=100), F1.media_id, was_added=True) + repo.apply_up_follower( + mid=100, name="UP", follower=2000, following=6, now=now + 60 + ) + assert repo.get_video(4)["up_follower"] == 2000 + assert repo.get_video(1)["up_follower"] == 1234 + profile = repo.get_up_profile(100) + assert profile["follower_first"] == 1234 + assert profile["follower_latest"] == 2000 + + # 回填只补空值，并标记 backfilled + assert repo.apply_up_follower( + mid=200, name="UP2", follower=88, now=now + 120, backfilled=True + ) == 1 + assert repo.get_video(3)["up_follower"] == 88 + assert repo.get_video(3)["up_follower_backfilled"] == 1 + + coverage = repo.up_follower_coverage() + assert coverage == { + "total": 4, "covered": 4, "missing": 0, "ups_covered": 2, + } + finally: + db.close() + + def main(): test_upsert_and_folder_resource() test_invalid_attr_and_missing() @@ -305,6 +359,7 @@ def main(): test_primary_folder_is_order_independent() test_dropped_invalid_preserves_original_title() test_pending_snapshot_aids() + test_up_follower_snapshot_and_backfill() print("OK: 收藏夹/视频状态机与黑名单测试全部通过")