# -*- coding: utf-8 -*- """每日快照与粉丝采集测试（transport 边界 Mock，不联网）: uv run python tests/test_snapshot.py """ import io import json import os import sys import tempfile import urllib.parse from pathlib import Path sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.stdout.reconfigure(encoding="utf-8") sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from fixtures import ( # noqa: E402 NAV_PAYLOAD, RELATION_PAYLOAD, VIEW_UNAVAILABLE, view_for, ) from reaction_board.bilibili.client import ( # noqa: E402 BilibiliClient, HTTPResponse, Pacer, ) from reaction_board.bilibili.models import FavResource # noqa: E402 from reaction_board.config import ConfigStore, FolderConfig # noqa: E402 from reaction_board.database import Database # noqa: E402 from reaction_board.logging_json import JsonLogger # noqa: E402 from reaction_board.repository import Repository # noqa: E402 from reaction_board.services.snapshot import SnapshotService # noqa: E402 DATE = "2026-10-31" FOLDER = FolderConfig(year=2026, media_id=3700023351, title="2026") def http_json(payload): return HTTPResponse( 200, {"content-type": "application/json"}, json.dumps(payload, ensure_ascii=False).encode("utf-8"), ) class SnapshotTransport: def __init__(self, views, *, nav=None, fan=None): self.views = views self.nav = NAV_PAYLOAD if nav is None else nav self.fan = RELATION_PAYLOAD if fan is None else fan self.calls = [] def request(self, method, url, headers, data=None, timeout=30): self.calls.append(url) if "/x/web-interface/nav" in url: return http_json(self.nav) if "/x/web-interface/view" in url: aid = int(urllib.parse.parse_qs( urllib.parse.urlparse(url).query )["aid"][0]) payload = self.views[aid] if isinstance(payload, BaseException): raise payload if isinstance(payload, HTTPResponse): return payload return http_json(payload) if "/x/relation/stat" in url: return http_json(self.fan) raise AssertionError(f"未预期的 URL: {url}") def count(self, part): return sum(1 for url in self.calls if part in url) def make_service(tmp, transport, *, track_fans=(37754047,), logger=None, retries=3, cookie="SESSDATA=secret; buvid3=device"): path = Path(tmp) / "config.json" path.write_text(json.dumps({ "bilibili": { "read": {"mid": 28802351, "csrf": "csrf", "cookie": cookie}, "write": {"mid": 2, "csrf": "write-csrf", "cookie": "write-cookie"}, }, "storage": { "database_path": str(Path(tmp) / "app.db"), "backup_dir": str(Path(tmp) / "backup"), "backup_keep_days": 14, }, "folders": [], "track_fans": list(track_fans), }, ensure_ascii=False), encoding="utf-8") store = ConfigStore(path, env={}, require_auth=False) db = Database(Path(tmp) / "app.db", backup_dir=Path(tmp) / "backup") db.initialize() repo = Repository(db, clock=lambda: 1_800_000_000) repo.ensure_folder(FOLDER) client = BilibiliClient( cookie=cookie, csrf="csrf", mid=28802351, transport=transport, pacer=Pacer(0, 0), retries=1, ) service = SnapshotService( store, repo, client, clock=lambda: 1_800_000_000, sleeper=lambda seconds: None, logger=logger, retries=retries, backoff=0, ) return db, repo, service def add_video(repo, aid, *, status="active"): repo.upsert_from_fav( FavResource( aid=aid, bvid=f"BV{aid}", title=f"视频{aid}", up_mid=100, up_name="UP", pubdate=1_700_000_000, duration=600, cover=None, attr=0, fav_time=None, ), FOLDER.media_id, was_added=True, ) if status != "active": repo.set_video_status(aid, status) def test_snapshot_success_resume_and_force(): with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({ 1: view_for(1, view=100, like=10, coin=2, favorite=3, share=1, reply=4, danmaku=5), 2: view_for(2, view=200, like=20, coin=4, favorite=6, share=2, reply=8, danmaku=10), }) db, repo, service = make_service(tmp, transport) try: add_video(repo, 1) add_video(repo, 2) result = service.run(date=DATE) assert result.completed is True assert result.total == 2 assert result.success == 2 assert result.failed == 0 assert result.fans_ok == 1 assert result.fans_failed == 0 assert result.auth_ok is True assert Path(result.backup_path or "").exists() row = repo.get_snapshot(DATE, 1) assert (row["view"], row["like"], row["favorite"], row["coin"], row["share"], row["reply"], row["danmaku"]) == (100, 10, 3, 2, 1, 4, 5) assert row["ok"] == 1 and row["src"] == "view" assert repo.get_video(1)["up_face"] == ( "http://i1.hdslb.com/bfs/face/example.jpg" ) fan = repo.get_up_stat(DATE, 37754047) assert fan["follower"] == 2767268 and fan["ok"] == 1 assert repo.kv_get(service.completed_key(DATE)) == "1" assert repo.kv_get("snapshot:last_completed") == DATE assert repo.snapshot_counts(DATE) == {"total": 2, "ok": 2, "failed": 0} # 已完成 -> 不再请求 view_before = transport.count("/web-interface/view") again = service.run_if_needed(date=DATE) assert again.already_completed is True assert transport.count("/web-interface/view") == view_before # force 重新采集 forced = service.run(date=DATE, force=True) assert forced.success == 2 assert transport.count("/web-interface/view") == view_before + 2 finally: db.close() def test_resume_skips_ok_and_retries_failed(): with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({ 1: view_for(1, view=111), 2: view_for(2, view=222), }) db, repo, service = make_service(tmp, transport) try: add_video(repo, 1) add_video(repo, 2) repo.upsert_snapshot( DATE, 1, metrics={"view": 1, "like": 1, "favorite": 1, "coin": 1, "share": 1, "reply": 1, "danmaku": 1}, ok=True, err=None, captured_at=1, ) repo.upsert_snapshot( DATE, 2, metrics=None, ok=False, err="previous error", captured_at=1, ) result = service.run(date=DATE) assert result.skipped == 1 assert result.success == 1 assert result.completed is True assert transport.count("/web-interface/view") == 1 assert repo.get_snapshot(DATE, 1)["view"] == 1 # 未被覆盖 assert repo.get_snapshot(DATE, 2)["view"] == 222 finally: db.close() def test_unavailable_marks_removed(): with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({ 1: VIEW_UNAVAILABLE, 2: HTTPResponse(404, {}, b"not found"), }) db, repo, service = make_service(tmp, transport, track_fans=()) try: add_video(repo, 1) add_video(repo, 2) result = service.run(date=DATE) assert result.removed == 2 assert result.failed == 0 assert result.completed is True assert repo.get_video(1)["status"] == "removed" assert repo.get_video(2)["status"] == "removed" assert repo.get_snapshot(DATE, 1) is None assert repo.get_snapshot(DATE, 2) is None finally: db.close() def test_other_error_writes_failed_placeholder(): with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({ 1: {"code": -500, "message": "服务异常", "data": None}, }) db, repo, service = make_service(tmp, transport, track_fans=(), retries=3) try: add_video(repo, 1) result = service.run(date=DATE) assert result.failed == 1 assert result.success == 0 assert result.completed is False row = repo.get_snapshot(DATE, 1) assert row["ok"] == 0 assert row["view"] is None and row["like"] is None assert "code=-500" in row["err"] assert repo.kv_get(service.completed_key(DATE)) == "0" # 每次业务失败会走匿名+完整 Cookie 两条请求，服务层重试 3 次 assert transport.count("/web-interface/view") == 6 finally: db.close() def test_fan_failure_placeholder(): with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({}, fan={"code": -500, "message": "x", "data": None}) db, repo, service = make_service(tmp, transport, retries=2) try: # 合并跑：视频完成日不受粉丝失败影响 result = service.run(date=DATE) assert result.total == 0 assert result.fans_failed == 1 assert result.completed is True assert repo.kv_get("snapshot:last_completed") == DATE row = repo.get_up_stat(DATE, 37754047) assert row["ok"] == 0 assert row["follower"] is None and row["following"] is None fans_only = service.run_fans(date=DATE, force=True) assert fans_only.fans_failed == 1 assert fans_only.completed is False finally: db.close() def test_auth_invalid_is_warned_but_snapshot_continues(): with tempfile.TemporaryDirectory() as tmp: stream = io.StringIO() logger = JsonLogger(stream) transport = SnapshotTransport( {}, nav={"code": 0, "message": "0", "ttl": 1, "data": {"isLogin": False, "mid": 0, "uname": ""}}, fan=RELATION_PAYLOAD, ) db, repo, service = make_service(tmp, transport, logger=logger) try: result = service.run(date=DATE) assert result.auth_ok is False assert result.completed is True assert repo.kv_get("auth:ok") == "0" events = [json.loads(line) for line in stream.getvalue().splitlines()] assert any(e["event"] == "auth.invalid" for e in events) finally: db.close() def test_no_cookie_skips_nav_request(): with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({}, fan=RELATION_PAYLOAD) db, repo, service = make_service(tmp, transport, cookie="") try: result = service.run(date=DATE) assert result.auth_ok is False assert result.auth_message == "未配置 cookie" assert transport.count("/web-interface/nav") == 0 finally: db.close() def test_force_failure_clears_last_completed(): with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({1: view_for(1, view=100)}) db, repo, service = make_service(tmp, transport, track_fans=(), retries=1) try: add_video(repo, 1) first = service.run(date=DATE) assert first.completed is True assert repo.kv_get("snapshot:last_completed") == DATE transport.views[1] = {"code": -500, "message": "x", "data": None} second = service.run(date=DATE, force=True) assert second.failed == 1 assert second.completed is False assert repo.kv_get("snapshot:last_completed") == "" assert repo.get_snapshot(DATE, 1)["ok"] == 0 finally: db.close() def test_new_video_after_completion_backfills_same_day(): with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({1: view_for(1, view=100)}) db, repo, service = make_service(tmp, transport, track_fans=()) try: add_video(repo, 1) first = service.run(date=DATE) assert first.completed is True assert transport.count("/web-interface/view") == 1 # 当天 11:00 新入库：完成标记仍是 1，但必须补采新视频 add_video(repo, 2) transport.views[2] = view_for(2, view=222) second = service.run_if_needed(date=DATE) assert second.already_completed is False assert second.success == 1 assert second.skipped == 1 assert transport.count("/web-interface/view") == 2 assert repo.get_snapshot(DATE, 2)["view"] == 222 assert repo.get_snapshot(DATE, 1)["view"] == 100 finally: db.close() def test_fan_resume_preserves_successful_value(): with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport( {1: {"code": -500, "message": "x", "data": None}}, ) db, repo, service = make_service(tmp, transport, retries=1) try: add_video(repo, 1) first = service.run(date=DATE) assert first.failed == 1 assert first.fans_ok == 1 assert repo.get_up_stat(DATE, 37754047)["follower"] == 2767268 transport.views[1] = view_for(1, view=111) transport.fan = {"code": -500, "message": "x", "data": None} second = service.run(date=DATE) assert second.fans_skipped == 1 assert second.fans_failed == 0 fan = repo.get_up_stat(DATE, 37754047) assert fan["follower"] == 2767268 assert fan["ok"] == 1 finally: db.close() def test_force_interrupt_leaves_incomplete_marker(): with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({ 1: view_for(1), 2: RuntimeError("simulated interrupt"), }) db, repo, service = make_service(tmp, transport, track_fans=()) try: add_video(repo, 1) add_video(repo, 2) repo.kv_set(service.completed_key(DATE), "1") repo.kv_set("snapshot:last_completed", DATE) try: service.run(date=DATE, force=True) except RuntimeError: pass else: raise AssertionError("应模拟中断") assert repo.kv_get(service.completed_key(DATE)) == "0" assert repo.kv_get("snapshot:last_completed") == "" transport.views[2] = view_for(2) resumed = service.run_if_needed(date=DATE) assert resumed.completed is True assert resumed.success == 1 assert resumed.skipped == 1 finally: db.close() def test_backup_failure_retries_without_api(): import reaction_board.services.snapshot as snapshot_module with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({1: view_for(1)}) db, repo, service = make_service(tmp, transport, track_fans=()) original = snapshot_module.backup_daily def fail_backup(*args, **kwargs): raise OSError("disk full") snapshot_module.backup_daily = fail_backup try: first = service.run(date=DATE) finally: snapshot_module.backup_daily = original assert first.completed is True assert first.backup_error assert repo.kv_get(service.backup_key(DATE)) == "0" view_calls = transport.count("/web-interface/view") retried = service.run_if_needed(date=DATE) assert retried.already_completed is True assert retried.backup_error is None assert Path(retried.backup_path or "").exists() assert repo.kv_get(service.backup_key(DATE)) == "1" assert transport.count("/web-interface/view") == view_calls db.close() def test_run_recent_videos_refreshes_without_completion_markers(): """小时任务只刷新近窗新片，不写完成日 / 备份。""" clock = 1_800_000_000 with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({ 1: view_for(1, view=111), 2: view_for(2, view=222), 3: view_for(3, view=333), 4: view_for(4, view=444), }) db, repo, service = make_service(tmp, transport, track_fans=()) try: # 1/2 在墙钟近 2 天内；3 为旧片；4 发布已久但当天入库，不进小时任务 add_video(repo, 1) add_video(repo, 2) add_video(repo, 3) add_video(repo, 4) repo.db.write(lambda conn: conn.executemany( "UPDATE video SET pubdate = ? WHERE aid = ?", [ (clock - 3600, 1), (clock - 86400, 2), (clock - 10 * 86400, 3), (clock - 10 * 86400, 4), ], )) # 预置同日 ok=1，验证小时任务可覆盖刷新 repo.upsert_snapshot( DATE, 1, metrics={"view": 1, "like": 0, "coin": 0, "favorite": 0, "share": 0, "reply": 0, "danmaku": 0}, ok=True, err=None, captured_at=clock - 100, ) assert repo.kv_get(service.completed_key(DATE)) is None summary = service.run_recent_videos(date=DATE, recent_days=2) assert summary.total == 2 assert summary.success == 2 assert summary.failed == 0 assert set(repo.video_aids_recent_for_snapshot( since_pubdate=clock - 2 * 86400 )) == {1, 2} row = repo.db.read(lambda conn: dict(conn.execute( "SELECT view, ok, captured_at, view_open FROM snapshot " "WHERE stat_date=? AND aid=1", (DATE,), ).fetchone())) assert row["view"] == 111 assert row["ok"] == 1 assert row["captured_at"] == clock assert row["view_open"] == 1 # 完成日 / 备份标记保持原样 assert repo.kv_get(service.completed_key(DATE)) is None assert repo.kv_get("snapshot:last_completed") is None assert repo.kv_get(service.backup_key(DATE)) is None assert transport.count("/web-interface/view") == 2 assert repo.get_snapshot(DATE, 4) is None # 全量任务随后才写完成日；且已 ok=1 的跳过 full = service.run_videos(date=DATE) assert full.skipped >= 2 # 1、2 已被小时任务写过 assert full.success >= 1 # 至少旧片 3、4 assert repo.get_snapshot(DATE, 4) is not None assert repo.kv_get(service.completed_key(DATE)) == "1" assert repo.kv_get("snapshot:last_completed") == DATE finally: db.close() def test_full_snapshot_skips_ok_rows_including_hourly_refresh(): """晚间全量（未 force）：同日 ok=1（含小时任务刚刷）直接跳过。""" clock = 1_800_000_000 with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({1: view_for(1), 2: view_for(2)}) db, repo, service = make_service(tmp, transport, track_fans=()) try: add_video(repo, 1) add_video(repo, 2) repo.upsert_snapshot( DATE, 1, metrics={"view": 9, "like": 0, "coin": 0, "favorite": 0, "share": 0, "reply": 0, "danmaku": 0}, ok=True, err=None, captured_at=clock - 30, # 「1 小时内」 ) result = service.run_videos(date=DATE) assert result.skipped == 1 assert result.success == 1 # aid=1 保持小时任务写入的 view=9，未被全量覆盖 row = repo.db.read(lambda conn: dict(conn.execute( "SELECT view FROM snapshot WHERE stat_date=? AND aid=1", (DATE,), ).fetchone())) assert row["view"] == 9 assert transport.count("/web-interface/view") == 1 finally: db.close() def test_seed_same_day_completed_is_already_completed_not_no_source(): """全站完成后 last_completed==target：应 already_completed，不误判 no_source。""" clock = 1_800_000_000 with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({1: view_for(1)}) db, repo, service = make_service(tmp, transport, track_fans=()) try: add_video(repo, 1) today = "2026-10-31" repo.upsert_snapshot( today, 1, metrics={"view": 10, "like": 1, "coin": 0, "favorite": 0, "share": 0, "reply": 0, "danmaku": 0}, ok=True, err=None, captured_at=clock, ) # 正常收尾态：last_completed 已推进到当日，且 completed 标记在位 repo.kv_set("snapshot:last_completed", today) repo.kv_set(service.completed_key(today), "1") summary = service.run_seed_from_last_completed(date=today) assert summary.reason == "already_completed" assert summary.already_completed is True assert summary.completed is True assert summary.reason != "no_source" assert transport.count("/web-interface/view") == 0 finally: db.close() def test_seed_from_last_completed_does_not_mark_completed(): clock = 1_800_000_000 with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({1: view_for(1)}) db, repo, service = make_service(tmp, transport, track_fans=(37754047,)) try: add_video(repo, 1) yday = "2026-10-30" today = "2026-10-31" repo.upsert_snapshot( yday, 1, metrics={"view": 10, "like": 1, "coin": 0, "favorite": 0, "share": 0, "reply": 0, "danmaku": 0}, ok=True, err=None, captured_at=clock - 86400, ) repo.upsert_up_stat( yday, 37754047, follower=100, following=1, ok=True, captured_at=clock - 86400, ) repo.kv_set("snapshot:last_completed", yday) repo.kv_set(service.completed_key(yday), "1") summary = service.run_seed_from_last_completed(date=today) assert summary.success == 1 assert summary.fans_ok == 1 assert repo.kv_get(service.seeded_key(today)) == "1" assert repo.kv_get(service.fans_seeded_key(today)) == "1" assert repo.kv_get(service.completed_key(today)) is None assert repo.kv_get("snapshot:last_completed") == yday row = repo.get_snapshot(today, 1) assert row is not None assert int(row["view"]) == 10 assert int(row["view_open"]) == 10 assert row["src"] == "seed" fan = repo.get_up_stat(today, 37754047) assert fan is not None and int(fan["follower"]) == 100 # 幂等：再 seed 不重复 again = service.run_seed_from_last_completed(date=today) assert again.already_completed is True assert transport.count("/web-interface/view") == 0 finally: db.close() def test_seeded_day_skips_without_force_and_seals_with_force(): clock = 1_800_000_000 with tempfile.TemporaryDirectory() as tmp: fan_payload = dict(RELATION_PAYLOAD) fan_payload["data"] = dict(RELATION_PAYLOAD["data"]) fan_payload["data"]["follower"] = 999 transport = SnapshotTransport({1: view_for(1, view=555)}, fan=fan_payload) db, repo, service = make_service(tmp, transport, track_fans=(37754047,)) try: add_video(repo, 1) yday = "2026-10-30" today = DATE repo.upsert_snapshot( yday, 1, metrics={"view": 10, "like": 1, "coin": 0, "favorite": 0, "share": 0, "reply": 0, "danmaku": 0}, ok=True, err=None, captured_at=clock - 86400, ) repo.upsert_up_stat( yday, 37754047, follower=100, following=1, ok=True, captured_at=clock - 86400, ) repo.kv_set("snapshot:last_completed", yday) service.run_seed_from_last_completed(date=today) pending = service.run_if_needed(date=today) assert pending.completed is False assert pending.already_completed is False assert repo.kv_get(service.seeded_key(today)) == "1" assert transport.count("/web-interface/view") == 0 sealed = service.run_if_needed(date=today, force=True) assert sealed.completed is True assert sealed.success >= 1 assert repo.kv_get(service.completed_key(today)) == "1" assert repo.kv_get("snapshot:last_completed") == today assert repo.kv_get(service.seeded_key(today)) is None row = repo.get_snapshot(today, 1) assert int(row["view"]) == 555 # 被真实 view 覆盖 assert row["src"] == "view" fans = service.run_fans_if_needed(date=today, force=True) assert fans.fans_ok >= 1 assert repo.kv_get(service.fans_seeded_key(today)) is None assert int(repo.get_up_stat(today, 37754047)["follower"]) == 999 finally: db.close() def main(): test_snapshot_success_resume_and_force() test_resume_skips_ok_and_retries_failed() test_unavailable_marks_removed() test_other_error_writes_failed_placeholder() test_fan_failure_placeholder() test_auth_invalid_is_warned_but_snapshot_continues() test_no_cookie_skips_nav_request() test_force_failure_clears_last_completed() test_new_video_after_completion_backfills_same_day() test_fan_resume_preserves_successful_value() test_force_interrupt_leaves_incomplete_marker() test_backup_failure_retries_without_api() test_run_recent_videos_refreshes_without_completion_markers() test_full_snapshot_skips_ok_rows_including_hourly_refresh() test_seed_same_day_completed_is_already_completed_not_no_source() test_seed_from_last_completed_does_not_mark_completed() test_seeded_day_skips_without_force_and_seals_with_force() print("OK: 每日快照、断点续跑、失败占位与粉丝采集测试全部通过") if __name__ == "__main__": main()