feat(snapshot): refresh API video metadata; keep removed visible

入库 / 小时近新片 / 全量快照成功时以 view_detail 覆盖 title、duration、
pages、cover、bvid、up_*、pubdate 等接口字段,并照旧写七项快照。
原站失效仅标 removed,不删行、不空写元数据;列表与详情继续展示,
快照队列仍跳过 removed。
This commit is contained in:
bot-ci committed 2026-11-03 23:01:15 +08:00
1 parent 99b71c037c
commit 6c73904b25
8 files changed
+329 -34

No files matched your search

+31 -16
View File
@@ -3,12 +3,13 @@
规则(DEVELOPMENT.md §7.4):
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``;
* 成功写入七项指标;``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view_detail``(完整元数据 + 七项);
* 成功时以接口字段覆盖更新 ``video`` 元数据(title/duration/pages/cover/UP 等),并写入七项快照;
* ``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
* 午夜 ``seed_at`` 用上一完成日 ok=1 行克隆出当日壳(``snapshot:seeded`` / ``fans:seeded``),不推进完成日;
* 晚间 ``snapshot_at`` 全量必须 force 打穿 seed,成功后才写完成日并清除 seeded;
* ``run_recent_videos`` 只刷新近 N 天新片的当日行,不改完成日 / 备份;
* 视频不可访问时标记 ``removed``;
* 视频不可访问时标记 ``removed``(保留元数据与历史快照,站内继续展示,后续采集跳过);
* 其他错误最多重试 3 次,仍失败写 ``ok=0`` 占位,不写假零值;
* 对 ``track_fans`` 逐个请求 ``relation/stat``(独立任务,不影响视频完成日);
* 视频采集全部成功后写入当日完成标记并生成日备份。
@@ -224,7 +225,7 @@ class SnapshotService:
force: bool = False,
keep_scraped: bool = False,
) -> SnapshotSummary:
"""采集视频七项指标并在全部成功后标记完成日 / 备份。
"""采集视频元数据 + 七项指标,并在全部成功后标记完成日 / 备份。
``force``:忽略完成标记,重跑。
``keep_scraped``:force 时仍跳过非 seed 的 ok=1(seal 断点续跑);
@@ -264,7 +265,7 @@ class SnapshotService:
continue
# force 且非 keep_scraped:CLI 全量重采,不 skip
stat, error, unavailable = self._fetch_view(client, aid)
detail, error, unavailable = self._fetch_view_detail(client, aid)
if unavailable:
summary.removed += 1
self.repo.set_video_status(aid, "removed", now=now)
@@ -272,17 +273,16 @@ class SnapshotService:
"warning", "snapshot.removed",
stat_date=stat_date, aid=aid, error=error,
)
elif stat is not None:
elif detail is not None:
self._apply_detail(detail, now=self._now())
self.repo.upsert_snapshot(
stat_date,
aid,
metrics=stat.metrics(),
metrics=detail.metrics(),
ok=True,
err=None,
captured_at=self._now(),
)
if getattr(stat, "up_face", None):
self.repo.apply_video_up_face(aid, stat.up_face)
summary.success += 1
else:
summary.failed += 1
@@ -369,7 +369,7 @@ class SnapshotService:
videos = self.repo.video_aids_recent_for_snapshot(since_pubdate=since_pubdate)
summary.total = len(videos)
for index, aid in enumerate(videos, 1):
stat, error, unavailable = self._fetch_view(client, aid)
detail, error, unavailable = self._fetch_view_detail(client, aid)
if unavailable:
summary.removed += 1
self.repo.set_video_status(aid, "removed", now=now)
@@ -377,17 +377,16 @@ class SnapshotService:
"warning", "snapshot.recent_removed",
stat_date=stat_date, aid=aid, error=error,
)
elif stat is not None:
elif detail is not None:
self._apply_detail(detail, now=self._now())
self.repo.upsert_snapshot(
stat_date,
aid,
metrics=stat.metrics(),
metrics=detail.metrics(),
ok=True,
err=None,
captured_at=self._now(),
)
if getattr(stat, "up_face", None):
self.repo.apply_video_up_face(aid, stat.up_face)
summary.success += 1
else:
summary.failed += 1
@@ -693,11 +692,27 @@ class SnapshotService:
)
return state.is_login, state.message
def _fetch_view(self, client, aid: int):
def _apply_detail(self, detail, *, now: int) -> None:
"""快照成功:以 view_detail 字段覆盖 video 元数据(不改 status)。"""
self.repo.refresh_video_metadata(
aid=detail.aid,
bvid=detail.bvid,
title=detail.title,
up_mid=detail.up_mid,
up_name=detail.up_name,
pubdate=detail.pubdate,
duration=detail.duration,
pages=detail.pages,
cover=detail.cover,
up_face=detail.up_face,
now=now,
)
def _fetch_view_detail(self, client, aid: int):
last_error = ""
for attempt in range(1, self.retries + 1):
try:
return client.view(aid), None, False
return client.view_detail(aid=aid), None, False
except BilibiliAPIError as exc:
last_error = str(exc)
if exc.code in VIDEO_UNAVAILABLE_CODES: