feat(snapshot): refresh API video metadata; keep removed visible
入库 / 小时近新片 / 全量快照成功时以 view_detail 覆盖 title、duration、 pages、cover、bvid、up_*、pubdate 等接口字段,并照旧写七项快照。 原站失效仅标 removed,不删行、不空写元数据;列表与详情继续展示, 快照队列仍跳过 removed。
This commit is contained in:
1 parent
99b71c037c
commit
6c73904b25
8 files changed
+329
-34
No files matched your search
@@ -3,12 +3,13 @@
|
||||
|
||||
规则(DEVELOPMENT.md §7.4):
|
||||
|
||||
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``;
|
||||
* 成功写入七项指标;``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
|
||||
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view_detail``(完整元数据 + 七项);
|
||||
* 成功时以接口字段覆盖更新 ``video`` 元数据(title/duration/pages/cover/UP 等),并写入七项快照;
|
||||
* ``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
|
||||
* 午夜 ``seed_at`` 用上一完成日 ok=1 行克隆出当日壳(``snapshot:seeded`` / ``fans:seeded``),不推进完成日;
|
||||
* 晚间 ``snapshot_at`` 全量必须 force 打穿 seed,成功后才写完成日并清除 seeded;
|
||||
* ``run_recent_videos`` 只刷新近 N 天新片的当日行,不改完成日 / 备份;
|
||||
* 视频不可访问时标记 ``removed``;
|
||||
* 视频不可访问时标记 ``removed``(保留元数据与历史快照,站内继续展示,后续采集跳过);
|
||||
* 其他错误最多重试 3 次,仍失败写 ``ok=0`` 占位,不写假零值;
|
||||
* 对 ``track_fans`` 逐个请求 ``relation/stat``(独立任务,不影响视频完成日);
|
||||
* 视频采集全部成功后写入当日完成标记并生成日备份。
|
||||
@@ -224,7 +225,7 @@ class SnapshotService:
|
||||
force: bool = False,
|
||||
keep_scraped: bool = False,
|
||||
) -> SnapshotSummary:
|
||||
"""采集视频七项指标并在全部成功后标记完成日 / 备份。
|
||||
"""采集视频元数据 + 七项指标,并在全部成功后标记完成日 / 备份。
|
||||
|
||||
``force``:忽略完成标记,重跑。
|
||||
``keep_scraped``:force 时仍跳过非 seed 的 ok=1(seal 断点续跑);
|
||||
@@ -264,7 +265,7 @@ class SnapshotService:
|
||||
continue
|
||||
# force 且非 keep_scraped:CLI 全量重采,不 skip
|
||||
|
||||
stat, error, unavailable = self._fetch_view(client, aid)
|
||||
detail, error, unavailable = self._fetch_view_detail(client, aid)
|
||||
if unavailable:
|
||||
summary.removed += 1
|
||||
self.repo.set_video_status(aid, "removed", now=now)
|
||||
@@ -272,17 +273,16 @@ class SnapshotService:
|
||||
"warning", "snapshot.removed",
|
||||
stat_date=stat_date, aid=aid, error=error,
|
||||
)
|
||||
elif stat is not None:
|
||||
elif detail is not None:
|
||||
self._apply_detail(detail, now=self._now())
|
||||
self.repo.upsert_snapshot(
|
||||
stat_date,
|
||||
aid,
|
||||
metrics=stat.metrics(),
|
||||
metrics=detail.metrics(),
|
||||
ok=True,
|
||||
err=None,
|
||||
captured_at=self._now(),
|
||||
)
|
||||
if getattr(stat, "up_face", None):
|
||||
self.repo.apply_video_up_face(aid, stat.up_face)
|
||||
summary.success += 1
|
||||
else:
|
||||
summary.failed += 1
|
||||
@@ -369,7 +369,7 @@ class SnapshotService:
|
||||
videos = self.repo.video_aids_recent_for_snapshot(since_pubdate=since_pubdate)
|
||||
summary.total = len(videos)
|
||||
for index, aid in enumerate(videos, 1):
|
||||
stat, error, unavailable = self._fetch_view(client, aid)
|
||||
detail, error, unavailable = self._fetch_view_detail(client, aid)
|
||||
if unavailable:
|
||||
summary.removed += 1
|
||||
self.repo.set_video_status(aid, "removed", now=now)
|
||||
@@ -377,17 +377,16 @@ class SnapshotService:
|
||||
"warning", "snapshot.recent_removed",
|
||||
stat_date=stat_date, aid=aid, error=error,
|
||||
)
|
||||
elif stat is not None:
|
||||
elif detail is not None:
|
||||
self._apply_detail(detail, now=self._now())
|
||||
self.repo.upsert_snapshot(
|
||||
stat_date,
|
||||
aid,
|
||||
metrics=stat.metrics(),
|
||||
metrics=detail.metrics(),
|
||||
ok=True,
|
||||
err=None,
|
||||
captured_at=self._now(),
|
||||
)
|
||||
if getattr(stat, "up_face", None):
|
||||
self.repo.apply_video_up_face(aid, stat.up_face)
|
||||
summary.success += 1
|
||||
else:
|
||||
summary.failed += 1
|
||||
@@ -693,11 +692,27 @@ class SnapshotService:
|
||||
)
|
||||
return state.is_login, state.message
|
||||
|
||||
def _fetch_view(self, client, aid: int):
|
||||
def _apply_detail(self, detail, *, now: int) -> None:
|
||||
"""快照成功:以 view_detail 字段覆盖 video 元数据(不改 status)。"""
|
||||
self.repo.refresh_video_metadata(
|
||||
aid=detail.aid,
|
||||
bvid=detail.bvid,
|
||||
title=detail.title,
|
||||
up_mid=detail.up_mid,
|
||||
up_name=detail.up_name,
|
||||
pubdate=detail.pubdate,
|
||||
duration=detail.duration,
|
||||
pages=detail.pages,
|
||||
cover=detail.cover,
|
||||
up_face=detail.up_face,
|
||||
now=now,
|
||||
)
|
||||
|
||||
def _fetch_view_detail(self, client, aid: int):
|
||||
last_error = ""
|
||||
for attempt in range(1, self.retries + 1):
|
||||
try:
|
||||
return client.view(aid), None, False
|
||||
return client.view_detail(aid=aid), None, False
|
||||
except BilibiliAPIError as exc:
|
||||
last_error = str(exc)
|
||||
if exc.code in VIDEO_UNAVAILABLE_CODES:
|
||||
|
||||
Reference in new issue
Block a user