feat(snapshot): refresh API video metadata; keep removed visible

入库 / 小时近新片 / 全量快照成功时以 view_detail 覆盖 title、duration、
pages、cover、bvid、up_*、pubdate 等接口字段,并照旧写七项快照。
原站失效仅标 removed,不删行、不空写元数据;列表与详情继续展示,
快照队列仍跳过 removed。
This commit is contained in:
bot-ci committed 2026-11-03 23:01:15 +08:00
1 parent 99b71c037c
commit 6c73904b25
8 files changed
+329 -34

No files matched your search

+70
View File
@@ -410,6 +410,75 @@ class Repository:
return [dict(row) for row in rows]
return self.db.read(run)
def refresh_video_metadata(
self,
*,
aid: int,
bvid: str | None,
title: str,
up_mid: int | None,
up_name: str | None,
pubdate: int,
duration: int | None = None,
pages: int | None = None,
cover: str | None = None,
up_face: str | None = None,
now: int | None = None,
) -> bool:
"""用接口返回字段覆盖更新已有视频元数据;不改 status,空值不冲掉旧值。
入库与快照成功路径共用。视频不存在时返回 False。
"""
now = self._now() if now is None else int(now)
aid = int(aid)
def run(conn: sqlite3.Connection) -> bool:
existing = conn.execute(
"SELECT aid FROM video WHERE aid = ?", (aid,)
).fetchone()
if existing is None:
return False
conn.execute(
"""
UPDATE video SET
bvid = COALESCE(?, bvid),
title = CASE
WHEN ? != '' THEN ?
ELSE title
END,
up_mid = COALESCE(?, up_mid),
up_name = COALESCE(?, up_name),
pubdate = CASE
WHEN ? > 0 THEN ?
ELSE pubdate
END,
duration = COALESCE(?, duration),
pages = COALESCE(?, pages),
cover = COALESCE(?, cover),
up_face = COALESCE(?, up_face),
last_sync = ?
WHERE aid = ?
""",
(
bvid,
title or "",
title or "(无标题)",
up_mid,
up_name,
int(pubdate or 0),
int(pubdate or 0),
None if duration is None else int(duration),
None if pages is None else int(pages),
cover,
up_face,
now,
aid,
),
)
return True
return self.db.write(run)
def ingest_video(
self,
*,
@@ -430,6 +499,7 @@ class Repository:
不写 B 站收藏夹、不设置 ``folder_id``;同时删除同 aid 的候选行。
既有的 ``removed`` / ``dropped`` / ``unfaved`` 状态都会被重新激活。
已存在时仍以本次接口字段为准刷新元数据。
"""
now = self._now() if now is None else int(now)
aid = int(aid)
+29 -3
View File
@@ -116,11 +116,37 @@ class LibraryService:
if candidate is None:
video = self.repo.get_video(aid)
if video is not None:
# 已在库:仍以接口为准刷新元数据(不重复建档)
try:
detail = self.read_provider.get().view_detail(aid=aid)
except BilibiliError as exc:
self._log("warning", "ingest.detail_failed", aid=aid, error=str(exc))
return IngestResult(
ok=True, aid=aid, state="exists",
title=str(video.get("title") or ""),
up_name=str(video.get("up_name") or ""),
message="已在库,无需重复入库",
)
self.repo.refresh_video_metadata(
aid=detail.aid,
bvid=detail.bvid,
title=detail.title,
up_mid=detail.up_mid,
up_name=detail.up_name,
pubdate=detail.pubdate,
duration=detail.duration,
pages=detail.pages,
cover=detail.cover,
up_face=detail.up_face,
now=self._now(),
)
self._write_snapshot(detail)
self.repo.bump_cache_version("videos")
return IngestResult(
ok=True, aid=aid, state="exists",
title=str(video.get("title") or ""),
up_name=str(video.get("up_name") or ""),
message="已在库,无需重复入库",
title=detail.title,
up_name=detail.up_name or "",
message="已在库,已刷新元数据",
)
return IngestResult(
ok=False, aid=aid, state="missing", message="候选不存在或已处理"
+31 -16
View File
@@ -3,12 +3,13 @@
规则(DEVELOPMENT.md §7.4):
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``;
* 成功写入七项指标;``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view_detail``(完整元数据 + 七项);
* 成功时以接口字段覆盖更新 ``video`` 元数据(title/duration/pages/cover/UP 等),并写入七项快照;
* ``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
* 午夜 ``seed_at`` 用上一完成日 ok=1 行克隆出当日壳(``snapshot:seeded`` / ``fans:seeded``),不推进完成日;
* 晚间 ``snapshot_at`` 全量必须 force 打穿 seed,成功后才写完成日并清除 seeded;
* ``run_recent_videos`` 只刷新近 N 天新片的当日行,不改完成日 / 备份;
* 视频不可访问时标记 ``removed``;
* 视频不可访问时标记 ``removed``(保留元数据与历史快照,站内继续展示,后续采集跳过);
* 其他错误最多重试 3 次,仍失败写 ``ok=0`` 占位,不写假零值;
* 对 ``track_fans`` 逐个请求 ``relation/stat``(独立任务,不影响视频完成日);
* 视频采集全部成功后写入当日完成标记并生成日备份。
@@ -224,7 +225,7 @@ class SnapshotService:
force: bool = False,
keep_scraped: bool = False,
) -> SnapshotSummary:
"""采集视频七项指标并在全部成功后标记完成日 / 备份。
"""采集视频元数据 + 七项指标,并在全部成功后标记完成日 / 备份。
``force``:忽略完成标记,重跑。
``keep_scraped``:force 时仍跳过非 seed 的 ok=1(seal 断点续跑);
@@ -264,7 +265,7 @@ class SnapshotService:
continue
# force 且非 keep_scraped:CLI 全量重采,不 skip
stat, error, unavailable = self._fetch_view(client, aid)
detail, error, unavailable = self._fetch_view_detail(client, aid)
if unavailable:
summary.removed += 1
self.repo.set_video_status(aid, "removed", now=now)
@@ -272,17 +273,16 @@ class SnapshotService:
"warning", "snapshot.removed",
stat_date=stat_date, aid=aid, error=error,
)
elif stat is not None:
elif detail is not None:
self._apply_detail(detail, now=self._now())
self.repo.upsert_snapshot(
stat_date,
aid,
metrics=stat.metrics(),
metrics=detail.metrics(),
ok=True,
err=None,
captured_at=self._now(),
)
if getattr(stat, "up_face", None):
self.repo.apply_video_up_face(aid, stat.up_face)
summary.success += 1
else:
summary.failed += 1
@@ -369,7 +369,7 @@ class SnapshotService:
videos = self.repo.video_aids_recent_for_snapshot(since_pubdate=since_pubdate)
summary.total = len(videos)
for index, aid in enumerate(videos, 1):
stat, error, unavailable = self._fetch_view(client, aid)
detail, error, unavailable = self._fetch_view_detail(client, aid)
if unavailable:
summary.removed += 1
self.repo.set_video_status(aid, "removed", now=now)
@@ -377,17 +377,16 @@ class SnapshotService:
"warning", "snapshot.recent_removed",
stat_date=stat_date, aid=aid, error=error,
)
elif stat is not None:
elif detail is not None:
self._apply_detail(detail, now=self._now())
self.repo.upsert_snapshot(
stat_date,
aid,
metrics=stat.metrics(),
metrics=detail.metrics(),
ok=True,
err=None,
captured_at=self._now(),
)
if getattr(stat, "up_face", None):
self.repo.apply_video_up_face(aid, stat.up_face)
summary.success += 1
else:
summary.failed += 1
@@ -693,11 +692,27 @@ class SnapshotService:
)
return state.is_login, state.message
def _fetch_view(self, client, aid: int):
def _apply_detail(self, detail, *, now: int) -> None:
"""快照成功:以 view_detail 字段覆盖 video 元数据(不改 status)。"""
self.repo.refresh_video_metadata(
aid=detail.aid,
bvid=detail.bvid,
title=detail.title,
up_mid=detail.up_mid,
up_name=detail.up_name,
pubdate=detail.pubdate,
duration=detail.duration,
pages=detail.pages,
cover=detail.cover,
up_face=detail.up_face,
now=now,
)
def _fetch_view_detail(self, client, aid: int):
last_error = ""
for attempt in range(1, self.retries + 1):
try:
return client.view(aid), None, False
return client.view_detail(aid=aid), None, False
except BilibiliAPIError as exc:
last_error = str(exc)
if exc.code in VIDEO_UNAVAILABLE_CODES:
+4 -1
View File
@@ -45,6 +45,8 @@ from ..timeutil import (
from .cache import DASHBOARD_CACHE, TTL_FANS, TTL_VIDEOS, build_cache_key
TRACKED_STATUSES = ("active", "unfaved")
# 站内列表可见:含原站失效 removed(继续展示、badge 可标 status-removed);不含人工 dropped
VISIBLE_STATUSES = ("active", "unfaved", "removed")
VIDEO_SORTS = {
"heat", "view", "like", "coin", "favorite", "share", "reply", "danmaku",
"interaction", "growth", "growth_rate", "added", "published",
@@ -955,6 +957,7 @@ class DashboardQueries:
within_start = _date_minus(latest, int(within_days) - 1)
videos = self._videos(
conn,
statuses=VISIBLE_STATUSES,
folder_id=folder_id,
q=q,
within_field=within_field,
@@ -1157,7 +1160,7 @@ class DashboardQueries:
aid = int(aid)
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
video_rows = self._videos(conn)
video_rows = self._videos(conn, statuses=VISIBLE_STATUSES)
video = next((item for item in video_rows if int(item["aid"]) == aid), None)
if video is None:
row = conn.execute(