feat(snapshot): refresh API video metadata; keep removed visible
入库 / 小时近新片 / 全量快照成功时以 view_detail 覆盖 title、duration、 pages、cover、bvid、up_*、pubdate 等接口字段,并照旧写七项快照。 原站失效仅标 removed,不删行、不空写元数据;列表与详情继续展示, 快照队列仍跳过 removed。
This commit is contained in:
1 parent
99b71c037c
commit
6c73904b25
8 files changed
+329
-34
No files matched your search
@@ -410,6 +410,75 @@ class Repository:
|
||||
return [dict(row) for row in rows]
|
||||
return self.db.read(run)
|
||||
|
||||
def refresh_video_metadata(
|
||||
self,
|
||||
*,
|
||||
aid: int,
|
||||
bvid: str | None,
|
||||
title: str,
|
||||
up_mid: int | None,
|
||||
up_name: str | None,
|
||||
pubdate: int,
|
||||
duration: int | None = None,
|
||||
pages: int | None = None,
|
||||
cover: str | None = None,
|
||||
up_face: str | None = None,
|
||||
now: int | None = None,
|
||||
) -> bool:
|
||||
"""用接口返回字段覆盖更新已有视频元数据;不改 status,空值不冲掉旧值。
|
||||
|
||||
入库与快照成功路径共用。视频不存在时返回 False。
|
||||
"""
|
||||
now = self._now() if now is None else int(now)
|
||||
aid = int(aid)
|
||||
|
||||
def run(conn: sqlite3.Connection) -> bool:
|
||||
existing = conn.execute(
|
||||
"SELECT aid FROM video WHERE aid = ?", (aid,)
|
||||
).fetchone()
|
||||
if existing is None:
|
||||
return False
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE video SET
|
||||
bvid = COALESCE(?, bvid),
|
||||
title = CASE
|
||||
WHEN ? != '' THEN ?
|
||||
ELSE title
|
||||
END,
|
||||
up_mid = COALESCE(?, up_mid),
|
||||
up_name = COALESCE(?, up_name),
|
||||
pubdate = CASE
|
||||
WHEN ? > 0 THEN ?
|
||||
ELSE pubdate
|
||||
END,
|
||||
duration = COALESCE(?, duration),
|
||||
pages = COALESCE(?, pages),
|
||||
cover = COALESCE(?, cover),
|
||||
up_face = COALESCE(?, up_face),
|
||||
last_sync = ?
|
||||
WHERE aid = ?
|
||||
""",
|
||||
(
|
||||
bvid,
|
||||
title or "",
|
||||
title or "(无标题)",
|
||||
up_mid,
|
||||
up_name,
|
||||
int(pubdate or 0),
|
||||
int(pubdate or 0),
|
||||
None if duration is None else int(duration),
|
||||
None if pages is None else int(pages),
|
||||
cover,
|
||||
up_face,
|
||||
now,
|
||||
aid,
|
||||
),
|
||||
)
|
||||
return True
|
||||
|
||||
return self.db.write(run)
|
||||
|
||||
def ingest_video(
|
||||
self,
|
||||
*,
|
||||
@@ -430,6 +499,7 @@ class Repository:
|
||||
|
||||
不写 B 站收藏夹、不设置 ``folder_id``;同时删除同 aid 的候选行。
|
||||
既有的 ``removed`` / ``dropped`` / ``unfaved`` 状态都会被重新激活。
|
||||
已存在时仍以本次接口字段为准刷新元数据。
|
||||
"""
|
||||
now = self._now() if now is None else int(now)
|
||||
aid = int(aid)
|
||||
|
||||
@@ -116,11 +116,37 @@ class LibraryService:
|
||||
if candidate is None:
|
||||
video = self.repo.get_video(aid)
|
||||
if video is not None:
|
||||
# 已在库:仍以接口为准刷新元数据(不重复建档)
|
||||
try:
|
||||
detail = self.read_provider.get().view_detail(aid=aid)
|
||||
except BilibiliError as exc:
|
||||
self._log("warning", "ingest.detail_failed", aid=aid, error=str(exc))
|
||||
return IngestResult(
|
||||
ok=True, aid=aid, state="exists",
|
||||
title=str(video.get("title") or ""),
|
||||
up_name=str(video.get("up_name") or ""),
|
||||
message="已在库,无需重复入库",
|
||||
)
|
||||
self.repo.refresh_video_metadata(
|
||||
aid=detail.aid,
|
||||
bvid=detail.bvid,
|
||||
title=detail.title,
|
||||
up_mid=detail.up_mid,
|
||||
up_name=detail.up_name,
|
||||
pubdate=detail.pubdate,
|
||||
duration=detail.duration,
|
||||
pages=detail.pages,
|
||||
cover=detail.cover,
|
||||
up_face=detail.up_face,
|
||||
now=self._now(),
|
||||
)
|
||||
self._write_snapshot(detail)
|
||||
self.repo.bump_cache_version("videos")
|
||||
return IngestResult(
|
||||
ok=True, aid=aid, state="exists",
|
||||
title=str(video.get("title") or ""),
|
||||
up_name=str(video.get("up_name") or ""),
|
||||
message="已在库,无需重复入库",
|
||||
title=detail.title,
|
||||
up_name=detail.up_name or "",
|
||||
message="已在库,已刷新元数据",
|
||||
)
|
||||
return IngestResult(
|
||||
ok=False, aid=aid, state="missing", message="候选不存在或已处理"
|
||||
|
||||
@@ -3,12 +3,13 @@
|
||||
|
||||
规则(DEVELOPMENT.md §7.4):
|
||||
|
||||
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``;
|
||||
* 成功写入七项指标;``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
|
||||
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view_detail``(完整元数据 + 七项);
|
||||
* 成功时以接口字段覆盖更新 ``video`` 元数据(title/duration/pages/cover/UP 等),并写入七项快照;
|
||||
* ``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
|
||||
* 午夜 ``seed_at`` 用上一完成日 ok=1 行克隆出当日壳(``snapshot:seeded`` / ``fans:seeded``),不推进完成日;
|
||||
* 晚间 ``snapshot_at`` 全量必须 force 打穿 seed,成功后才写完成日并清除 seeded;
|
||||
* ``run_recent_videos`` 只刷新近 N 天新片的当日行,不改完成日 / 备份;
|
||||
* 视频不可访问时标记 ``removed``;
|
||||
* 视频不可访问时标记 ``removed``(保留元数据与历史快照,站内继续展示,后续采集跳过);
|
||||
* 其他错误最多重试 3 次,仍失败写 ``ok=0`` 占位,不写假零值;
|
||||
* 对 ``track_fans`` 逐个请求 ``relation/stat``(独立任务,不影响视频完成日);
|
||||
* 视频采集全部成功后写入当日完成标记并生成日备份。
|
||||
@@ -224,7 +225,7 @@ class SnapshotService:
|
||||
force: bool = False,
|
||||
keep_scraped: bool = False,
|
||||
) -> SnapshotSummary:
|
||||
"""采集视频七项指标并在全部成功后标记完成日 / 备份。
|
||||
"""采集视频元数据 + 七项指标,并在全部成功后标记完成日 / 备份。
|
||||
|
||||
``force``:忽略完成标记,重跑。
|
||||
``keep_scraped``:force 时仍跳过非 seed 的 ok=1(seal 断点续跑);
|
||||
@@ -264,7 +265,7 @@ class SnapshotService:
|
||||
continue
|
||||
# force 且非 keep_scraped:CLI 全量重采,不 skip
|
||||
|
||||
stat, error, unavailable = self._fetch_view(client, aid)
|
||||
detail, error, unavailable = self._fetch_view_detail(client, aid)
|
||||
if unavailable:
|
||||
summary.removed += 1
|
||||
self.repo.set_video_status(aid, "removed", now=now)
|
||||
@@ -272,17 +273,16 @@ class SnapshotService:
|
||||
"warning", "snapshot.removed",
|
||||
stat_date=stat_date, aid=aid, error=error,
|
||||
)
|
||||
elif stat is not None:
|
||||
elif detail is not None:
|
||||
self._apply_detail(detail, now=self._now())
|
||||
self.repo.upsert_snapshot(
|
||||
stat_date,
|
||||
aid,
|
||||
metrics=stat.metrics(),
|
||||
metrics=detail.metrics(),
|
||||
ok=True,
|
||||
err=None,
|
||||
captured_at=self._now(),
|
||||
)
|
||||
if getattr(stat, "up_face", None):
|
||||
self.repo.apply_video_up_face(aid, stat.up_face)
|
||||
summary.success += 1
|
||||
else:
|
||||
summary.failed += 1
|
||||
@@ -369,7 +369,7 @@ class SnapshotService:
|
||||
videos = self.repo.video_aids_recent_for_snapshot(since_pubdate=since_pubdate)
|
||||
summary.total = len(videos)
|
||||
for index, aid in enumerate(videos, 1):
|
||||
stat, error, unavailable = self._fetch_view(client, aid)
|
||||
detail, error, unavailable = self._fetch_view_detail(client, aid)
|
||||
if unavailable:
|
||||
summary.removed += 1
|
||||
self.repo.set_video_status(aid, "removed", now=now)
|
||||
@@ -377,17 +377,16 @@ class SnapshotService:
|
||||
"warning", "snapshot.recent_removed",
|
||||
stat_date=stat_date, aid=aid, error=error,
|
||||
)
|
||||
elif stat is not None:
|
||||
elif detail is not None:
|
||||
self._apply_detail(detail, now=self._now())
|
||||
self.repo.upsert_snapshot(
|
||||
stat_date,
|
||||
aid,
|
||||
metrics=stat.metrics(),
|
||||
metrics=detail.metrics(),
|
||||
ok=True,
|
||||
err=None,
|
||||
captured_at=self._now(),
|
||||
)
|
||||
if getattr(stat, "up_face", None):
|
||||
self.repo.apply_video_up_face(aid, stat.up_face)
|
||||
summary.success += 1
|
||||
else:
|
||||
summary.failed += 1
|
||||
@@ -693,11 +692,27 @@ class SnapshotService:
|
||||
)
|
||||
return state.is_login, state.message
|
||||
|
||||
def _fetch_view(self, client, aid: int):
|
||||
def _apply_detail(self, detail, *, now: int) -> None:
|
||||
"""快照成功:以 view_detail 字段覆盖 video 元数据(不改 status)。"""
|
||||
self.repo.refresh_video_metadata(
|
||||
aid=detail.aid,
|
||||
bvid=detail.bvid,
|
||||
title=detail.title,
|
||||
up_mid=detail.up_mid,
|
||||
up_name=detail.up_name,
|
||||
pubdate=detail.pubdate,
|
||||
duration=detail.duration,
|
||||
pages=detail.pages,
|
||||
cover=detail.cover,
|
||||
up_face=detail.up_face,
|
||||
now=now,
|
||||
)
|
||||
|
||||
def _fetch_view_detail(self, client, aid: int):
|
||||
last_error = ""
|
||||
for attempt in range(1, self.retries + 1):
|
||||
try:
|
||||
return client.view(aid), None, False
|
||||
return client.view_detail(aid=aid), None, False
|
||||
except BilibiliAPIError as exc:
|
||||
last_error = str(exc)
|
||||
if exc.code in VIDEO_UNAVAILABLE_CODES:
|
||||
|
||||
@@ -45,6 +45,8 @@ from ..timeutil import (
|
||||
from .cache import DASHBOARD_CACHE, TTL_FANS, TTL_VIDEOS, build_cache_key
|
||||
|
||||
TRACKED_STATUSES = ("active", "unfaved")
|
||||
# 站内列表可见:含原站失效 removed(继续展示、badge 可标 status-removed);不含人工 dropped
|
||||
VISIBLE_STATUSES = ("active", "unfaved", "removed")
|
||||
VIDEO_SORTS = {
|
||||
"heat", "view", "like", "coin", "favorite", "share", "reply", "danmaku",
|
||||
"interaction", "growth", "growth_rate", "added", "published",
|
||||
@@ -955,6 +957,7 @@ class DashboardQueries:
|
||||
within_start = _date_minus(latest, int(within_days) - 1)
|
||||
videos = self._videos(
|
||||
conn,
|
||||
statuses=VISIBLE_STATUSES,
|
||||
folder_id=folder_id,
|
||||
q=q,
|
||||
within_field=within_field,
|
||||
@@ -1157,7 +1160,7 @@ class DashboardQueries:
|
||||
aid = int(aid)
|
||||
|
||||
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
|
||||
video_rows = self._videos(conn)
|
||||
video_rows = self._videos(conn, statuses=VISIBLE_STATUSES)
|
||||
video = next((item for item in video_rows if int(item["aid"]) == aid), None)
|
||||
if video is None:
|
||||
row = conn.execute(
|
||||
|
||||
Reference in new issue
Block a user