fix: close data integrity and hot-reload review findings

Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.
This commit is contained in:
kairee committed 2026-10-31 20:58:36 +08:00
1 parent 178ab64b18
commit 6a21b5d905
21 files changed
+895 -82

No files matched your search

+16 -10
View File
@@ -262,6 +262,20 @@ class BilibiliClient:
raise self._api_error(payload)
return payload
@staticmethod
def _data_list(payload: Mapping[str, Any], name: str) -> list[Any]:
"""区分“真实空数组”与“协议缺字段”。
实测空收藏夹返回 ``{"code":0,"data":[]}``;缺少 ``data`` 或为 ``null``
都不是已知契约,必须中止,避免被误判成空收藏夹后批量改状态。
"""
if "data" not in payload:
raise BilibiliProtocolError(f"{name} 响应缺少 data 字段")
data = payload["data"]
if not isinstance(data, list):
raise BilibiliProtocolError(f"{name} data 不是数组")
return data
# ------------------------------------------------------------ fav
def fav_ids(self, media_id: int) -> list[FavId]:
@@ -269,11 +283,7 @@ class BilibiliClient:
f"{BASE}/x/v3/fav/resource/ids?media_id={int(media_id)}&platform=web"
)
payload = self.get_json(url)
data = payload.get("data")
if data is None:
data = []
if not isinstance(data, list):
raise BilibiliProtocolError("fav/resource/ids data 不是数组")
data = self._data_list(payload, "fav/resource/ids")
out: list[FavId] = []
for index, item in enumerate(data):
if not isinstance(item, Mapping):
@@ -320,11 +330,7 @@ class BilibiliClient:
f"&platform=web&folder_mid={fmid}&folder_id={int(media_id)}"
)
payload = self.get_json(url)
data = payload.get("data")
if data is None:
data = []
if not isinstance(data, list):
raise BilibiliProtocolError("fav/resource/infos data 不是数组")
data = self._data_list(payload, "fav/resource/infos")
chunk_set = set(chunk)
for item in data:
if not isinstance(item, Mapping):
+38
View File
@@ -0,0 +1,38 @@
# -*- coding: utf-8 -*-
"""按账号角色提供 B 站客户端,并在凭据变化时自动重建。"""
from __future__ import annotations
from typing import Any, Callable
from ..config import ConfigStore
from .client import BilibiliClient
class ClientProvider:
"""缓存客户端;仅当 ``mid`` / ``cookie`` / ``csrf`` 变化时重建。
``factory`` 负责固化该角色的限速与重试策略,因此重建不会丢失这些设置。
"""
def __init__(
self,
store: ConfigStore,
role: str,
factory: Callable[[Any], BilibiliClient],
) -> None:
if role not in {"read", "write"}:
raise ValueError(f"未知账号角色: {role}")
self.store = store
self.role = role
self.factory = factory
self._key: tuple[int, str, str] | None = None
self._client: BilibiliClient | None = None
def get(self) -> BilibiliClient:
credentials = getattr(self.store.get().bilibili, self.role)
key = (credentials.mid, credentials.cookie, credentials.csrf)
if self._client is None or key != self._key:
self._client = self.factory(credentials)
self._key = key
return self._client
+58 -12
View File
@@ -21,6 +21,7 @@ from typing import Sequence
from .bilibili.auth_flow import AuthService
from .bilibili.client import BilibiliClient, Pacer
from .bilibili.provider import ClientProvider
from .config import ConfigError, ConfigStore
from .database import Database
from .logging_json import JsonLogger
@@ -53,6 +54,11 @@ def _build_parser() -> argparse.ArgumentParser:
sync.add_argument(
"--full", action="store_true", help="强制全量刷新元数据(忽略 6 小时周期)"
)
sync.add_argument(
"--allow-empty-folder",
action="store_true",
help="允许把远端 0 条视为真实清空(默认按异常跳过差集)",
)
snapshot = sub.add_parser(
"snapshot", parents=[common], help="采集当日七项快照与粉丝数"
)
@@ -72,6 +78,11 @@ def _build_parser() -> argparse.ArgumentParser:
run.add_argument(
"--once", action="store_true", help="只执行一次到期任务后退出"
)
run.add_argument(
"--allow-empty-folder",
action="store_true",
help="允许把远端 0 条视为真实清空(默认按异常跳过差集)",
)
sub.add_parser("status", parents=[common], help="查看数据库状态(离线)")
return parser
@@ -130,7 +141,13 @@ def cmd_sync(args, logger: JsonLogger) -> int:
try:
repo = Repository(db)
client = _client(cfg.bilibili.read, logger)
service = FolderSyncService(store, repo, client, logger=logger)
service = FolderSyncService(
store,
repo,
client,
logger=logger,
allow_empty_folders=args.allow_empty_folder,
)
summary = service.run(full_refresh=args.full)
payload = summary.as_dict()
payload["database"] = str(cfg.storage.database_path)
@@ -157,7 +174,7 @@ def cmd_snapshot(args, logger: JsonLogger) -> int:
payload["snapshot_counts"] = repo.snapshot_counts(summary.stat_date)
payload["counts"] = repo.counts()
_print(payload)
return 0 if summary.completed else 1
return 0 if summary.completed and not summary.backup_error else 1
finally:
db.close()
@@ -205,16 +222,26 @@ def cmd_run(args, logger: JsonLogger) -> int:
last: dict[str, object] = {}
try:
repo = Repository(db)
folder_client = _client(cfg.bilibili.read, logger)
folder_service = FolderSyncService(store, repo, folder_client, logger=logger)
snapshot_client = _client(cfg.bilibili.read, logger, pacer=Pacer(1.0, 1.6), retries=1)
snapshot_service = SnapshotService(store, repo, snapshot_client, logger=logger)
def run_sync():
summary = folder_service.run()
last["folder"] = summary
logger.info("folder_sync.summary", **summary.as_dict())
return summary
folder_provider = ClientProvider(
store, "read", lambda credentials: _client(credentials, logger)
)
snapshot_provider = ClientProvider(
store,
"read",
lambda credentials: _client(
credentials, logger, pacer=Pacer(1.0, 1.6), retries=1
),
)
folder_service = FolderSyncService(
store,
repo,
client_provider=folder_provider,
logger=logger,
allow_empty_folders=args.allow_empty_folder,
)
snapshot_service = SnapshotService(
store, repo, client_provider=snapshot_provider, logger=logger
)
def run_snapshot():
summary = snapshot_service.run_if_needed()
@@ -222,6 +249,22 @@ def cmd_run(args, logger: JsonLogger) -> int:
logger.info("snapshot.summary", **summary.as_dict())
return summary
def run_sync():
summary = folder_service.run()
last["folder"] = summary
logger.info("folder_sync.summary", **summary.as_dict())
# 新入库/恢复的视频立即补当天快照,而不是等到次日。
if summary.added > 0:
run_snapshot()
return summary
def run_config_watch():
before = store.revision
store.get()
if store.revision != before:
logger.info("config.reloaded", revision=store.revision)
return {"revision": store.revision}
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
snapshot_at = f"{hh:02d}:{mm:02d}"
scheduler = Scheduler(
@@ -238,6 +281,7 @@ def cmd_run(args, logger: JsonLogger) -> int:
next_run=0.0,
next_run_fn=lambda started: next_daily_run(started, snapshot_at),
),
ScheduledTask("config_watch", 60.0, run_config_watch),
],
logger=logger,
)
@@ -275,6 +319,8 @@ def cmd_status(args, logger: JsonLogger) -> int:
"last_completed": repo.kv_get("snapshot:last_completed"),
"auth_ok": repo.kv_get("auth:ok"),
"auth_message": repo.kv_get("auth:message"),
"backup_ok": repo.kv_get(f"snapshot:backup:{today}"),
"backup_error": repo.kv_get(f"snapshot:backup_error:{today}"),
})
_print({
"ok": True,
+2
View File
@@ -478,6 +478,7 @@ class ConfigStore:
self._config = load_config(path, env=env, require_auth=require_auth)
self._mtime_ns = self._read_mtime_ns()
self.last_error: str | None = None
self.revision = 0
def _read_mtime_ns(self) -> int | None:
try:
@@ -499,4 +500,5 @@ class ConfigStore:
self.last_error = str(exc)
return self._config
self.last_error = None
self.revision += 1
return self._config
+7
View File
@@ -167,6 +167,13 @@ MIGRATIONS: tuple[Migration, ...] = (
""",
),
),
Migration(
version=2,
name="folder_resource fav_time",
statements=(
"ALTER TABLE folder_resource ADD COLUMN fav_time INTEGER",
),
),
)
+64 -7
View File
@@ -166,6 +166,35 @@ class Repository:
# ------------------------------------------------------------ video
@staticmethod
def _recompute_primary_folder(conn: sqlite3.Connection, aid: int) -> int | None:
"""按有效收藏关系重算主归属,避免依赖收藏夹遍历顺序。
优先使用逐夹 ``fav_time``(后加入者优先),其次 ``first_seen``,
最后用年度与 media_id 做确定性决胜。
"""
row = conn.execute(
"""
SELECT fr.media_id
FROM folder_resource fr
JOIN folder f ON f.media_id = fr.media_id
WHERE fr.aid = ? AND fr.active = 1 AND f.enabled = 1
ORDER BY COALESCE(fr.fav_time, 0) DESC,
fr.first_seen DESC,
f.year DESC,
fr.media_id DESC
LIMIT 1
""",
(int(aid),),
).fetchone()
if row is None:
return None
folder_id = int(row["media_id"])
conn.execute(
"UPDATE video SET folder_id = ? WHERE aid = ?", (folder_id, int(aid))
)
return folder_id
def upsert_from_fav(
self,
resource,
@@ -206,7 +235,8 @@ class Repository:
ON CONFLICT(aid) DO UPDATE SET
bvid = COALESCE(excluded.bvid, video.bvid),
title = CASE
WHEN excluded.status = 'removed' AND video.title != ''
WHEN (excluded.attr = 9 OR excluded.status = 'removed')
AND video.title != ''
THEN video.title
WHEN excluded.title != '' THEN excluded.title
ELSE video.title
@@ -219,7 +249,10 @@ class Repository:
END,
duration = COALESCE(excluded.duration, video.duration),
cover = COALESCE(excluded.cover, video.cover),
folder_id = excluded.folder_id,
folder_id = CASE
WHEN video.folder_id IS NULL THEN excluded.folder_id
ELSE video.folder_id
END,
fav_time = COALESCE(excluded.fav_time, video.fav_time),
status = excluded.status,
source = COALESCE(video.source, excluded.source),
@@ -246,19 +279,22 @@ class Repository:
)
conn.execute(
"""
INSERT INTO folder_resource(media_id, aid, first_seen, active, last_seen)
VALUES (?, ?, ?, 1, ?)
INSERT INTO folder_resource(media_id, aid, first_seen, active,
last_seen, fav_time)
VALUES (?, ?, ?, 1, ?, ?)
ON CONFLICT(media_id, aid) DO UPDATE SET
active = 1,
last_seen = excluded.last_seen
last_seen = excluded.last_seen,
fav_time = COALESCE(excluded.fav_time, folder_resource.fav_time)
""",
(folder_id, resource.aid, now, now),
(folder_id, resource.aid, now, now, resource.fav_time),
)
primary = self._recompute_primary_folder(conn, resource.aid)
return UpsertOutcome(
aid=resource.aid,
status=status,
previous_folder_id=previous_folder,
moved_folder=previous_folder is not None and previous_folder != folder_id,
moved_folder=previous_folder is not None and previous_folder != primary,
)
return self.db.write(run)
@@ -304,6 +340,7 @@ class Repository:
""",
(folder_id, aid, now, now),
)
self._recompute_primary_folder(conn, aid)
return status
return self.db.write(run)
@@ -320,6 +357,7 @@ class Repository:
"WHERE media_id = ? AND aid = ?",
(now, media_id, aid),
)
self._recompute_primary_folder(conn, aid)
other = conn.execute(
"""
SELECT 1
@@ -379,6 +417,25 @@ class Repository:
return [int(row["aid"]) for row in rows]
return self.db.read(run)
def pending_snapshot_aids(self, stat_date: str) -> list[int]:
"""当天仍缺成功快照的追踪视频;用于新入库/恢复后的同日补采。"""
def run(conn: sqlite3.Connection) -> list[int]:
rows = conn.execute(
"""
SELECT v.aid
FROM video v
WHERE v.status IN ('active', 'unfaved')
AND NOT EXISTS (
SELECT 1 FROM snapshot s
WHERE s.stat_date = ? AND s.aid = v.aid AND s.ok = 1
)
ORDER BY v.aid
""",
(stat_date,),
).fetchall()
return [int(row["aid"]) for row in rows]
return self.db.read(run)
def get_snapshot(self, stat_date: str, aid: int) -> dict[str, Any] | None:
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
row = conn.execute(
+26 -5
View File
@@ -19,7 +19,6 @@ import time
from dataclasses import dataclass, field
from typing import Any
from ..bilibili.client import BilibiliClient
from ..config import AppConfig, ConfigStore, FolderConfig
from ..repository import Repository
@@ -113,18 +112,29 @@ class FolderSyncService:
self,
config_store: ConfigStore,
repository: Repository,
client: BilibiliClient,
client=None,
*,
client_provider=None,
clock=time.time,
logger=None,
capacity_warn: int = CAPACITY_WARN,
allow_empty_folders: bool = False,
) -> None:
if client is None and client_provider is None:
raise ValueError("必须提供 client 或 client_provider")
self.config_store = config_store
self.repo = repository
self.client = client
self.client_provider = client_provider
self._clock = clock
self.logger = logger
self.capacity_warn = capacity_warn
self.allow_empty_folders = allow_empty_folders
def _client(self):
if self.client_provider is not None:
return self.client_provider.get()
return self.client
def _now(self) -> int:
return int(self._clock())
@@ -135,6 +145,7 @@ class FolderSyncService:
def run(self, *, full_refresh: bool = False) -> SyncSummary:
cfg = self.config_store.get()
client = self._client()
now = self._now()
summary = SyncSummary(started_at=now)
self._log(
@@ -151,7 +162,9 @@ class FolderSyncService:
for folder in cfg.folders:
try:
result = self._sync_folder(cfg, folder, now=now, force_full=full_refresh)
result = self._sync_folder(
client, cfg, folder, now=now, force_full=full_refresh
)
except Exception as exc: # noqa: BLE001 单个夹失败不阻塞其它夹
result = FolderSyncResult(
media_id=folder.media_id,
@@ -191,6 +204,7 @@ class FolderSyncService:
def _sync_folder(
self,
client,
cfg: AppConfig,
folder: FolderConfig,
*,
@@ -206,13 +220,20 @@ class FolderSyncService:
>= cfg.schedule.full_folder_refresh_hours * 3600
)
resources = self.client.fav_ids(folder.media_id)
resources = client.fav_ids(folder.media_id)
remote_map = {item.aid: item for item in resources if item.type == 2}
remote = set(remote_map)
local = self.repo.folder_active_aids(folder.media_id)
added = remote - local
removed = local - remote
if not remote and local and not self.allow_empty_folders:
raise RuntimeError(
f"收藏夹 {folder.media_id} 远端返回 0 条,但本地仍有 {len(local)} 条;"
f"疑似接口异常,已跳过差集。确认收藏夹确实被清空后,"
f"可使用 --allow-empty-folder 重试"
)
result = FolderSyncResult(
media_id=folder.media_id,
year=folder.year,
@@ -228,7 +249,7 @@ class FolderSyncService:
targets |= remote
if targets:
infos = self.client.fav_infos(
infos = client.fav_infos(
folder.media_id,
sorted(targets),
folder_mid=cfg.bilibili.read.mid,
+88 -23
View File
@@ -20,7 +20,6 @@ from typing import Any
from ..bilibili.client import (
VIDEO_UNAVAILABLE_CODES,
BilibiliAPIError,
BilibiliClient,
BilibiliError,
BilibiliHTTPError,
)
@@ -41,6 +40,7 @@ class SnapshotSummary:
removed: int = 0
fans_ok: int = 0
fans_failed: int = 0
fans_skipped: int = 0
auth_ok: bool | None = None
auth_message: str = ""
backup_path: str | None = None
@@ -63,6 +63,7 @@ class SnapshotSummary:
"removed": self.removed,
"fans_ok": self.fans_ok,
"fans_failed": self.fans_failed,
"fans_skipped": self.fans_skipped,
"auth_ok": self.auth_ok,
"auth_message": self.auth_message,
"backup_path": self.backup_path,
@@ -78,8 +79,9 @@ class SnapshotService:
self,
config_store: ConfigStore,
repository: Repository,
client: BilibiliClient,
client=None,
*,
client_provider=None,
clock=time.time,
sleeper=time.sleep,
logger=None,
@@ -89,9 +91,12 @@ class SnapshotService:
) -> None:
if retries < 1:
raise ValueError("retries 必须 >= 1")
if client is None and client_provider is None:
raise ValueError("必须提供 client 或 client_provider")
self.config_store = config_store
self.repo = repository
self.client = client
self.client_provider = client_provider
self._clock = clock
self._sleep = sleeper
self.logger = logger
@@ -102,6 +107,11 @@ class SnapshotService:
def _now(self) -> int:
return int(self._clock())
def _client(self):
if self.client_provider is not None:
return self.client_provider.get()
return self.client
def _log(self, level: str, event: str, **fields) -> None:
if self.logger is not None:
getattr(self.logger, level)(event, **fields)
@@ -110,6 +120,14 @@ class SnapshotService:
def completed_key(stat_date: str) -> str:
return f"snapshot:completed:{stat_date}"
@staticmethod
def backup_key(stat_date: str) -> str:
return f"snapshot:backup:{stat_date}"
@staticmethod
def backup_error_key(stat_date: str) -> str:
return f"snapshot:backup_error:{stat_date}"
def run_if_needed(
self, *, date: str | None = None, force: bool = False
) -> SnapshotSummary:
@@ -117,7 +135,14 @@ class SnapshotService:
stat_date = date or shanghai_date(now)
if not is_date(stat_date):
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
if not force and self.repo.kv_get(self.completed_key(stat_date)) == "1":
if force:
return self.run(date=stat_date, force=True)
completed = self.repo.kv_get(self.completed_key(stat_date)) == "1"
pending = self.repo.pending_snapshot_aids(stat_date)
if completed and not pending:
if self.repo.kv_get(self.backup_key(stat_date)) != "1":
return self.backup_only(stat_date)
self._log("info", "snapshot.skip", stat_date=stat_date, reason="already_completed")
return SnapshotSummary(
stat_date=stat_date,
@@ -126,10 +151,27 @@ class SnapshotService:
completed=True,
already_completed=True,
)
return self.run(date=stat_date, force=force)
return self.run(date=stat_date, force=False)
def backup_only(self, stat_date: str) -> SnapshotSummary:
"""采集已完成但备份缺失时,只重试备份,不请求任何外部接口。"""
now = self._now()
cfg = self.config_store.get()
summary = SnapshotSummary(
stat_date=stat_date,
started_at=now,
completed=True,
already_completed=True,
)
self._log("info", "snapshot.backup_retry", stat_date=stat_date)
self._run_backup(cfg, stat_date, summary)
summary.finished_at = self._now()
self._log("info", "snapshot.done", **summary.as_dict())
return summary
def run(self, *, date: str | None = None, force: bool = False) -> SnapshotSummary:
cfg = self.config_store.get()
client = self._client()
now = self._now()
stat_date = date or shanghai_date(now)
if not is_date(stat_date):
@@ -137,8 +179,12 @@ class SnapshotService:
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
self._log("info", "snapshot.start", stat_date=stat_date, force=force)
# 先落“未完成”,保证强制重采或中途中断后仍可续跑。
self.repo.kv_set(self.completed_key(stat_date), "0")
if self.repo.kv_get("snapshot:last_completed") == stat_date:
self.repo.kv_set("snapshot:last_completed", "")
summary.auth_ok, summary.auth_message = self._check_auth(cfg, now)
summary.auth_ok, summary.auth_message = self._check_auth(client, cfg, now)
videos = self.repo.video_aids_for_snapshot()
summary.total = len(videos)
@@ -148,7 +194,7 @@ class SnapshotService:
summary.skipped += 1
continue
stat, error, unavailable = self._fetch_view(aid)
stat, error, unavailable = self._fetch_view(client, aid)
if unavailable:
summary.removed += 1
self.repo.set_video_status(aid, "removed", now=now)
@@ -193,7 +239,15 @@ class SnapshotService:
)
for mid in cfg.track_fans:
stat, error = self._fetch_fan(mid)
existing_fan = self.repo.get_up_stat(stat_date, mid)
if (
not force
and existing_fan is not None
and int(existing_fan.get("ok") or 0) == 1
):
summary.fans_skipped += 1
continue
stat, error = self._fetch_fan(client, mid)
if stat is not None:
self.repo.upsert_up_stat(
stat_date,
@@ -228,6 +282,21 @@ class SnapshotService:
elif self.repo.kv_get("snapshot:last_completed") == stat_date:
self.repo.kv_set("snapshot:last_completed", "")
self._run_backup(cfg, stat_date, summary)
summary.finished_at = self._now()
self._log(
"info" if summary.completed else "warning",
"snapshot.done",
**summary.as_dict(),
)
return summary
# ------------------------------------------------------------ internals
def _run_backup(
self, cfg: AppConfig, stat_date: str, summary: SnapshotSummary
) -> None:
try:
path = backup_daily(
self.repo.db,
@@ -240,28 +309,24 @@ class SnapshotService:
keep_days=cfg.storage.backup_keep_days,
)
summary.backup_path = str(path)
self.repo.kv_set(self.backup_key(stat_date), "1")
self.repo.kv_set(self.backup_error_key(stat_date), "")
except Exception as exc: # noqa: BLE001 备份失败不推翻已采集数据
summary.backup_error = str(exc)
self.repo.kv_set(self.backup_key(stat_date), "0")
self.repo.kv_set(self.backup_error_key(stat_date), str(exc))
self._log("error", "backup.failed", error=str(exc), stat_date=stat_date)
summary.finished_at = self._now()
self._log(
"info" if summary.completed else "warning",
"snapshot.done",
**summary.as_dict(),
)
return summary
# ------------------------------------------------------------ internals
def _check_auth(self, cfg: AppConfig, now: int) -> tuple[bool, str]:
def _check_auth(
self, client, cfg: AppConfig, now: int
) -> tuple[bool, str]:
if not cfg.bilibili.read.cookie:
self.repo.kv_set("auth:last_check", str(now))
self.repo.kv_set("auth:ok", "0")
self.repo.kv_set("auth:message", "未配置 cookie")
return False, "未配置 cookie"
try:
state = self.client.check_login()
state = client.check_login()
except BilibiliError as exc:
message = str(exc)
self.repo.kv_set("auth:last_check", str(now))
@@ -282,11 +347,11 @@ class SnapshotService:
)
return state.is_login, state.message
def _fetch_view(self, aid: int):
def _fetch_view(self, client, aid: int):
last_error = ""
for attempt in range(1, self.retries + 1):
try:
return self.client.view(aid), None, False
return client.view(aid), None, False
except BilibiliAPIError as exc:
last_error = str(exc)
if exc.code in VIDEO_UNAVAILABLE_CODES:
@@ -301,11 +366,11 @@ class SnapshotService:
self._sleep(self.backoff * (2 ** (attempt - 1)))
return None, last_error or "未知错误", False
def _fetch_fan(self, mid: int):
def _fetch_fan(self, client, mid: int):
last_error = ""
for attempt in range(1, self.retries + 1):
try:
return self.client.relation_stat(mid), None
return client.relation_stat(mid), None
except BilibiliError as exc:
last_error = str(exc)
if attempt < self.retries: