fix: close data integrity and hot-reload review findings
Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.
This commit is contained in:
1 parent
178ab64b18
commit
6a21b5d905
21 files changed
+895
-82
No files matched your search
@@ -262,6 +262,20 @@ class BilibiliClient:
|
||||
raise self._api_error(payload)
|
||||
return payload
|
||||
|
||||
@staticmethod
|
||||
def _data_list(payload: Mapping[str, Any], name: str) -> list[Any]:
|
||||
"""区分“真实空数组”与“协议缺字段”。
|
||||
|
||||
实测空收藏夹返回 ``{"code":0,"data":[]}``;缺少 ``data`` 或为 ``null``
|
||||
都不是已知契约,必须中止,避免被误判成空收藏夹后批量改状态。
|
||||
"""
|
||||
if "data" not in payload:
|
||||
raise BilibiliProtocolError(f"{name} 响应缺少 data 字段")
|
||||
data = payload["data"]
|
||||
if not isinstance(data, list):
|
||||
raise BilibiliProtocolError(f"{name} data 不是数组")
|
||||
return data
|
||||
|
||||
# ------------------------------------------------------------ fav
|
||||
|
||||
def fav_ids(self, media_id: int) -> list[FavId]:
|
||||
@@ -269,11 +283,7 @@ class BilibiliClient:
|
||||
f"{BASE}/x/v3/fav/resource/ids?media_id={int(media_id)}&platform=web"
|
||||
)
|
||||
payload = self.get_json(url)
|
||||
data = payload.get("data")
|
||||
if data is None:
|
||||
data = []
|
||||
if not isinstance(data, list):
|
||||
raise BilibiliProtocolError("fav/resource/ids data 不是数组")
|
||||
data = self._data_list(payload, "fav/resource/ids")
|
||||
out: list[FavId] = []
|
||||
for index, item in enumerate(data):
|
||||
if not isinstance(item, Mapping):
|
||||
@@ -320,11 +330,7 @@ class BilibiliClient:
|
||||
f"&platform=web&folder_mid={fmid}&folder_id={int(media_id)}"
|
||||
)
|
||||
payload = self.get_json(url)
|
||||
data = payload.get("data")
|
||||
if data is None:
|
||||
data = []
|
||||
if not isinstance(data, list):
|
||||
raise BilibiliProtocolError("fav/resource/infos data 不是数组")
|
||||
data = self._data_list(payload, "fav/resource/infos")
|
||||
chunk_set = set(chunk)
|
||||
for item in data:
|
||||
if not isinstance(item, Mapping):
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""按账号角色提供 B 站客户端,并在凭据变化时自动重建。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
from ..config import ConfigStore
|
||||
from .client import BilibiliClient
|
||||
|
||||
|
||||
class ClientProvider:
|
||||
"""缓存客户端;仅当 ``mid`` / ``cookie`` / ``csrf`` 变化时重建。
|
||||
|
||||
``factory`` 负责固化该角色的限速与重试策略,因此重建不会丢失这些设置。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: ConfigStore,
|
||||
role: str,
|
||||
factory: Callable[[Any], BilibiliClient],
|
||||
) -> None:
|
||||
if role not in {"read", "write"}:
|
||||
raise ValueError(f"未知账号角色: {role}")
|
||||
self.store = store
|
||||
self.role = role
|
||||
self.factory = factory
|
||||
self._key: tuple[int, str, str] | None = None
|
||||
self._client: BilibiliClient | None = None
|
||||
|
||||
def get(self) -> BilibiliClient:
|
||||
credentials = getattr(self.store.get().bilibili, self.role)
|
||||
key = (credentials.mid, credentials.cookie, credentials.csrf)
|
||||
if self._client is None or key != self._key:
|
||||
self._client = self.factory(credentials)
|
||||
self._key = key
|
||||
return self._client
|
||||
+58
-12
@@ -21,6 +21,7 @@ from typing import Sequence
|
||||
|
||||
from .bilibili.auth_flow import AuthService
|
||||
from .bilibili.client import BilibiliClient, Pacer
|
||||
from .bilibili.provider import ClientProvider
|
||||
from .config import ConfigError, ConfigStore
|
||||
from .database import Database
|
||||
from .logging_json import JsonLogger
|
||||
@@ -53,6 +54,11 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
sync.add_argument(
|
||||
"--full", action="store_true", help="强制全量刷新元数据(忽略 6 小时周期)"
|
||||
)
|
||||
sync.add_argument(
|
||||
"--allow-empty-folder",
|
||||
action="store_true",
|
||||
help="允许把远端 0 条视为真实清空(默认按异常跳过差集)",
|
||||
)
|
||||
snapshot = sub.add_parser(
|
||||
"snapshot", parents=[common], help="采集当日七项快照与粉丝数"
|
||||
)
|
||||
@@ -72,6 +78,11 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
run.add_argument(
|
||||
"--once", action="store_true", help="只执行一次到期任务后退出"
|
||||
)
|
||||
run.add_argument(
|
||||
"--allow-empty-folder",
|
||||
action="store_true",
|
||||
help="允许把远端 0 条视为真实清空(默认按异常跳过差集)",
|
||||
)
|
||||
sub.add_parser("status", parents=[common], help="查看数据库状态(离线)")
|
||||
return parser
|
||||
|
||||
@@ -130,7 +141,13 @@ def cmd_sync(args, logger: JsonLogger) -> int:
|
||||
try:
|
||||
repo = Repository(db)
|
||||
client = _client(cfg.bilibili.read, logger)
|
||||
service = FolderSyncService(store, repo, client, logger=logger)
|
||||
service = FolderSyncService(
|
||||
store,
|
||||
repo,
|
||||
client,
|
||||
logger=logger,
|
||||
allow_empty_folders=args.allow_empty_folder,
|
||||
)
|
||||
summary = service.run(full_refresh=args.full)
|
||||
payload = summary.as_dict()
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
@@ -157,7 +174,7 @@ def cmd_snapshot(args, logger: JsonLogger) -> int:
|
||||
payload["snapshot_counts"] = repo.snapshot_counts(summary.stat_date)
|
||||
payload["counts"] = repo.counts()
|
||||
_print(payload)
|
||||
return 0 if summary.completed else 1
|
||||
return 0 if summary.completed and not summary.backup_error else 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -205,16 +222,26 @@ def cmd_run(args, logger: JsonLogger) -> int:
|
||||
last: dict[str, object] = {}
|
||||
try:
|
||||
repo = Repository(db)
|
||||
folder_client = _client(cfg.bilibili.read, logger)
|
||||
folder_service = FolderSyncService(store, repo, folder_client, logger=logger)
|
||||
snapshot_client = _client(cfg.bilibili.read, logger, pacer=Pacer(1.0, 1.6), retries=1)
|
||||
snapshot_service = SnapshotService(store, repo, snapshot_client, logger=logger)
|
||||
|
||||
def run_sync():
|
||||
summary = folder_service.run()
|
||||
last["folder"] = summary
|
||||
logger.info("folder_sync.summary", **summary.as_dict())
|
||||
return summary
|
||||
folder_provider = ClientProvider(
|
||||
store, "read", lambda credentials: _client(credentials, logger)
|
||||
)
|
||||
snapshot_provider = ClientProvider(
|
||||
store,
|
||||
"read",
|
||||
lambda credentials: _client(
|
||||
credentials, logger, pacer=Pacer(1.0, 1.6), retries=1
|
||||
),
|
||||
)
|
||||
folder_service = FolderSyncService(
|
||||
store,
|
||||
repo,
|
||||
client_provider=folder_provider,
|
||||
logger=logger,
|
||||
allow_empty_folders=args.allow_empty_folder,
|
||||
)
|
||||
snapshot_service = SnapshotService(
|
||||
store, repo, client_provider=snapshot_provider, logger=logger
|
||||
)
|
||||
|
||||
def run_snapshot():
|
||||
summary = snapshot_service.run_if_needed()
|
||||
@@ -222,6 +249,22 @@ def cmd_run(args, logger: JsonLogger) -> int:
|
||||
logger.info("snapshot.summary", **summary.as_dict())
|
||||
return summary
|
||||
|
||||
def run_sync():
|
||||
summary = folder_service.run()
|
||||
last["folder"] = summary
|
||||
logger.info("folder_sync.summary", **summary.as_dict())
|
||||
# 新入库/恢复的视频立即补当天快照,而不是等到次日。
|
||||
if summary.added > 0:
|
||||
run_snapshot()
|
||||
return summary
|
||||
|
||||
def run_config_watch():
|
||||
before = store.revision
|
||||
store.get()
|
||||
if store.revision != before:
|
||||
logger.info("config.reloaded", revision=store.revision)
|
||||
return {"revision": store.revision}
|
||||
|
||||
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
|
||||
snapshot_at = f"{hh:02d}:{mm:02d}"
|
||||
scheduler = Scheduler(
|
||||
@@ -238,6 +281,7 @@ def cmd_run(args, logger: JsonLogger) -> int:
|
||||
next_run=0.0,
|
||||
next_run_fn=lambda started: next_daily_run(started, snapshot_at),
|
||||
),
|
||||
ScheduledTask("config_watch", 60.0, run_config_watch),
|
||||
],
|
||||
logger=logger,
|
||||
)
|
||||
@@ -275,6 +319,8 @@ def cmd_status(args, logger: JsonLogger) -> int:
|
||||
"last_completed": repo.kv_get("snapshot:last_completed"),
|
||||
"auth_ok": repo.kv_get("auth:ok"),
|
||||
"auth_message": repo.kv_get("auth:message"),
|
||||
"backup_ok": repo.kv_get(f"snapshot:backup:{today}"),
|
||||
"backup_error": repo.kv_get(f"snapshot:backup_error:{today}"),
|
||||
})
|
||||
_print({
|
||||
"ok": True,
|
||||
|
||||
@@ -478,6 +478,7 @@ class ConfigStore:
|
||||
self._config = load_config(path, env=env, require_auth=require_auth)
|
||||
self._mtime_ns = self._read_mtime_ns()
|
||||
self.last_error: str | None = None
|
||||
self.revision = 0
|
||||
|
||||
def _read_mtime_ns(self) -> int | None:
|
||||
try:
|
||||
@@ -499,4 +500,5 @@ class ConfigStore:
|
||||
self.last_error = str(exc)
|
||||
return self._config
|
||||
self.last_error = None
|
||||
self.revision += 1
|
||||
return self._config
|
||||
@@ -167,6 +167,13 @@ MIGRATIONS: tuple[Migration, ...] = (
|
||||
""",
|
||||
),
|
||||
),
|
||||
Migration(
|
||||
version=2,
|
||||
name="folder_resource fav_time",
|
||||
statements=(
|
||||
"ALTER TABLE folder_resource ADD COLUMN fav_time INTEGER",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -166,6 +166,35 @@ class Repository:
|
||||
|
||||
# ------------------------------------------------------------ video
|
||||
|
||||
@staticmethod
|
||||
def _recompute_primary_folder(conn: sqlite3.Connection, aid: int) -> int | None:
|
||||
"""按有效收藏关系重算主归属,避免依赖收藏夹遍历顺序。
|
||||
|
||||
优先使用逐夹 ``fav_time``(后加入者优先),其次 ``first_seen``,
|
||||
最后用年度与 media_id 做确定性决胜。
|
||||
"""
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT fr.media_id
|
||||
FROM folder_resource fr
|
||||
JOIN folder f ON f.media_id = fr.media_id
|
||||
WHERE fr.aid = ? AND fr.active = 1 AND f.enabled = 1
|
||||
ORDER BY COALESCE(fr.fav_time, 0) DESC,
|
||||
fr.first_seen DESC,
|
||||
f.year DESC,
|
||||
fr.media_id DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(int(aid),),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
folder_id = int(row["media_id"])
|
||||
conn.execute(
|
||||
"UPDATE video SET folder_id = ? WHERE aid = ?", (folder_id, int(aid))
|
||||
)
|
||||
return folder_id
|
||||
|
||||
def upsert_from_fav(
|
||||
self,
|
||||
resource,
|
||||
@@ -206,7 +235,8 @@ class Repository:
|
||||
ON CONFLICT(aid) DO UPDATE SET
|
||||
bvid = COALESCE(excluded.bvid, video.bvid),
|
||||
title = CASE
|
||||
WHEN excluded.status = 'removed' AND video.title != ''
|
||||
WHEN (excluded.attr = 9 OR excluded.status = 'removed')
|
||||
AND video.title != ''
|
||||
THEN video.title
|
||||
WHEN excluded.title != '' THEN excluded.title
|
||||
ELSE video.title
|
||||
@@ -219,7 +249,10 @@ class Repository:
|
||||
END,
|
||||
duration = COALESCE(excluded.duration, video.duration),
|
||||
cover = COALESCE(excluded.cover, video.cover),
|
||||
folder_id = excluded.folder_id,
|
||||
folder_id = CASE
|
||||
WHEN video.folder_id IS NULL THEN excluded.folder_id
|
||||
ELSE video.folder_id
|
||||
END,
|
||||
fav_time = COALESCE(excluded.fav_time, video.fav_time),
|
||||
status = excluded.status,
|
||||
source = COALESCE(video.source, excluded.source),
|
||||
@@ -246,19 +279,22 @@ class Repository:
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO folder_resource(media_id, aid, first_seen, active, last_seen)
|
||||
VALUES (?, ?, ?, 1, ?)
|
||||
INSERT INTO folder_resource(media_id, aid, first_seen, active,
|
||||
last_seen, fav_time)
|
||||
VALUES (?, ?, ?, 1, ?, ?)
|
||||
ON CONFLICT(media_id, aid) DO UPDATE SET
|
||||
active = 1,
|
||||
last_seen = excluded.last_seen
|
||||
last_seen = excluded.last_seen,
|
||||
fav_time = COALESCE(excluded.fav_time, folder_resource.fav_time)
|
||||
""",
|
||||
(folder_id, resource.aid, now, now),
|
||||
(folder_id, resource.aid, now, now, resource.fav_time),
|
||||
)
|
||||
primary = self._recompute_primary_folder(conn, resource.aid)
|
||||
return UpsertOutcome(
|
||||
aid=resource.aid,
|
||||
status=status,
|
||||
previous_folder_id=previous_folder,
|
||||
moved_folder=previous_folder is not None and previous_folder != folder_id,
|
||||
moved_folder=previous_folder is not None and previous_folder != primary,
|
||||
)
|
||||
return self.db.write(run)
|
||||
|
||||
@@ -304,6 +340,7 @@ class Repository:
|
||||
""",
|
||||
(folder_id, aid, now, now),
|
||||
)
|
||||
self._recompute_primary_folder(conn, aid)
|
||||
return status
|
||||
return self.db.write(run)
|
||||
|
||||
@@ -320,6 +357,7 @@ class Repository:
|
||||
"WHERE media_id = ? AND aid = ?",
|
||||
(now, media_id, aid),
|
||||
)
|
||||
self._recompute_primary_folder(conn, aid)
|
||||
other = conn.execute(
|
||||
"""
|
||||
SELECT 1
|
||||
@@ -379,6 +417,25 @@ class Repository:
|
||||
return [int(row["aid"]) for row in rows]
|
||||
return self.db.read(run)
|
||||
|
||||
def pending_snapshot_aids(self, stat_date: str) -> list[int]:
|
||||
"""当天仍缺成功快照的追踪视频;用于新入库/恢复后的同日补采。"""
|
||||
def run(conn: sqlite3.Connection) -> list[int]:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT v.aid
|
||||
FROM video v
|
||||
WHERE v.status IN ('active', 'unfaved')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM snapshot s
|
||||
WHERE s.stat_date = ? AND s.aid = v.aid AND s.ok = 1
|
||||
)
|
||||
ORDER BY v.aid
|
||||
""",
|
||||
(stat_date,),
|
||||
).fetchall()
|
||||
return [int(row["aid"]) for row in rows]
|
||||
return self.db.read(run)
|
||||
|
||||
def get_snapshot(self, stat_date: str, aid: int) -> dict[str, Any] | None:
|
||||
def run(conn: sqlite3.Connection) -> dict[str, Any] | None:
|
||||
row = conn.execute(
|
||||
|
||||
@@ -19,7 +19,6 @@ import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from ..bilibili.client import BilibiliClient
|
||||
from ..config import AppConfig, ConfigStore, FolderConfig
|
||||
from ..repository import Repository
|
||||
|
||||
@@ -113,18 +112,29 @@ class FolderSyncService:
|
||||
self,
|
||||
config_store: ConfigStore,
|
||||
repository: Repository,
|
||||
client: BilibiliClient,
|
||||
client=None,
|
||||
*,
|
||||
client_provider=None,
|
||||
clock=time.time,
|
||||
logger=None,
|
||||
capacity_warn: int = CAPACITY_WARN,
|
||||
allow_empty_folders: bool = False,
|
||||
) -> None:
|
||||
if client is None and client_provider is None:
|
||||
raise ValueError("必须提供 client 或 client_provider")
|
||||
self.config_store = config_store
|
||||
self.repo = repository
|
||||
self.client = client
|
||||
self.client_provider = client_provider
|
||||
self._clock = clock
|
||||
self.logger = logger
|
||||
self.capacity_warn = capacity_warn
|
||||
self.allow_empty_folders = allow_empty_folders
|
||||
|
||||
def _client(self):
|
||||
if self.client_provider is not None:
|
||||
return self.client_provider.get()
|
||||
return self.client
|
||||
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
@@ -135,6 +145,7 @@ class FolderSyncService:
|
||||
|
||||
def run(self, *, full_refresh: bool = False) -> SyncSummary:
|
||||
cfg = self.config_store.get()
|
||||
client = self._client()
|
||||
now = self._now()
|
||||
summary = SyncSummary(started_at=now)
|
||||
self._log(
|
||||
@@ -151,7 +162,9 @@ class FolderSyncService:
|
||||
|
||||
for folder in cfg.folders:
|
||||
try:
|
||||
result = self._sync_folder(cfg, folder, now=now, force_full=full_refresh)
|
||||
result = self._sync_folder(
|
||||
client, cfg, folder, now=now, force_full=full_refresh
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 单个夹失败不阻塞其它夹
|
||||
result = FolderSyncResult(
|
||||
media_id=folder.media_id,
|
||||
@@ -191,6 +204,7 @@ class FolderSyncService:
|
||||
|
||||
def _sync_folder(
|
||||
self,
|
||||
client,
|
||||
cfg: AppConfig,
|
||||
folder: FolderConfig,
|
||||
*,
|
||||
@@ -206,13 +220,20 @@ class FolderSyncService:
|
||||
>= cfg.schedule.full_folder_refresh_hours * 3600
|
||||
)
|
||||
|
||||
resources = self.client.fav_ids(folder.media_id)
|
||||
resources = client.fav_ids(folder.media_id)
|
||||
remote_map = {item.aid: item for item in resources if item.type == 2}
|
||||
remote = set(remote_map)
|
||||
local = self.repo.folder_active_aids(folder.media_id)
|
||||
added = remote - local
|
||||
removed = local - remote
|
||||
|
||||
if not remote and local and not self.allow_empty_folders:
|
||||
raise RuntimeError(
|
||||
f"收藏夹 {folder.media_id} 远端返回 0 条,但本地仍有 {len(local)} 条;"
|
||||
f"疑似接口异常,已跳过差集。确认收藏夹确实被清空后,"
|
||||
f"可使用 --allow-empty-folder 重试"
|
||||
)
|
||||
|
||||
result = FolderSyncResult(
|
||||
media_id=folder.media_id,
|
||||
year=folder.year,
|
||||
@@ -228,7 +249,7 @@ class FolderSyncService:
|
||||
targets |= remote
|
||||
|
||||
if targets:
|
||||
infos = self.client.fav_infos(
|
||||
infos = client.fav_infos(
|
||||
folder.media_id,
|
||||
sorted(targets),
|
||||
folder_mid=cfg.bilibili.read.mid,
|
||||
|
||||
@@ -20,7 +20,6 @@ from typing import Any
|
||||
from ..bilibili.client import (
|
||||
VIDEO_UNAVAILABLE_CODES,
|
||||
BilibiliAPIError,
|
||||
BilibiliClient,
|
||||
BilibiliError,
|
||||
BilibiliHTTPError,
|
||||
)
|
||||
@@ -41,6 +40,7 @@ class SnapshotSummary:
|
||||
removed: int = 0
|
||||
fans_ok: int = 0
|
||||
fans_failed: int = 0
|
||||
fans_skipped: int = 0
|
||||
auth_ok: bool | None = None
|
||||
auth_message: str = ""
|
||||
backup_path: str | None = None
|
||||
@@ -63,6 +63,7 @@ class SnapshotSummary:
|
||||
"removed": self.removed,
|
||||
"fans_ok": self.fans_ok,
|
||||
"fans_failed": self.fans_failed,
|
||||
"fans_skipped": self.fans_skipped,
|
||||
"auth_ok": self.auth_ok,
|
||||
"auth_message": self.auth_message,
|
||||
"backup_path": self.backup_path,
|
||||
@@ -78,8 +79,9 @@ class SnapshotService:
|
||||
self,
|
||||
config_store: ConfigStore,
|
||||
repository: Repository,
|
||||
client: BilibiliClient,
|
||||
client=None,
|
||||
*,
|
||||
client_provider=None,
|
||||
clock=time.time,
|
||||
sleeper=time.sleep,
|
||||
logger=None,
|
||||
@@ -89,9 +91,12 @@ class SnapshotService:
|
||||
) -> None:
|
||||
if retries < 1:
|
||||
raise ValueError("retries 必须 >= 1")
|
||||
if client is None and client_provider is None:
|
||||
raise ValueError("必须提供 client 或 client_provider")
|
||||
self.config_store = config_store
|
||||
self.repo = repository
|
||||
self.client = client
|
||||
self.client_provider = client_provider
|
||||
self._clock = clock
|
||||
self._sleep = sleeper
|
||||
self.logger = logger
|
||||
@@ -102,6 +107,11 @@ class SnapshotService:
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
|
||||
def _client(self):
|
||||
if self.client_provider is not None:
|
||||
return self.client_provider.get()
|
||||
return self.client
|
||||
|
||||
def _log(self, level: str, event: str, **fields) -> None:
|
||||
if self.logger is not None:
|
||||
getattr(self.logger, level)(event, **fields)
|
||||
@@ -110,6 +120,14 @@ class SnapshotService:
|
||||
def completed_key(stat_date: str) -> str:
|
||||
return f"snapshot:completed:{stat_date}"
|
||||
|
||||
@staticmethod
|
||||
def backup_key(stat_date: str) -> str:
|
||||
return f"snapshot:backup:{stat_date}"
|
||||
|
||||
@staticmethod
|
||||
def backup_error_key(stat_date: str) -> str:
|
||||
return f"snapshot:backup_error:{stat_date}"
|
||||
|
||||
def run_if_needed(
|
||||
self, *, date: str | None = None, force: bool = False
|
||||
) -> SnapshotSummary:
|
||||
@@ -117,7 +135,14 @@ class SnapshotService:
|
||||
stat_date = date or shanghai_date(now)
|
||||
if not is_date(stat_date):
|
||||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||||
if not force and self.repo.kv_get(self.completed_key(stat_date)) == "1":
|
||||
if force:
|
||||
return self.run(date=stat_date, force=True)
|
||||
|
||||
completed = self.repo.kv_get(self.completed_key(stat_date)) == "1"
|
||||
pending = self.repo.pending_snapshot_aids(stat_date)
|
||||
if completed and not pending:
|
||||
if self.repo.kv_get(self.backup_key(stat_date)) != "1":
|
||||
return self.backup_only(stat_date)
|
||||
self._log("info", "snapshot.skip", stat_date=stat_date, reason="already_completed")
|
||||
return SnapshotSummary(
|
||||
stat_date=stat_date,
|
||||
@@ -126,10 +151,27 @@ class SnapshotService:
|
||||
completed=True,
|
||||
already_completed=True,
|
||||
)
|
||||
return self.run(date=stat_date, force=force)
|
||||
return self.run(date=stat_date, force=False)
|
||||
|
||||
def backup_only(self, stat_date: str) -> SnapshotSummary:
|
||||
"""采集已完成但备份缺失时,只重试备份,不请求任何外部接口。"""
|
||||
now = self._now()
|
||||
cfg = self.config_store.get()
|
||||
summary = SnapshotSummary(
|
||||
stat_date=stat_date,
|
||||
started_at=now,
|
||||
completed=True,
|
||||
already_completed=True,
|
||||
)
|
||||
self._log("info", "snapshot.backup_retry", stat_date=stat_date)
|
||||
self._run_backup(cfg, stat_date, summary)
|
||||
summary.finished_at = self._now()
|
||||
self._log("info", "snapshot.done", **summary.as_dict())
|
||||
return summary
|
||||
|
||||
def run(self, *, date: str | None = None, force: bool = False) -> SnapshotSummary:
|
||||
cfg = self.config_store.get()
|
||||
client = self._client()
|
||||
now = self._now()
|
||||
stat_date = date or shanghai_date(now)
|
||||
if not is_date(stat_date):
|
||||
@@ -137,8 +179,12 @@ class SnapshotService:
|
||||
|
||||
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
|
||||
self._log("info", "snapshot.start", stat_date=stat_date, force=force)
|
||||
# 先落“未完成”,保证强制重采或中途中断后仍可续跑。
|
||||
self.repo.kv_set(self.completed_key(stat_date), "0")
|
||||
if self.repo.kv_get("snapshot:last_completed") == stat_date:
|
||||
self.repo.kv_set("snapshot:last_completed", "")
|
||||
|
||||
summary.auth_ok, summary.auth_message = self._check_auth(cfg, now)
|
||||
summary.auth_ok, summary.auth_message = self._check_auth(client, cfg, now)
|
||||
|
||||
videos = self.repo.video_aids_for_snapshot()
|
||||
summary.total = len(videos)
|
||||
@@ -148,7 +194,7 @@ class SnapshotService:
|
||||
summary.skipped += 1
|
||||
continue
|
||||
|
||||
stat, error, unavailable = self._fetch_view(aid)
|
||||
stat, error, unavailable = self._fetch_view(client, aid)
|
||||
if unavailable:
|
||||
summary.removed += 1
|
||||
self.repo.set_video_status(aid, "removed", now=now)
|
||||
@@ -193,7 +239,15 @@ class SnapshotService:
|
||||
)
|
||||
|
||||
for mid in cfg.track_fans:
|
||||
stat, error = self._fetch_fan(mid)
|
||||
existing_fan = self.repo.get_up_stat(stat_date, mid)
|
||||
if (
|
||||
not force
|
||||
and existing_fan is not None
|
||||
and int(existing_fan.get("ok") or 0) == 1
|
||||
):
|
||||
summary.fans_skipped += 1
|
||||
continue
|
||||
stat, error = self._fetch_fan(client, mid)
|
||||
if stat is not None:
|
||||
self.repo.upsert_up_stat(
|
||||
stat_date,
|
||||
@@ -228,6 +282,21 @@ class SnapshotService:
|
||||
elif self.repo.kv_get("snapshot:last_completed") == stat_date:
|
||||
self.repo.kv_set("snapshot:last_completed", "")
|
||||
|
||||
self._run_backup(cfg, stat_date, summary)
|
||||
|
||||
summary.finished_at = self._now()
|
||||
self._log(
|
||||
"info" if summary.completed else "warning",
|
||||
"snapshot.done",
|
||||
**summary.as_dict(),
|
||||
)
|
||||
return summary
|
||||
|
||||
# ------------------------------------------------------------ internals
|
||||
|
||||
def _run_backup(
|
||||
self, cfg: AppConfig, stat_date: str, summary: SnapshotSummary
|
||||
) -> None:
|
||||
try:
|
||||
path = backup_daily(
|
||||
self.repo.db,
|
||||
@@ -240,28 +309,24 @@ class SnapshotService:
|
||||
keep_days=cfg.storage.backup_keep_days,
|
||||
)
|
||||
summary.backup_path = str(path)
|
||||
self.repo.kv_set(self.backup_key(stat_date), "1")
|
||||
self.repo.kv_set(self.backup_error_key(stat_date), "")
|
||||
except Exception as exc: # noqa: BLE001 备份失败不推翻已采集数据
|
||||
summary.backup_error = str(exc)
|
||||
self.repo.kv_set(self.backup_key(stat_date), "0")
|
||||
self.repo.kv_set(self.backup_error_key(stat_date), str(exc))
|
||||
self._log("error", "backup.failed", error=str(exc), stat_date=stat_date)
|
||||
|
||||
summary.finished_at = self._now()
|
||||
self._log(
|
||||
"info" if summary.completed else "warning",
|
||||
"snapshot.done",
|
||||
**summary.as_dict(),
|
||||
)
|
||||
return summary
|
||||
|
||||
# ------------------------------------------------------------ internals
|
||||
|
||||
def _check_auth(self, cfg: AppConfig, now: int) -> tuple[bool, str]:
|
||||
def _check_auth(
|
||||
self, client, cfg: AppConfig, now: int
|
||||
) -> tuple[bool, str]:
|
||||
if not cfg.bilibili.read.cookie:
|
||||
self.repo.kv_set("auth:last_check", str(now))
|
||||
self.repo.kv_set("auth:ok", "0")
|
||||
self.repo.kv_set("auth:message", "未配置 cookie")
|
||||
return False, "未配置 cookie"
|
||||
try:
|
||||
state = self.client.check_login()
|
||||
state = client.check_login()
|
||||
except BilibiliError as exc:
|
||||
message = str(exc)
|
||||
self.repo.kv_set("auth:last_check", str(now))
|
||||
@@ -282,11 +347,11 @@ class SnapshotService:
|
||||
)
|
||||
return state.is_login, state.message
|
||||
|
||||
def _fetch_view(self, aid: int):
|
||||
def _fetch_view(self, client, aid: int):
|
||||
last_error = ""
|
||||
for attempt in range(1, self.retries + 1):
|
||||
try:
|
||||
return self.client.view(aid), None, False
|
||||
return client.view(aid), None, False
|
||||
except BilibiliAPIError as exc:
|
||||
last_error = str(exc)
|
||||
if exc.code in VIDEO_UNAVAILABLE_CODES:
|
||||
@@ -301,11 +366,11 @@ class SnapshotService:
|
||||
self._sleep(self.backoff * (2 ** (attempt - 1)))
|
||||
return None, last_error or "未知错误", False
|
||||
|
||||
def _fetch_fan(self, mid: int):
|
||||
def _fetch_fan(self, client, mid: int):
|
||||
last_error = ""
|
||||
for attempt in range(1, self.retries + 1):
|
||||
try:
|
||||
return self.client.relation_stat(mid), None
|
||||
return client.relation_stat(mid), None
|
||||
except BilibiliError as exc:
|
||||
last_error = str(exc)
|
||||
if attempt < self.retries:
|
||||
|
||||
Reference in new issue
Block a user