bot-ci
8a9cb21408
Move daily snapshot/fans to 23:15 and add 00:01 seed that clones last_completed ok=1 rows into today without advancing completed. Evening runs force through seed markers; growth previous is per-aid adjacent. Home shows short-update clock + 全站数据 date, and 收录视频 counts.
684 lines
26 KiB
Python
684 lines
26 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""每日视频七项快照与粉丝数采集。
|
||
|
||
规则（DEVELOPMENT.md §7.4）：
|
||
|
||
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``；
|
||
* 成功写入七项指标；``ok=1`` 的同日快照直接跳过（断点续跑；含小时任务 1 小时内刚刷新的）；
|
||
* 午夜 ``seed_at`` 用上一完成日 ok=1 行克隆出当日壳（``snapshot:seeded`` / ``fans:seeded``），不推进完成日；
|
||
* 晚间 ``snapshot_at`` 全量必须 force 打穿 seed，成功后才写完成日并清除 seeded；
|
||
* ``run_recent_videos`` 只刷新近 N 天新片的当日行，不改完成日 / 备份；
|
||
* 视频不可访问时标记 ``removed``；
|
||
* 其他错误最多重试 3 次，仍失败写 ``ok=0`` 占位，不写假零值；
|
||
* 对 ``track_fans`` 逐个请求 ``relation/stat``（独立任务，不影响视频完成日）；
|
||
* 视频采集全部成功后写入当日完成标记并生成日备份。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from typing import Any
|
||
|
||
from ..bilibili.client import (
|
||
VIDEO_UNAVAILABLE_CODES,
|
||
BilibiliAPIError,
|
||
BilibiliError,
|
||
BilibiliHTTPError,
|
||
)
|
||
from ..config import AppConfig, ConfigStore
|
||
from ..repository import Repository
|
||
from ..timeutil import is_date, shanghai_date
|
||
from .maintenance import backup_daily, prune_daily_backups
|
||
|
||
|
||
@dataclass
|
||
class SnapshotSummary:
|
||
stat_date: str
|
||
started_at: int
|
||
total: int = 0
|
||
success: int = 0
|
||
failed: int = 0
|
||
skipped: int = 0
|
||
removed: int = 0
|
||
fans_ok: int = 0
|
||
fans_failed: int = 0
|
||
fans_skipped: int = 0
|
||
auth_ok: bool | None = None
|
||
auth_message: str = ""
|
||
backup_path: str | None = None
|
||
backup_error: str | None = None
|
||
completed: bool = False
|
||
already_completed: bool = False
|
||
finished_at: int = 0
|
||
errors: list[str] = field(default_factory=list)
|
||
|
||
def as_dict(self) -> dict[str, Any]:
|
||
return {
|
||
"stat_date": self.stat_date,
|
||
"started_at": self.started_at,
|
||
"finished_at": self.finished_at,
|
||
"elapsed": max(0, self.finished_at - self.started_at),
|
||
"total": self.total,
|
||
"success": self.success,
|
||
"failed": self.failed,
|
||
"skipped": self.skipped,
|
||
"removed": self.removed,
|
||
"fans_ok": self.fans_ok,
|
||
"fans_failed": self.fans_failed,
|
||
"fans_skipped": self.fans_skipped,
|
||
"auth_ok": self.auth_ok,
|
||
"auth_message": self.auth_message,
|
||
"backup_path": self.backup_path,
|
||
"backup_error": self.backup_error,
|
||
"completed": self.completed,
|
||
"already_completed": self.already_completed,
|
||
"errors": self.errors,
|
||
}
|
||
|
||
|
||
class SnapshotService:
|
||
def __init__(
|
||
self,
|
||
config_store: ConfigStore,
|
||
repository: Repository,
|
||
client=None,
|
||
*,
|
||
client_provider=None,
|
||
clock=time.time,
|
||
sleeper=time.sleep,
|
||
logger=None,
|
||
retries: int = 3,
|
||
backoff: float = 5.0,
|
||
progress_every: int = 50,
|
||
) -> None:
|
||
if retries < 1:
|
||
raise ValueError("retries 必须 >= 1")
|
||
if client is None and client_provider is None:
|
||
raise ValueError("必须提供 client 或 client_provider")
|
||
self.config_store = config_store
|
||
self.repo = repository
|
||
self.client = client
|
||
self.client_provider = client_provider
|
||
self._clock = clock
|
||
self._sleep = sleeper
|
||
self.logger = logger
|
||
self.retries = retries
|
||
self.backoff = backoff
|
||
self.progress_every = progress_every
|
||
|
||
def _now(self) -> int:
|
||
return int(self._clock())
|
||
|
||
def _client(self):
|
||
if self.client_provider is not None:
|
||
return self.client_provider.get()
|
||
return self.client
|
||
|
||
def _log(self, level: str, event: str, **fields) -> None:
|
||
if self.logger is not None:
|
||
getattr(self.logger, level)(event, **fields)
|
||
|
||
@staticmethod
|
||
def completed_key(stat_date: str) -> str:
|
||
return f"snapshot:completed:{stat_date}"
|
||
|
||
@staticmethod
|
||
def backup_key(stat_date: str) -> str:
|
||
return f"snapshot:backup:{stat_date}"
|
||
|
||
@staticmethod
|
||
def backup_error_key(stat_date: str) -> str:
|
||
return f"snapshot:backup_error:{stat_date}"
|
||
|
||
@staticmethod
|
||
def seeded_key(stat_date: str) -> str:
|
||
return f"snapshot:seeded:{stat_date}"
|
||
|
||
@staticmethod
|
||
def fans_seeded_key(stat_date: str) -> str:
|
||
return f"fans:seeded:{stat_date}"
|
||
|
||
def run_if_needed(
|
||
self, *, date: str | None = None, force: bool = False
|
||
) -> SnapshotSummary:
|
||
"""视频快照（含 UP 头像补齐）；粉丝采集请用 ``run_fans_if_needed``。
|
||
|
||
当日若仅有午夜 seed 壳且未 ``force``，直接返回等待晚间落盘，
|
||
避免把「全员 skip」误标成 completed。
|
||
"""
|
||
now = self._now()
|
||
stat_date = date or shanghai_date(now)
|
||
if not is_date(stat_date):
|
||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||
seeded = self.repo.kv_get(self.seeded_key(stat_date)) == "1"
|
||
if force:
|
||
return self.run_videos(date=stat_date, force=True)
|
||
|
||
completed = self.repo.kv_get(self.completed_key(stat_date)) == "1"
|
||
if seeded and not completed:
|
||
self._log(
|
||
"info", "snapshot.skip",
|
||
stat_date=stat_date, reason="seeded_pending_seal",
|
||
)
|
||
return SnapshotSummary(
|
||
stat_date=stat_date,
|
||
started_at=now,
|
||
finished_at=now,
|
||
completed=False,
|
||
already_completed=False,
|
||
)
|
||
pending = self.repo.pending_snapshot_aids(stat_date)
|
||
if completed and not pending:
|
||
if self.repo.kv_get(self.backup_key(stat_date)) != "1":
|
||
return self.backup_only(stat_date)
|
||
self._log("info", "snapshot.skip", stat_date=stat_date, reason="already_completed")
|
||
return SnapshotSummary(
|
||
stat_date=stat_date,
|
||
started_at=now,
|
||
finished_at=now,
|
||
completed=True,
|
||
already_completed=True,
|
||
)
|
||
return self.run_videos(date=stat_date, force=False)
|
||
|
||
def backup_only(self, stat_date: str) -> SnapshotSummary:
|
||
"""采集已完成但备份缺失时，只重试备份，不请求任何外部接口。"""
|
||
now = self._now()
|
||
cfg = self.config_store.get()
|
||
summary = SnapshotSummary(
|
||
stat_date=stat_date,
|
||
started_at=now,
|
||
completed=True,
|
||
already_completed=True,
|
||
)
|
||
self._log("info", "snapshot.backup_retry", stat_date=stat_date)
|
||
self._run_backup(cfg, stat_date, summary)
|
||
summary.finished_at = self._now()
|
||
self._log("info", "snapshot.done", **summary.as_dict())
|
||
return summary
|
||
|
||
def run(self, *, date: str | None = None, force: bool = False) -> SnapshotSummary:
|
||
"""兼容 CLI：先跑视频快照，再跑粉丝采集，合并进同一摘要。"""
|
||
summary = self.run_videos(date=date, force=force)
|
||
return self.run_fans(date=summary.stat_date, force=force, summary=summary)
|
||
|
||
def run_videos(self, *, date: str | None = None, force: bool = False) -> SnapshotSummary:
|
||
"""采集视频七项指标并在全部成功后标记完成日 / 备份。"""
|
||
cfg = self.config_store.get()
|
||
client = self._client()
|
||
now = self._now()
|
||
stat_date = date or shanghai_date(now)
|
||
if not is_date(stat_date):
|
||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||
|
||
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
|
||
self._log("info", "snapshot.start", stat_date=stat_date, force=force)
|
||
# 先落“未完成”，保证强制重采或中途中断后仍可续跑。
|
||
self.repo.kv_set(self.completed_key(stat_date), "0")
|
||
if self.repo.kv_get("snapshot:last_completed") == stat_date:
|
||
self.repo.kv_set("snapshot:last_completed", "")
|
||
|
||
summary.auth_ok, summary.auth_message = self._check_auth(client, cfg, now)
|
||
|
||
videos = self.repo.video_aids_for_snapshot()
|
||
summary.total = len(videos)
|
||
for index, aid in enumerate(videos, 1):
|
||
existing = self.repo.get_snapshot(stat_date, aid)
|
||
if existing is not None and int(existing.get("ok") or 0) == 1 and not force:
|
||
# 断点续跑：同日已成功则跳过。小时任务刷新过的行也是 ok=1，
|
||
# 自然覆盖「1 小时内 captured_at 已更新则不再次更新」的要求。
|
||
summary.skipped += 1
|
||
continue
|
||
|
||
stat, error, unavailable = self._fetch_view(client, aid)
|
||
if unavailable:
|
||
summary.removed += 1
|
||
self.repo.set_video_status(aid, "removed", now=now)
|
||
self._log(
|
||
"warning", "snapshot.removed",
|
||
stat_date=stat_date, aid=aid, error=error,
|
||
)
|
||
elif stat is not None:
|
||
self.repo.upsert_snapshot(
|
||
stat_date,
|
||
aid,
|
||
metrics=stat.metrics(),
|
||
ok=True,
|
||
err=None,
|
||
captured_at=self._now(),
|
||
)
|
||
if getattr(stat, "up_face", None):
|
||
self.repo.apply_video_up_face(aid, stat.up_face)
|
||
summary.success += 1
|
||
else:
|
||
summary.failed += 1
|
||
self.repo.upsert_snapshot(
|
||
stat_date,
|
||
aid,
|
||
metrics=None,
|
||
ok=False,
|
||
err=error,
|
||
captured_at=self._now(),
|
||
)
|
||
summary.errors.append(f"aid {aid}: {error}")
|
||
self._log(
|
||
"error", "snapshot.failed",
|
||
stat_date=stat_date, aid=aid, error=error,
|
||
)
|
||
|
||
if self.progress_every > 0 and (
|
||
index % self.progress_every == 0 or index == summary.total
|
||
):
|
||
self._log(
|
||
"info", "snapshot.progress",
|
||
stat_date=stat_date, done=index, total=summary.total,
|
||
success=summary.success, failed=summary.failed,
|
||
skipped=summary.skipped, removed=summary.removed,
|
||
)
|
||
|
||
# 完成日只看视频；粉丝是独立调度任务。
|
||
summary.completed = summary.failed == 0
|
||
self.repo.kv_set(self.completed_key(stat_date), "1" if summary.completed else "0")
|
||
self.repo.kv_set("snapshot:last_attempt", stat_date)
|
||
if summary.completed:
|
||
self.repo.kv_set("snapshot:last_completed", stat_date)
|
||
# 真落盘后清掉午夜 seed 标记。
|
||
self.repo.kv_delete(self.seeded_key(stat_date))
|
||
elif self.repo.kv_get("snapshot:last_completed") == stat_date:
|
||
self.repo.kv_set("snapshot:last_completed", "")
|
||
|
||
self._run_backup(cfg, stat_date, summary)
|
||
|
||
if summary.success or summary.removed or summary.failed:
|
||
self.repo.bump_cache_version("videos")
|
||
|
||
summary.finished_at = self._now()
|
||
self._log(
|
||
"info" if summary.completed else "warning",
|
||
"snapshot.done",
|
||
**summary.as_dict(),
|
||
)
|
||
return summary
|
||
|
||
def run_recent_videos(
|
||
self,
|
||
*,
|
||
date: str | None = None,
|
||
recent_days: int | None = None,
|
||
) -> SnapshotSummary:
|
||
"""刷新近 N 天新视频的当日快照；不碰完成日标记与日备份。
|
||
|
||
「近 N 天」按 pubdate 墙钟窗口：``now - recent_video_days * 86400``，
|
||
不是上海日历日差。允许覆盖同日已有 ``ok=1`` 行以便小时级刷新。
|
||
"""
|
||
cfg = self.config_store.get()
|
||
client = self._client()
|
||
now = self._now()
|
||
stat_date = date or shanghai_date(now)
|
||
if not is_date(stat_date):
|
||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||
days = (
|
||
int(cfg.schedule.recent_video_days)
|
||
if recent_days is None else int(recent_days)
|
||
)
|
||
if days < 1:
|
||
raise ValueError("recent_days 必须 >= 1")
|
||
# 墙钟窗口，而非按上海日历截断到日界。
|
||
since_pubdate = now - days * 86400
|
||
|
||
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
|
||
self._log(
|
||
"info", "snapshot.recent_start",
|
||
stat_date=stat_date, recent_days=days, since_pubdate=since_pubdate,
|
||
)
|
||
videos = self.repo.video_aids_recent_for_snapshot(since_pubdate=since_pubdate)
|
||
summary.total = len(videos)
|
||
for index, aid in enumerate(videos, 1):
|
||
stat, error, unavailable = self._fetch_view(client, aid)
|
||
if unavailable:
|
||
summary.removed += 1
|
||
self.repo.set_video_status(aid, "removed", now=now)
|
||
self._log(
|
||
"warning", "snapshot.recent_removed",
|
||
stat_date=stat_date, aid=aid, error=error,
|
||
)
|
||
elif stat is not None:
|
||
self.repo.upsert_snapshot(
|
||
stat_date,
|
||
aid,
|
||
metrics=stat.metrics(),
|
||
ok=True,
|
||
err=None,
|
||
captured_at=self._now(),
|
||
)
|
||
if getattr(stat, "up_face", None):
|
||
self.repo.apply_video_up_face(aid, stat.up_face)
|
||
summary.success += 1
|
||
else:
|
||
summary.failed += 1
|
||
self.repo.upsert_snapshot(
|
||
stat_date,
|
||
aid,
|
||
metrics=None,
|
||
ok=False,
|
||
err=error,
|
||
captured_at=self._now(),
|
||
)
|
||
summary.errors.append(f"aid {aid}: {error}")
|
||
self._log(
|
||
"error", "snapshot.recent_failed",
|
||
stat_date=stat_date, aid=aid, error=error,
|
||
)
|
||
|
||
if self.progress_every > 0 and (
|
||
index % self.progress_every == 0 or index == summary.total
|
||
):
|
||
self._log(
|
||
"info", "snapshot.recent_progress",
|
||
stat_date=stat_date, done=index, total=summary.total,
|
||
success=summary.success, failed=summary.failed,
|
||
removed=summary.removed,
|
||
)
|
||
|
||
# 刻意不写 snapshot:completed / last_completed / 日备份：完成日只归全量任务。
|
||
summary.completed = summary.failed == 0
|
||
if summary.success or summary.removed or summary.failed:
|
||
self.repo.bump_cache_version("videos")
|
||
summary.finished_at = self._now()
|
||
self._log(
|
||
"info" if summary.completed else "warning",
|
||
"snapshot.recent_done",
|
||
**summary.as_dict(),
|
||
)
|
||
return summary
|
||
|
||
def run_fans_if_needed(
|
||
self, *, date: str | None = None, force: bool = False
|
||
) -> SnapshotSummary:
|
||
now = self._now()
|
||
stat_date = date or shanghai_date(now)
|
||
if not is_date(stat_date):
|
||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||
cfg = self.config_store.get()
|
||
if not cfg.track_fans:
|
||
summary = SnapshotSummary(
|
||
stat_date=stat_date, started_at=now, finished_at=now, completed=True,
|
||
)
|
||
self._log("info", "fans.skip", stat_date=stat_date, reason="no_track_fans")
|
||
return summary
|
||
seeded = self.repo.kv_get(self.fans_seeded_key(stat_date)) == "1"
|
||
if force or seeded:
|
||
# 晚间/显式 force，或存在 seed 壳时必须重采（seeded 时由调度传入 force）。
|
||
if seeded and not force:
|
||
self._log(
|
||
"info", "fans.skip",
|
||
stat_date=stat_date, reason="seeded_pending_seal",
|
||
)
|
||
return SnapshotSummary(
|
||
stat_date=stat_date,
|
||
started_at=now,
|
||
finished_at=now,
|
||
completed=False,
|
||
already_completed=False,
|
||
fans_skipped=len(cfg.track_fans),
|
||
)
|
||
return self.run_fans(date=stat_date, force=True)
|
||
if not self._pending_fan_mids(stat_date, cfg.track_fans):
|
||
self._log("info", "fans.skip", stat_date=stat_date, reason="already_completed")
|
||
return SnapshotSummary(
|
||
stat_date=stat_date,
|
||
started_at=now,
|
||
finished_at=now,
|
||
completed=True,
|
||
already_completed=True,
|
||
fans_skipped=len(cfg.track_fans),
|
||
)
|
||
return self.run_fans(date=stat_date, force=force)
|
||
|
||
def run_fans(
|
||
self,
|
||
*,
|
||
date: str | None = None,
|
||
force: bool = False,
|
||
summary: SnapshotSummary | None = None,
|
||
) -> SnapshotSummary:
|
||
"""采集 track_fans 粉丝数；成败不影响视频完成日。"""
|
||
cfg = self.config_store.get()
|
||
client = self._client()
|
||
now = self._now()
|
||
stat_date = date or shanghai_date(now)
|
||
if not is_date(stat_date):
|
||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||
|
||
standalone = summary is None
|
||
if summary is None:
|
||
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
|
||
else:
|
||
# CLI 合并跑：保留视频摘要，只追加粉丝段耗时终点
|
||
summary.stat_date = stat_date
|
||
|
||
self._log("info", "fans.start", stat_date=stat_date, force=force, total=len(cfg.track_fans))
|
||
for mid in cfg.track_fans:
|
||
existing_fan = self.repo.get_up_stat(stat_date, mid)
|
||
if (
|
||
not force
|
||
and existing_fan is not None
|
||
and int(existing_fan.get("ok") or 0) == 1
|
||
):
|
||
summary.fans_skipped += 1
|
||
continue
|
||
stat, error = self._fetch_fan(client, mid)
|
||
if stat is not None:
|
||
self.repo.upsert_up_stat(
|
||
stat_date,
|
||
mid,
|
||
follower=stat.follower,
|
||
following=stat.following,
|
||
ok=True,
|
||
captured_at=self._now(),
|
||
)
|
||
summary.fans_ok += 1
|
||
else:
|
||
summary.fans_failed += 1
|
||
self.repo.upsert_up_stat(
|
||
stat_date,
|
||
mid,
|
||
follower=None,
|
||
following=None,
|
||
ok=False,
|
||
captured_at=self._now(),
|
||
)
|
||
summary.errors.append(f"fans mid {mid}: {error}")
|
||
self._log(
|
||
"error", "snapshot.fan_failed",
|
||
stat_date=stat_date, mid=mid, error=error,
|
||
)
|
||
|
||
if summary.fans_ok or summary.fans_failed:
|
||
self.repo.bump_cache_version("fans")
|
||
|
||
fans_ok = summary.fans_failed == 0
|
||
# 独立粉丝任务用 completed 表示粉丝段成败；合并进视频摘要时不改动视频完成标记。
|
||
if standalone:
|
||
summary.completed = fans_ok
|
||
if fans_ok:
|
||
self.repo.kv_delete(self.fans_seeded_key(stat_date))
|
||
elif fans_ok:
|
||
# CLI 合并跑：粉丝段成功也清 seed。
|
||
self.repo.kv_delete(self.fans_seeded_key(stat_date))
|
||
summary.finished_at = self._now()
|
||
self._log(
|
||
"info" if fans_ok else "warning",
|
||
"fans.done",
|
||
stat_date=stat_date,
|
||
fans_ok=summary.fans_ok,
|
||
fans_failed=summary.fans_failed,
|
||
fans_skipped=summary.fans_skipped,
|
||
completed=summary.completed,
|
||
)
|
||
return summary
|
||
|
||
|
||
def run_seed_from_last_completed(
|
||
self, *, date: str | None = None
|
||
) -> SnapshotSummary:
|
||
"""用上一完成日的 ok=1 行克隆出当日壳（视频 + 粉丝）。
|
||
|
||
不写 ``snapshot:completed`` / ``snapshot:last_completed``；
|
||
仅打 ``snapshot:seeded`` / ``fans:seeded``，供白天展示与晚间 force 重采。
|
||
"""
|
||
cfg = self.config_store.get()
|
||
now = self._now()
|
||
stat_date = date or shanghai_date(now)
|
||
if not is_date(stat_date):
|
||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||
|
||
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
|
||
source = self.repo.kv_get("snapshot:last_completed")
|
||
if not source or source >= stat_date:
|
||
self._log(
|
||
"info", "snapshot.seed_skip",
|
||
stat_date=stat_date, reason="no_source", source=source,
|
||
)
|
||
summary.finished_at = self._now()
|
||
summary.completed = True
|
||
return summary
|
||
if self.repo.kv_get(self.completed_key(stat_date)) == "1":
|
||
self._log(
|
||
"info", "snapshot.seed_skip",
|
||
stat_date=stat_date, reason="already_completed",
|
||
)
|
||
summary.finished_at = self._now()
|
||
summary.completed = True
|
||
summary.already_completed = True
|
||
return summary
|
||
if self.repo.kv_get(self.seeded_key(stat_date)) == "1":
|
||
self._log(
|
||
"info", "snapshot.seed_skip",
|
||
stat_date=stat_date, reason="already_seeded", source=source,
|
||
)
|
||
summary.finished_at = self._now()
|
||
summary.completed = True
|
||
summary.already_completed = True
|
||
return summary
|
||
|
||
copied = self.repo.copy_ok_snapshots(
|
||
source, stat_date, captured_at=now, src="seed",
|
||
)
|
||
fans_copied = 0
|
||
if cfg.track_fans:
|
||
fans_copied = self.repo.copy_ok_up_stats(
|
||
source, stat_date, captured_at=now, mids=cfg.track_fans,
|
||
)
|
||
self.repo.kv_set(self.fans_seeded_key(stat_date), "1")
|
||
self.repo.kv_set(self.seeded_key(stat_date), "1")
|
||
summary.total = copied
|
||
summary.success = copied
|
||
summary.fans_ok = fans_copied
|
||
if copied or fans_copied:
|
||
self.repo.bump_cache_version("videos")
|
||
if fans_copied:
|
||
self.repo.bump_cache_version("fans")
|
||
summary.finished_at = self._now()
|
||
summary.completed = True # seed 任务本身成功；不等于快照落盘完成
|
||
self._log(
|
||
"info", "snapshot.seed_done",
|
||
stat_date=stat_date, source=source,
|
||
copied=copied, fans_copied=fans_copied,
|
||
)
|
||
return summary
|
||
|
||
def _pending_fan_mids(self, stat_date: str, mids: tuple[int, ...] | list[int]) -> list[int]:
|
||
pending: list[int] = []
|
||
for mid in mids:
|
||
existing = self.repo.get_up_stat(stat_date, int(mid))
|
||
if existing is None or int(existing.get("ok") or 0) != 1:
|
||
pending.append(int(mid))
|
||
return pending
|
||
|
||
# ------------------------------------------------------------ internals
|
||
|
||
def _run_backup(
|
||
self, cfg: AppConfig, stat_date: str, summary: SnapshotSummary
|
||
) -> None:
|
||
try:
|
||
path = backup_daily(
|
||
self.repo.db,
|
||
cfg.storage.backup_dir,
|
||
stat_date,
|
||
logger=self.logger,
|
||
)
|
||
prune_daily_backups(
|
||
cfg.storage.backup_dir,
|
||
keep_days=cfg.storage.backup_keep_days,
|
||
)
|
||
summary.backup_path = str(path)
|
||
self.repo.kv_set(self.backup_key(stat_date), "1")
|
||
self.repo.kv_set(self.backup_error_key(stat_date), "")
|
||
except Exception as exc: # noqa: BLE001 备份失败不推翻已采集数据
|
||
summary.backup_error = str(exc)
|
||
self.repo.kv_set(self.backup_key(stat_date), "0")
|
||
self.repo.kv_set(self.backup_error_key(stat_date), str(exc))
|
||
self._log("error", "backup.failed", error=str(exc), stat_date=stat_date)
|
||
|
||
def _check_auth(
|
||
self, client, cfg: AppConfig, now: int
|
||
) -> tuple[bool, str]:
|
||
if not cfg.bilibili.read.cookie:
|
||
self.repo.kv_set("auth:last_check", str(now))
|
||
self.repo.kv_set("auth:ok", "0")
|
||
self.repo.kv_set("auth:message", "未配置 cookie")
|
||
return False, "未配置 cookie"
|
||
try:
|
||
state = client.check_login()
|
||
except BilibiliError as exc:
|
||
message = str(exc)
|
||
self.repo.kv_set("auth:last_check", str(now))
|
||
self.repo.kv_set("auth:ok", "0")
|
||
self.repo.kv_set("auth:message", message)
|
||
self._log("warning", "auth.invalid", message=message)
|
||
return False, message
|
||
|
||
self.repo.kv_set("auth:last_check", str(now))
|
||
self.repo.kv_set("auth:ok", "1" if state.is_login else "0")
|
||
self.repo.kv_set("auth:message", state.message)
|
||
if state.mid is not None:
|
||
self.repo.kv_set("auth:mid", str(state.mid))
|
||
if not state.is_login:
|
||
self._log(
|
||
"warning", "auth.invalid",
|
||
code=state.code, message=state.message,
|
||
)
|
||
return state.is_login, state.message
|
||
|
||
def _fetch_view(self, client, aid: int):
|
||
last_error = ""
|
||
for attempt in range(1, self.retries + 1):
|
||
try:
|
||
return client.view(aid), None, False
|
||
except BilibiliAPIError as exc:
|
||
last_error = str(exc)
|
||
if exc.code in VIDEO_UNAVAILABLE_CODES:
|
||
return None, last_error, True
|
||
except BilibiliHTTPError as exc:
|
||
last_error = str(exc)
|
||
if exc.status == 404:
|
||
return None, last_error, True
|
||
except BilibiliError as exc:
|
||
last_error = str(exc)
|
||
if attempt < self.retries:
|
||
self._sleep(self.backoff * (2 ** (attempt - 1)))
|
||
return None, last_error or "未知错误", False
|
||
|
||
def _fetch_fan(self, client, mid: int):
|
||
last_error = ""
|
||
for attempt in range(1, self.retries + 1):
|
||
try:
|
||
return client.relation_stat(mid), None
|
||
except BilibiliError as exc:
|
||
last_error = str(exc)
|
||
if attempt < self.retries:
|
||
self._sleep(self.backoff * (2 ** (attempt - 1)))
|
||
return None, last_error or "未知错误"
|