feat(p1): add daily snapshots and fan tracking
Implement P1: seven-metric view snapshots, relation/stat fan snapshots, failure placeholders, resume by daily completion markers, Cookie login warnings, daily SQLite backups, Shanghai daily scheduling, and CLI snapshot/auth-check commands. Real read-only run captured 795/795 videos successfully over 1124s with a verified daily backup.
This commit is contained in:
kairee committed
1 parent
ee289bfa56
commit
5c2ab7b187
20 files changed
+1680
-54
No files matched your search
@@ -0,0 +1,50 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""备份与保留期清理。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from ..database import Database
|
||||
|
||||
|
||||
def daily_backup_path(backup_dir: str | Path, stat_date: str) -> Path:
|
||||
compact = stat_date.replace("-", "")
|
||||
return Path(backup_dir) / f"app-{compact}.db"
|
||||
|
||||
|
||||
def backup_daily(
|
||||
db: Database,
|
||||
backup_dir: str | Path,
|
||||
stat_date: str,
|
||||
*,
|
||||
logger=None,
|
||||
) -> Path:
|
||||
target = daily_backup_path(backup_dir, stat_date)
|
||||
db.backup_to(target)
|
||||
if logger is not None:
|
||||
logger.info("backup.daily", path=str(target), stat_date=stat_date)
|
||||
return target
|
||||
|
||||
|
||||
def prune_daily_backups(
|
||||
backup_dir: str | Path,
|
||||
*,
|
||||
keep_days: int,
|
||||
now: float | None = None,
|
||||
) -> list[Path]:
|
||||
"""只清理 ``app-YYYYMMDD.db`` 日备份，不动迁移备份。"""
|
||||
target_dir = Path(backup_dir)
|
||||
if not target_dir.is_dir():
|
||||
return []
|
||||
cutoff = (time.time() if now is None else now) - keep_days * 86400
|
||||
removed: list[Path] = []
|
||||
for path in target_dir.glob("app-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9].db"):
|
||||
try:
|
||||
if path.stat().st_mtime < cutoff:
|
||||
path.unlink()
|
||||
removed.append(path)
|
||||
except OSError:
|
||||
continue
|
||||
return removed
|
||||
@@ -0,0 +1,313 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""每日视频七项快照与粉丝数采集。
|
||||
|
||||
规则（DEVELOPMENT.md §7.4）：
|
||||
|
||||
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``；
|
||||
* 成功写入七项指标；``ok=1`` 的同日快照直接跳过（断点续跑）；
|
||||
* 视频不可访问时标记 ``removed``；
|
||||
* 其他错误最多重试 3 次，仍失败写 ``ok=0`` 占位，不写假零值；
|
||||
* 对 ``track_fans`` 逐个请求 ``relation/stat``；
|
||||
* 完成后写入当日完成标记并生成日备份。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from ..bilibili.client import (
|
||||
VIDEO_UNAVAILABLE_CODES,
|
||||
BilibiliAPIError,
|
||||
BilibiliClient,
|
||||
BilibiliError,
|
||||
BilibiliHTTPError,
|
||||
)
|
||||
from ..config import AppConfig, ConfigStore
|
||||
from ..repository import Repository
|
||||
from ..timeutil import is_date, shanghai_date
|
||||
from .maintenance import backup_daily, prune_daily_backups
|
||||
|
||||
|
||||
@dataclass
|
||||
class SnapshotSummary:
|
||||
stat_date: str
|
||||
started_at: int
|
||||
total: int = 0
|
||||
success: int = 0
|
||||
failed: int = 0
|
||||
skipped: int = 0
|
||||
removed: int = 0
|
||||
fans_ok: int = 0
|
||||
fans_failed: int = 0
|
||||
auth_ok: bool | None = None
|
||||
auth_message: str = ""
|
||||
backup_path: str | None = None
|
||||
backup_error: str | None = None
|
||||
completed: bool = False
|
||||
already_completed: bool = False
|
||||
finished_at: int = 0
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"stat_date": self.stat_date,
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"elapsed": max(0, self.finished_at - self.started_at),
|
||||
"total": self.total,
|
||||
"success": self.success,
|
||||
"failed": self.failed,
|
||||
"skipped": self.skipped,
|
||||
"removed": self.removed,
|
||||
"fans_ok": self.fans_ok,
|
||||
"fans_failed": self.fans_failed,
|
||||
"auth_ok": self.auth_ok,
|
||||
"auth_message": self.auth_message,
|
||||
"backup_path": self.backup_path,
|
||||
"backup_error": self.backup_error,
|
||||
"completed": self.completed,
|
||||
"already_completed": self.already_completed,
|
||||
"errors": self.errors,
|
||||
}
|
||||
|
||||
|
||||
class SnapshotService:
|
||||
def __init__(
|
||||
self,
|
||||
config_store: ConfigStore,
|
||||
repository: Repository,
|
||||
client: BilibiliClient,
|
||||
*,
|
||||
clock=time.time,
|
||||
sleeper=time.sleep,
|
||||
logger=None,
|
||||
retries: int = 3,
|
||||
backoff: float = 5.0,
|
||||
progress_every: int = 50,
|
||||
) -> None:
|
||||
if retries < 1:
|
||||
raise ValueError("retries 必须 >= 1")
|
||||
self.config_store = config_store
|
||||
self.repo = repository
|
||||
self.client = client
|
||||
self._clock = clock
|
||||
self._sleep = sleeper
|
||||
self.logger = logger
|
||||
self.retries = retries
|
||||
self.backoff = backoff
|
||||
self.progress_every = progress_every
|
||||
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
|
||||
def _log(self, level: str, event: str, **fields) -> None:
|
||||
if self.logger is not None:
|
||||
getattr(self.logger, level)(event, **fields)
|
||||
|
||||
@staticmethod
|
||||
def completed_key(stat_date: str) -> str:
|
||||
return f"snapshot:completed:{stat_date}"
|
||||
|
||||
def run_if_needed(
|
||||
self, *, date: str | None = None, force: bool = False
|
||||
) -> SnapshotSummary:
|
||||
now = self._now()
|
||||
stat_date = date or shanghai_date(now)
|
||||
if not is_date(stat_date):
|
||||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||||
if not force and self.repo.kv_get(self.completed_key(stat_date)) == "1":
|
||||
self._log("info", "snapshot.skip", stat_date=stat_date, reason="already_completed")
|
||||
return SnapshotSummary(
|
||||
stat_date=stat_date,
|
||||
started_at=now,
|
||||
finished_at=now,
|
||||
completed=True,
|
||||
already_completed=True,
|
||||
)
|
||||
return self.run(date=stat_date, force=force)
|
||||
|
||||
def run(self, *, date: str | None = None, force: bool = False) -> SnapshotSummary:
|
||||
cfg = self.config_store.get()
|
||||
now = self._now()
|
||||
stat_date = date or shanghai_date(now)
|
||||
if not is_date(stat_date):
|
||||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||||
|
||||
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
|
||||
self._log("info", "snapshot.start", stat_date=stat_date, force=force)
|
||||
|
||||
summary.auth_ok, summary.auth_message = self._check_auth(cfg, now)
|
||||
|
||||
videos = self.repo.video_aids_for_snapshot()
|
||||
summary.total = len(videos)
|
||||
for index, aid in enumerate(videos, 1):
|
||||
existing = self.repo.get_snapshot(stat_date, aid)
|
||||
if existing is not None and int(existing.get("ok") or 0) == 1 and not force:
|
||||
summary.skipped += 1
|
||||
continue
|
||||
|
||||
stat, error, unavailable = self._fetch_view(aid)
|
||||
if unavailable:
|
||||
summary.removed += 1
|
||||
self.repo.set_video_status(aid, "removed", now=now)
|
||||
self._log(
|
||||
"warning", "snapshot.removed",
|
||||
stat_date=stat_date, aid=aid, error=error,
|
||||
)
|
||||
elif stat is not None:
|
||||
self.repo.upsert_snapshot(
|
||||
stat_date,
|
||||
aid,
|
||||
metrics=stat.metrics(),
|
||||
ok=True,
|
||||
err=None,
|
||||
captured_at=self._now(),
|
||||
)
|
||||
summary.success += 1
|
||||
else:
|
||||
summary.failed += 1
|
||||
self.repo.upsert_snapshot(
|
||||
stat_date,
|
||||
aid,
|
||||
metrics=None,
|
||||
ok=False,
|
||||
err=error,
|
||||
captured_at=self._now(),
|
||||
)
|
||||
summary.errors.append(f"aid {aid}: {error}")
|
||||
self._log(
|
||||
"error", "snapshot.failed",
|
||||
stat_date=stat_date, aid=aid, error=error,
|
||||
)
|
||||
|
||||
if self.progress_every > 0 and (
|
||||
index % self.progress_every == 0 or index == summary.total
|
||||
):
|
||||
self._log(
|
||||
"info", "snapshot.progress",
|
||||
stat_date=stat_date, done=index, total=summary.total,
|
||||
success=summary.success, failed=summary.failed,
|
||||
skipped=summary.skipped, removed=summary.removed,
|
||||
)
|
||||
|
||||
for mid in cfg.track_fans:
|
||||
stat, error = self._fetch_fan(mid)
|
||||
if stat is not None:
|
||||
self.repo.upsert_up_stat(
|
||||
stat_date,
|
||||
mid,
|
||||
follower=stat.follower,
|
||||
following=stat.following,
|
||||
ok=True,
|
||||
captured_at=self._now(),
|
||||
)
|
||||
summary.fans_ok += 1
|
||||
else:
|
||||
summary.fans_failed += 1
|
||||
self.repo.upsert_up_stat(
|
||||
stat_date,
|
||||
mid,
|
||||
follower=None,
|
||||
following=None,
|
||||
ok=False,
|
||||
captured_at=self._now(),
|
||||
)
|
||||
summary.errors.append(f"fans mid {mid}: {error}")
|
||||
self._log(
|
||||
"error", "snapshot.fan_failed",
|
||||
stat_date=stat_date, mid=mid, error=error,
|
||||
)
|
||||
|
||||
summary.completed = summary.failed == 0 and summary.fans_failed == 0
|
||||
self.repo.kv_set(self.completed_key(stat_date), "1" if summary.completed else "0")
|
||||
self.repo.kv_set("snapshot:last_attempt", stat_date)
|
||||
if summary.completed:
|
||||
self.repo.kv_set("snapshot:last_completed", stat_date)
|
||||
elif self.repo.kv_get("snapshot:last_completed") == stat_date:
|
||||
self.repo.kv_set("snapshot:last_completed", "")
|
||||
|
||||
try:
|
||||
path = backup_daily(
|
||||
self.repo.db,
|
||||
cfg.storage.backup_dir,
|
||||
stat_date,
|
||||
logger=self.logger,
|
||||
)
|
||||
prune_daily_backups(
|
||||
cfg.storage.backup_dir,
|
||||
keep_days=cfg.storage.backup_keep_days,
|
||||
)
|
||||
summary.backup_path = str(path)
|
||||
except Exception as exc: # noqa: BLE001 备份失败不推翻已采集数据
|
||||
summary.backup_error = str(exc)
|
||||
self._log("error", "backup.failed", error=str(exc), stat_date=stat_date)
|
||||
|
||||
summary.finished_at = self._now()
|
||||
self._log(
|
||||
"info" if summary.completed else "warning",
|
||||
"snapshot.done",
|
||||
**summary.as_dict(),
|
||||
)
|
||||
return summary
|
||||
|
||||
# ------------------------------------------------------------ internals
|
||||
|
||||
def _check_auth(self, cfg: AppConfig, now: int) -> tuple[bool, str]:
|
||||
if not cfg.bilibili.cookie:
|
||||
self.repo.kv_set("auth:last_check", str(now))
|
||||
self.repo.kv_set("auth:ok", "0")
|
||||
self.repo.kv_set("auth:message", "未配置 cookie")
|
||||
return False, "未配置 cookie"
|
||||
try:
|
||||
state = self.client.check_login()
|
||||
except BilibiliError as exc:
|
||||
message = str(exc)
|
||||
self.repo.kv_set("auth:last_check", str(now))
|
||||
self.repo.kv_set("auth:ok", "0")
|
||||
self.repo.kv_set("auth:message", message)
|
||||
self._log("warning", "auth.invalid", message=message)
|
||||
return False, message
|
||||
|
||||
self.repo.kv_set("auth:last_check", str(now))
|
||||
self.repo.kv_set("auth:ok", "1" if state.is_login else "0")
|
||||
self.repo.kv_set("auth:message", state.message)
|
||||
if state.mid is not None:
|
||||
self.repo.kv_set("auth:mid", str(state.mid))
|
||||
if not state.is_login:
|
||||
self._log(
|
||||
"warning", "auth.invalid",
|
||||
code=state.code, message=state.message,
|
||||
)
|
||||
return state.is_login, state.message
|
||||
|
||||
def _fetch_view(self, aid: int):
|
||||
last_error = ""
|
||||
for attempt in range(1, self.retries + 1):
|
||||
try:
|
||||
return self.client.view(aid), None, False
|
||||
except BilibiliAPIError as exc:
|
||||
last_error = str(exc)
|
||||
if exc.code in VIDEO_UNAVAILABLE_CODES:
|
||||
return None, last_error, True
|
||||
except BilibiliHTTPError as exc:
|
||||
last_error = str(exc)
|
||||
if exc.status == 404:
|
||||
return None, last_error, True
|
||||
except BilibiliError as exc:
|
||||
last_error = str(exc)
|
||||
if attempt < self.retries:
|
||||
self._sleep(self.backoff * (2 ** (attempt - 1)))
|
||||
return None, last_error or "未知错误", False
|
||||
|
||||
def _fetch_fan(self, mid: int):
|
||||
last_error = ""
|
||||
for attempt in range(1, self.retries + 1):
|
||||
try:
|
||||
return self.client.relation_stat(mid), None
|
||||
except BilibiliError as exc:
|
||||
last_error = str(exc)
|
||||
if attempt < self.retries:
|
||||
self._sleep(self.backoff * (2 ** (attempt - 1)))
|
||||
return None, last_error or "未知错误"
|
||||
Reference in new issue
Block a user