kairee
6a21b5d905
Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.
300 lines
9.5 KiB
Python
300 lines
9.5 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""收藏夹同步（库的真相源）。
|
|
|
|
每次运行：
|
|
|
|
1. 读取每个配置年度夹的全部 aid；
|
|
2. 与本地 ``folder_resource`` 做差集；
|
|
3. 新增资源按 200 条分块拉取 ``infos`` 并入库；
|
|
4. 从收藏夹移除的资源标记 ``unfaved``（不删除历史）；
|
|
5. 每 ``full_folder_refresh_hours`` 小时全量刷新一次元数据并识别失效资源；
|
|
6. 容量接近上限时预警，但绝不自动换夹。
|
|
|
|
黑名单只在发现流程生效，本服务不读取 ``up_blacklist``。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from ..config import AppConfig, ConfigStore, FolderConfig
|
|
from ..repository import Repository
|
|
|
|
CAPACITY_WARN = 950
|
|
|
|
|
|
@dataclass
|
|
class FolderSyncResult:
|
|
media_id: int
|
|
year: int
|
|
title: str
|
|
remote_count: int = 0
|
|
added: int = 0
|
|
removed: int = 0
|
|
refreshed: int = 0
|
|
missing: int = 0
|
|
full_refresh: bool = False
|
|
warning: str | None = None
|
|
error: str | None = None
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"media_id": self.media_id,
|
|
"year": self.year,
|
|
"title": self.title,
|
|
"remote_count": self.remote_count,
|
|
"added": self.added,
|
|
"removed": self.removed,
|
|
"refreshed": self.refreshed,
|
|
"missing": self.missing,
|
|
"full_refresh": self.full_refresh,
|
|
"warning": self.warning,
|
|
"error": self.error,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class SyncSummary:
|
|
started_at: int
|
|
finished_at: int = 0
|
|
folders: list[FolderSyncResult] = field(default_factory=list)
|
|
blacklist_seeded: int = 0
|
|
|
|
@property
|
|
def added(self) -> int:
|
|
return sum(f.added for f in self.folders)
|
|
|
|
@property
|
|
def removed(self) -> int:
|
|
return sum(f.removed for f in self.folders)
|
|
|
|
@property
|
|
def refreshed(self) -> int:
|
|
return sum(f.refreshed for f in self.folders)
|
|
|
|
@property
|
|
def missing(self) -> int:
|
|
return sum(f.missing for f in self.folders)
|
|
|
|
@property
|
|
def errors(self) -> list[str]:
|
|
return [f"folder {f.media_id}: {f.error}" for f in self.folders if f.error]
|
|
|
|
@property
|
|
def warnings(self) -> list[str]:
|
|
return [f"folder {f.media_id}: {f.warning}" for f in self.folders if f.warning]
|
|
|
|
@property
|
|
def ok(self) -> bool:
|
|
return not self.errors
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"started_at": self.started_at,
|
|
"finished_at": self.finished_at,
|
|
"elapsed": max(0, self.finished_at - self.started_at),
|
|
"added": self.added,
|
|
"removed": self.removed,
|
|
"refreshed": self.refreshed,
|
|
"missing": self.missing,
|
|
"blacklist_seeded": self.blacklist_seeded,
|
|
"folders": [f.as_dict() for f in self.folders],
|
|
"errors": self.errors,
|
|
"warnings": self.warnings,
|
|
"ok": self.ok,
|
|
}
|
|
|
|
|
|
class FolderSyncService:
|
|
def __init__(
|
|
self,
|
|
config_store: ConfigStore,
|
|
repository: Repository,
|
|
client=None,
|
|
*,
|
|
client_provider=None,
|
|
clock=time.time,
|
|
logger=None,
|
|
capacity_warn: int = CAPACITY_WARN,
|
|
allow_empty_folders: bool = False,
|
|
) -> None:
|
|
if client is None and client_provider is None:
|
|
raise ValueError("必须提供 client 或 client_provider")
|
|
self.config_store = config_store
|
|
self.repo = repository
|
|
self.client = client
|
|
self.client_provider = client_provider
|
|
self._clock = clock
|
|
self.logger = logger
|
|
self.capacity_warn = capacity_warn
|
|
self.allow_empty_folders = allow_empty_folders
|
|
|
|
def _client(self):
|
|
if self.client_provider is not None:
|
|
return self.client_provider.get()
|
|
return self.client
|
|
|
|
def _now(self) -> int:
|
|
return int(self._clock())
|
|
|
|
def _log(self, level: str, event: str, **fields) -> None:
|
|
if self.logger is not None:
|
|
getattr(self.logger, level)(event, **fields)
|
|
|
|
def run(self, *, full_refresh: bool = False) -> SyncSummary:
|
|
cfg = self.config_store.get()
|
|
client = self._client()
|
|
now = self._now()
|
|
summary = SyncSummary(started_at=now)
|
|
self._log(
|
|
"info", "folder_sync.start", folders=len(cfg.folders), full=full_refresh
|
|
)
|
|
|
|
for folder in cfg.folders:
|
|
self.repo.ensure_folder(folder, now=now)
|
|
self.repo.disable_folders_except(f.media_id for f in cfg.folders)
|
|
if cfg.up_blacklist_seed:
|
|
summary.blacklist_seeded = self.repo.seed_blacklist(
|
|
cfg.up_blacklist_seed, now=now
|
|
)
|
|
|
|
for folder in cfg.folders:
|
|
try:
|
|
result = self._sync_folder(
|
|
client, cfg, folder, now=now, force_full=full_refresh
|
|
)
|
|
except Exception as exc: # noqa: BLE001 单个夹失败不阻塞其它夹
|
|
result = FolderSyncResult(
|
|
media_id=folder.media_id,
|
|
year=folder.year,
|
|
title=folder.title,
|
|
error=str(exc),
|
|
)
|
|
self._log(
|
|
"error",
|
|
"folder_sync.folder_error",
|
|
media_id=folder.media_id,
|
|
error=str(exc),
|
|
)
|
|
else:
|
|
self._log("info", "folder_sync.folder", **result.as_dict())
|
|
if result.warning:
|
|
self._log(
|
|
"warning",
|
|
"folder_sync.capacity",
|
|
media_id=folder.media_id,
|
|
warning=result.warning,
|
|
)
|
|
summary.folders.append(result)
|
|
|
|
summary.finished_at = self._now()
|
|
self._log(
|
|
"info" if summary.ok else "error",
|
|
"folder_sync.done",
|
|
added=summary.added,
|
|
removed=summary.removed,
|
|
refreshed=summary.refreshed,
|
|
missing=summary.missing,
|
|
errors=len(summary.errors),
|
|
elapsed=max(0, summary.finished_at - summary.started_at),
|
|
)
|
|
return summary
|
|
|
|
def _sync_folder(
|
|
self,
|
|
client,
|
|
cfg: AppConfig,
|
|
folder: FolderConfig,
|
|
*,
|
|
now: int,
|
|
force_full: bool,
|
|
) -> FolderSyncResult:
|
|
row = self.repo.get_folder(folder.media_id)
|
|
full_due = (
|
|
force_full
|
|
or row is None
|
|
or row.last_full_at is None
|
|
or now - int(row.last_full_at)
|
|
>= cfg.schedule.full_folder_refresh_hours * 3600
|
|
)
|
|
|
|
resources = client.fav_ids(folder.media_id)
|
|
remote_map = {item.aid: item for item in resources if item.type == 2}
|
|
remote = set(remote_map)
|
|
local = self.repo.folder_active_aids(folder.media_id)
|
|
added = remote - local
|
|
removed = local - remote
|
|
|
|
if not remote and local and not self.allow_empty_folders:
|
|
raise RuntimeError(
|
|
f"收藏夹 {folder.media_id} 远端返回 0 条，但本地仍有 {len(local)} 条；"
|
|
f"疑似接口异常，已跳过差集。确认收藏夹确实被清空后，"
|
|
f"可使用 --allow-empty-folder 重试"
|
|
)
|
|
|
|
result = FolderSyncResult(
|
|
media_id=folder.media_id,
|
|
year=folder.year,
|
|
title=folder.title or (row.title if row else ""),
|
|
remote_count=len(remote),
|
|
added=len(added),
|
|
removed=len(removed),
|
|
full_refresh=full_due,
|
|
)
|
|
|
|
targets = set(added)
|
|
if full_due:
|
|
targets |= remote
|
|
|
|
if targets:
|
|
infos = client.fav_infos(
|
|
folder.media_id,
|
|
sorted(targets),
|
|
folder_mid=cfg.bilibili.read.mid,
|
|
)
|
|
for aid, resource in infos.items.items():
|
|
if aid not in remote:
|
|
continue
|
|
outcome = self.repo.upsert_from_fav(
|
|
resource, folder.media_id, was_added=aid in added, now=now
|
|
)
|
|
if outcome.moved_folder:
|
|
self._log(
|
|
"warning",
|
|
"folder_sync.multi_folder",
|
|
aid=aid,
|
|
previous_folder_id=outcome.previous_folder_id,
|
|
folder_id=folder.media_id,
|
|
)
|
|
for aid in infos.missing:
|
|
source = remote_map.get(aid)
|
|
self.repo.mark_missing_resource(
|
|
aid,
|
|
folder_id=folder.media_id,
|
|
bvid=source.bvid if source else None,
|
|
now=now,
|
|
)
|
|
result.refreshed = len(infos.items)
|
|
result.missing = len(infos.missing)
|
|
|
|
for aid in removed:
|
|
self.repo.mark_resource_removed(folder.media_id, aid, now=now)
|
|
|
|
self.repo.touch_folder_resources(folder.media_id, now=now)
|
|
self.repo.finish_folder_sync(
|
|
folder,
|
|
media_count=len(remote),
|
|
full_refresh=full_due,
|
|
now=now,
|
|
)
|
|
|
|
if len(remote) >= self.capacity_warn:
|
|
result.warning = (
|
|
f"收藏夹「{result.title or folder.media_id}」已有 {len(remote)} 条，"
|
|
f"达到预警线 {self.capacity_warn}；请人工准备下一年度收藏夹并更新配置，"
|
|
f"系统不会自动换夹"
|
|
)
|
|
return result
|