336 lines
11 KiB
Python
336 lines
11 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""收藏夹同步（库的真相源）。
|
|
|
|
每次运行：
|
|
|
|
1. 读取每个配置年度夹的全部 aid；
|
|
2. 与本地 ``folder_resource`` 做差集；
|
|
3. 新增资源按 200 条分块拉取 ``infos`` 并入库；
|
|
4. 从收藏夹移除的资源标记 ``unfaved``（不删除历史）；
|
|
5. 每 ``full_folder_refresh_hours`` 小时全量刷新一次元数据并识别失效资源；
|
|
6. 容量接近上限时预警，但绝不自动换夹。
|
|
|
|
黑名单只在发现流程生效，本服务不读取 ``up_blacklist``。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from ..config import AppConfig, ConfigStore, FolderConfig
|
|
from ..repository import Repository
|
|
from .up_profile import UpProfileService
|
|
|
|
CAPACITY_WARN = 950
|
|
|
|
|
|
@dataclass
|
|
class FolderSyncResult:
|
|
media_id: int
|
|
year: int
|
|
title: str
|
|
remote_count: int = 0
|
|
added: int = 0
|
|
removed: int = 0
|
|
refreshed: int = 0
|
|
missing: int = 0
|
|
full_refresh: bool = False
|
|
warning: str | None = None
|
|
error: str | None = None
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"media_id": self.media_id,
|
|
"year": self.year,
|
|
"title": self.title,
|
|
"remote_count": self.remote_count,
|
|
"added": self.added,
|
|
"removed": self.removed,
|
|
"refreshed": self.refreshed,
|
|
"missing": self.missing,
|
|
"full_refresh": self.full_refresh,
|
|
"warning": self.warning,
|
|
"error": self.error,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class SyncSummary:
|
|
started_at: int
|
|
finished_at: int = 0
|
|
folders: list[FolderSyncResult] = field(default_factory=list)
|
|
blacklist_seeded: int = 0
|
|
followers_updated: int = 0
|
|
followers_failed: int = 0
|
|
|
|
@property
|
|
def added(self) -> int:
|
|
return sum(f.added for f in self.folders)
|
|
|
|
@property
|
|
def removed(self) -> int:
|
|
return sum(f.removed for f in self.folders)
|
|
|
|
@property
|
|
def refreshed(self) -> int:
|
|
return sum(f.refreshed for f in self.folders)
|
|
|
|
@property
|
|
def missing(self) -> int:
|
|
return sum(f.missing for f in self.folders)
|
|
|
|
@property
|
|
def errors(self) -> list[str]:
|
|
return [f"folder {f.media_id}: {f.error}" for f in self.folders if f.error]
|
|
|
|
@property
|
|
def warnings(self) -> list[str]:
|
|
return [f"folder {f.media_id}: {f.warning}" for f in self.folders if f.warning]
|
|
|
|
@property
|
|
def ok(self) -> bool:
|
|
return not self.errors
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"started_at": self.started_at,
|
|
"finished_at": self.finished_at,
|
|
"elapsed": max(0, self.finished_at - self.started_at),
|
|
"added": self.added,
|
|
"removed": self.removed,
|
|
"refreshed": self.refreshed,
|
|
"missing": self.missing,
|
|
"blacklist_seeded": self.blacklist_seeded,
|
|
"followers_updated": self.followers_updated,
|
|
"followers_failed": self.followers_failed,
|
|
"folders": [f.as_dict() for f in self.folders],
|
|
"errors": self.errors,
|
|
"warnings": self.warnings,
|
|
"ok": self.ok,
|
|
}
|
|
|
|
|
|
class FolderSyncService:
|
|
def __init__(
|
|
self,
|
|
config_store: ConfigStore,
|
|
repository: Repository,
|
|
client=None,
|
|
*,
|
|
client_provider=None,
|
|
clock=time.time,
|
|
logger=None,
|
|
capacity_warn: int = CAPACITY_WARN,
|
|
allow_empty_folders: bool = False,
|
|
up_profile_service: UpProfileService | None = None,
|
|
capture_followers: bool = True,
|
|
) -> None:
|
|
if client is None and client_provider is None:
|
|
raise ValueError("必须提供 client 或 client_provider")
|
|
self.config_store = config_store
|
|
self.repo = repository
|
|
self.client = client
|
|
self.client_provider = client_provider
|
|
self._clock = clock
|
|
self.logger = logger
|
|
self.capacity_warn = capacity_warn
|
|
self.allow_empty_folders = allow_empty_folders
|
|
self.capture_followers = capture_followers
|
|
self.up_profile = up_profile_service or UpProfileService(
|
|
repository, self._client, clock=clock, logger=logger
|
|
)
|
|
|
|
def _client(self):
|
|
if self.client_provider is not None:
|
|
return self.client_provider.get()
|
|
return self.client
|
|
|
|
def _now(self) -> int:
|
|
return int(self._clock())
|
|
|
|
def _log(self, level: str, event: str, **fields) -> None:
|
|
if self.logger is not None:
|
|
getattr(self.logger, level)(event, **fields)
|
|
|
|
def run(self, *, full_refresh: bool = False) -> SyncSummary:
|
|
cfg = self.config_store.get()
|
|
client = self._client()
|
|
now = self._now()
|
|
summary = SyncSummary(started_at=now)
|
|
self._log(
|
|
"info", "folder_sync.start", folders=len(cfg.folders), full=full_refresh
|
|
)
|
|
|
|
for folder in cfg.folders:
|
|
self.repo.ensure_folder(folder, now=now)
|
|
self.repo.disable_folders_except(f.media_id for f in cfg.folders)
|
|
if cfg.up_blacklist_seed:
|
|
summary.blacklist_seeded = self.repo.seed_blacklist(
|
|
cfg.up_blacklist_seed, now=now
|
|
)
|
|
|
|
for folder in cfg.folders:
|
|
try:
|
|
result = self._sync_folder(
|
|
client, cfg, folder, now=now, force_full=full_refresh
|
|
)
|
|
except Exception as exc: # noqa: BLE001 单个夹失败不阻塞其它夹
|
|
result = FolderSyncResult(
|
|
media_id=folder.media_id,
|
|
year=folder.year,
|
|
title=folder.title,
|
|
error=str(exc),
|
|
)
|
|
self._log(
|
|
"error",
|
|
"folder_sync.folder_error",
|
|
media_id=folder.media_id,
|
|
error=str(exc),
|
|
)
|
|
else:
|
|
self._log("info", "folder_sync.folder", **result.as_dict())
|
|
if result.warning:
|
|
self._log(
|
|
"warning",
|
|
"folder_sync.capacity",
|
|
media_id=folder.media_id,
|
|
warning=result.warning,
|
|
)
|
|
summary.folders.append(result)
|
|
|
|
if self.capture_followers:
|
|
try:
|
|
missing = self.repo.up_mids_missing_follower()
|
|
if missing:
|
|
capture = self.up_profile.capture(
|
|
[int(row["up_mid"]) for row in missing],
|
|
names={
|
|
int(row["up_mid"]): str(row.get("up_name") or "")
|
|
for row in missing
|
|
},
|
|
live_since=summary.started_at - 3600,
|
|
)
|
|
summary.followers_updated = capture.updated_videos
|
|
summary.followers_failed = capture.failed
|
|
except Exception as exc: # noqa: BLE001 粉丝采集失败不影响收藏夹同步
|
|
summary.followers_failed += 1
|
|
self._log(
|
|
"warning", "folder_sync.followers_failed", error=str(exc)
|
|
)
|
|
|
|
summary.finished_at = self._now()
|
|
self._log(
|
|
"info" if summary.ok else "error",
|
|
"folder_sync.done",
|
|
added=summary.added,
|
|
removed=summary.removed,
|
|
refreshed=summary.refreshed,
|
|
missing=summary.missing,
|
|
errors=len(summary.errors),
|
|
elapsed=max(0, summary.finished_at - summary.started_at),
|
|
)
|
|
if summary.added or summary.removed or summary.refreshed or summary.followers_updated:
|
|
self.repo.bump_cache_version("videos")
|
|
return summary
|
|
|
|
def _sync_folder(
|
|
self,
|
|
client,
|
|
cfg: AppConfig,
|
|
folder: FolderConfig,
|
|
*,
|
|
now: int,
|
|
force_full: bool,
|
|
) -> FolderSyncResult:
|
|
row = self.repo.get_folder(folder.media_id)
|
|
full_due = (
|
|
force_full
|
|
or row is None
|
|
or row.last_full_at is None
|
|
or now - int(row.last_full_at)
|
|
>= cfg.schedule.full_folder_refresh_hours * 3600
|
|
)
|
|
|
|
resources = client.fav_ids(folder.media_id)
|
|
remote_map = {item.aid: item for item in resources if item.type == 2}
|
|
remote = set(remote_map)
|
|
local = self.repo.folder_active_aids(folder.media_id)
|
|
added = remote - local
|
|
removed = local - remote
|
|
|
|
if not remote and local and not self.allow_empty_folders:
|
|
raise RuntimeError(
|
|
f"收藏夹 {folder.media_id} 远端返回 0 条，但本地仍有 {len(local)} 条；"
|
|
f"疑似接口异常，已跳过差集。确认收藏夹确实被清空后，"
|
|
f"可使用 --allow-empty-folder 重试"
|
|
)
|
|
|
|
result = FolderSyncResult(
|
|
media_id=folder.media_id,
|
|
year=folder.year,
|
|
title=folder.title or (row.title if row else ""),
|
|
remote_count=len(remote),
|
|
added=len(added),
|
|
removed=len(removed),
|
|
full_refresh=full_due,
|
|
)
|
|
|
|
targets = set(added)
|
|
if full_due:
|
|
targets |= remote
|
|
|
|
if targets:
|
|
infos = client.fav_infos(
|
|
folder.media_id,
|
|
sorted(targets),
|
|
folder_mid=cfg.bilibili.read.mid,
|
|
)
|
|
for aid, resource in infos.items.items():
|
|
if aid not in remote:
|
|
continue
|
|
outcome = self.repo.upsert_from_fav(
|
|
resource, folder.media_id, was_added=aid in added, now=now
|
|
)
|
|
if aid in added:
|
|
# 人工已加入收藏夹：候选池不再保留同一条视频
|
|
self.repo.delete_candidate(aid)
|
|
if outcome.moved_folder:
|
|
self._log(
|
|
"warning",
|
|
"folder_sync.multi_folder",
|
|
aid=aid,
|
|
previous_folder_id=outcome.previous_folder_id,
|
|
folder_id=folder.media_id,
|
|
)
|
|
for aid in infos.missing:
|
|
source = remote_map.get(aid)
|
|
self.repo.mark_missing_resource(
|
|
aid,
|
|
folder_id=folder.media_id,
|
|
bvid=source.bvid if source else None,
|
|
now=now,
|
|
)
|
|
result.refreshed = len(infos.items)
|
|
result.missing = len(infos.missing)
|
|
|
|
for aid in removed:
|
|
self.repo.mark_resource_removed(folder.media_id, aid, now=now)
|
|
|
|
self.repo.touch_folder_resources(folder.media_id, now=now)
|
|
self.repo.finish_folder_sync(
|
|
folder,
|
|
media_count=len(remote),
|
|
full_refresh=full_due,
|
|
now=now,
|
|
)
|
|
|
|
if len(remote) >= self.capacity_warn:
|
|
result.warning = (
|
|
f"收藏夹「{result.title or folder.media_id}」已有 {len(remote)} 条，"
|
|
f"达到预警线 {self.capacity_warn}；请人工准备下一年度收藏夹并更新配置，"
|
|
f"系统不会自动换夹"
|
|
)
|
|
return result
|