refactor(sync): drop favorite-folder tracking and deal
This commit is contained in:
kairee committed
1 parent
62abbd7ea1
commit
6c4287b43a
21 files changed
+1390
-2766
No files matched your search
@@ -1,237 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""候选收藏、忽略、拉黑动作。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from ..bilibili.models import FavResource
|
||||
from ..config import ConfigStore
|
||||
from ..fav_position import VideoMeta, plan_insertion
|
||||
from ..repository import Repository
|
||||
from ..timeutil import shanghai_datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActionResult:
|
||||
ok: bool
|
||||
aid: int
|
||||
action: str
|
||||
state: str
|
||||
position: int | None = None
|
||||
already_in_folder: bool = False
|
||||
message: str = ""
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"ok": self.ok,
|
||||
"aid": self.aid,
|
||||
"action": self.action,
|
||||
"state": self.state,
|
||||
"position": self.position,
|
||||
"already_in_folder": self.already_in_folder,
|
||||
"message": self.message,
|
||||
}
|
||||
|
||||
|
||||
class FavoriteActionService:
|
||||
def __init__(
|
||||
self,
|
||||
config_store: ConfigStore,
|
||||
repository: Repository,
|
||||
write_provider,
|
||||
*,
|
||||
clock=time.time,
|
||||
logger=None,
|
||||
) -> None:
|
||||
self.config_store = config_store
|
||||
self.repo = repository
|
||||
self.write_provider = write_provider
|
||||
self._clock = clock
|
||||
self.logger = logger
|
||||
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
|
||||
def _log(self, level: str, event: str, **fields) -> None:
|
||||
if self.logger is not None:
|
||||
getattr(self.logger, level)(event, **fields)
|
||||
|
||||
def _target_folder(self, candidate: dict[str, Any]):
|
||||
cfg = self.config_store.get()
|
||||
up_mid = candidate.get("up_mid")
|
||||
if isinstance(up_mid, int) and not isinstance(up_mid, bool):
|
||||
folder_id = self.repo.folder_for_up(up_mid)
|
||||
if folder_id is not None:
|
||||
row = self.repo.get_folder(folder_id)
|
||||
if row is not None and row.enabled:
|
||||
return row
|
||||
year = shanghai_datetime(self._clock()).year
|
||||
folder = cfg.folder_for_year(year)
|
||||
if folder is not None:
|
||||
return folder
|
||||
latest = cfg.latest_folder()
|
||||
if latest is not None:
|
||||
return latest
|
||||
raise RuntimeError("没有可用的目标收藏夹，请检查 folders 配置")
|
||||
|
||||
def favorite(self, aid: int) -> ActionResult:
|
||||
aid = int(aid)
|
||||
candidate = self.repo.get_candidate(aid)
|
||||
if candidate is None:
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="favorite", state="missing",
|
||||
message="候选不存在或已处理",
|
||||
)
|
||||
try:
|
||||
folder = self._target_folder(candidate)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="favorite", state="failed",
|
||||
message=str(exc),
|
||||
)
|
||||
|
||||
token, state = self.repo.begin_fav_action(
|
||||
aid, folder.media_id, now=self._now()
|
||||
)
|
||||
if state == "done":
|
||||
action = self.repo.get_fav_action(token) or {}
|
||||
return ActionResult(
|
||||
ok=True, aid=aid, action="favorite", state="done",
|
||||
position=action.get("position"),
|
||||
message="已在收藏夹",
|
||||
)
|
||||
if state == "busy":
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="favorite", state="busy",
|
||||
message="该候选正在处理中，请稍后重试",
|
||||
)
|
||||
|
||||
already = False
|
||||
try:
|
||||
client = self.write_provider.get()
|
||||
ids = client.fav_ids(folder.media_id)
|
||||
if aid in {item.aid for item in ids}:
|
||||
already = True
|
||||
position = [item.aid for item in ids].index(aid) + 1
|
||||
else:
|
||||
client.deal(aid, folder.media_id)
|
||||
ids = client.fav_ids(folder.media_id)
|
||||
aid_order = [item.aid for item in ids]
|
||||
if aid not in aid_order:
|
||||
raise RuntimeError("deal 成功但回读收藏夹未找到该视频")
|
||||
metadata = {
|
||||
item_aid: VideoMeta(
|
||||
up_mid=(info.get("up_mid") if info else None),
|
||||
pubdate=int((info or {}).get("pubdate") or 0),
|
||||
)
|
||||
for item_aid, info in self.repo.video_metadata(aid_order).items()
|
||||
}
|
||||
plan = plan_insertion(
|
||||
aid_order,
|
||||
aid,
|
||||
metadata,
|
||||
VideoMeta(
|
||||
up_mid=candidate.get("up_mid"),
|
||||
pubdate=int(candidate.get("pubdate") or 0),
|
||||
),
|
||||
)
|
||||
type_map = {item.aid: item.type for item in ids}
|
||||
for predecessor, moved in plan.moves:
|
||||
client.sort(
|
||||
folder.media_id,
|
||||
f"{predecessor}:{type_map.get(predecessor, 2)}:"
|
||||
f"{moved}:{type_map.get(moved, 2)}",
|
||||
)
|
||||
final_order = [item.aid for item in client.fav_ids(folder.media_id)]
|
||||
if aid not in final_order:
|
||||
raise RuntimeError("排序后回读收藏夹未找到该视频")
|
||||
position = final_order.index(aid) + 1
|
||||
if position != plan.target_position:
|
||||
raise RuntimeError(
|
||||
f"回读位置不符：期望 #{plan.target_position}，实际 #{position}"
|
||||
)
|
||||
|
||||
resource = FavResource(
|
||||
aid=aid,
|
||||
bvid=candidate.get("bvid"),
|
||||
title=str(candidate.get("title") or "(无标题)"),
|
||||
up_mid=candidate.get("up_mid"),
|
||||
up_name=candidate.get("up_name"),
|
||||
pubdate=int(candidate.get("pubdate") or 0),
|
||||
duration=None,
|
||||
cover=candidate.get("cover"),
|
||||
attr=0,
|
||||
fav_time=None,
|
||||
)
|
||||
self.repo.upsert_from_fav(
|
||||
resource, folder.media_id, was_added=True, now=self._now()
|
||||
)
|
||||
self.repo.delete_candidate(aid)
|
||||
self.repo.finish_fav_action(token, position=position, now=self._now())
|
||||
self._log(
|
||||
"info", "favorite.done",
|
||||
aid=aid, folder_id=folder.media_id, position=position,
|
||||
already=already,
|
||||
)
|
||||
self.repo.bump_cache_version("videos")
|
||||
return ActionResult(
|
||||
ok=True, aid=aid, action="favorite", state="done",
|
||||
position=position, already_in_folder=already,
|
||||
message="已在收藏夹" if already else "已加入收藏夹",
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 失败保留候选，允许重试
|
||||
self.repo.fail_fav_action(token, error=str(exc), now=self._now())
|
||||
self._log(
|
||||
"error", "favorite.failed",
|
||||
aid=aid, folder_id=folder.media_id, error=str(exc),
|
||||
)
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="favorite", state="failed",
|
||||
message=str(exc),
|
||||
)
|
||||
|
||||
def ignore(self, aid: int) -> ActionResult:
|
||||
aid = int(aid)
|
||||
candidate = self.repo.get_candidate(aid)
|
||||
if candidate is None:
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="ignore", state="missing",
|
||||
message="候选不存在或已处理",
|
||||
)
|
||||
self.repo.mark_candidate_decision(aid, "rejected", now=self._now())
|
||||
self.repo.bump_cache_version("videos")
|
||||
return ActionResult(
|
||||
ok=True, aid=aid, action="ignore", state="rejected",
|
||||
message="已忽略",
|
||||
)
|
||||
|
||||
def blacklist(self, aid: int) -> ActionResult:
|
||||
aid = int(aid)
|
||||
candidate = self.repo.get_candidate(aid)
|
||||
if candidate is None:
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="blacklist", state="missing",
|
||||
message="候选不存在或已处理",
|
||||
)
|
||||
up_mid = candidate.get("up_mid")
|
||||
if not isinstance(up_mid, int) or isinstance(up_mid, bool):
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="blacklist", state="failed",
|
||||
message="候选缺少 UP mid，无法拉黑",
|
||||
)
|
||||
self.repo.add_blacklist(
|
||||
up_mid,
|
||||
name=candidate.get("up_name"),
|
||||
reason="telegram",
|
||||
now=self._now(),
|
||||
)
|
||||
rejected = self.repo.reject_pending_for_up(up_mid, now=self._now())
|
||||
self.repo.mark_candidate_decision(aid, "rejected", now=self._now())
|
||||
self.repo.bump_cache_version("videos")
|
||||
return ActionResult(
|
||||
ok=True, aid=aid, action="blacklist", state="rejected",
|
||||
message=f"已拉黑 UP，并忽略该 UP 的 {rejected} 条待决策候选",
|
||||
)
|
||||
@@ -1,335 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""收藏夹同步（库的真相源）。
|
||||
|
||||
每次运行：
|
||||
|
||||
1. 读取每个配置年度夹的全部 aid；
|
||||
2. 与本地 ``folder_resource`` 做差集；
|
||||
3. 新增资源按 200 条分块拉取 ``infos`` 并入库；
|
||||
4. 从收藏夹移除的资源标记 ``unfaved``（不删除历史）；
|
||||
5. 每 ``full_folder_refresh_hours`` 小时全量刷新一次元数据并识别失效资源；
|
||||
6. 容量接近上限时预警，但绝不自动换夹。
|
||||
|
||||
黑名单只在发现流程生效，本服务不读取 ``up_blacklist``。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from ..config import AppConfig, ConfigStore, FolderConfig
|
||||
from ..repository import Repository
|
||||
from .up_profile import UpProfileService
|
||||
|
||||
CAPACITY_WARN = 950
|
||||
|
||||
|
||||
@dataclass
|
||||
class FolderSyncResult:
|
||||
media_id: int
|
||||
year: int
|
||||
title: str
|
||||
remote_count: int = 0
|
||||
added: int = 0
|
||||
removed: int = 0
|
||||
refreshed: int = 0
|
||||
missing: int = 0
|
||||
full_refresh: bool = False
|
||||
warning: str | None = None
|
||||
error: str | None = None
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"media_id": self.media_id,
|
||||
"year": self.year,
|
||||
"title": self.title,
|
||||
"remote_count": self.remote_count,
|
||||
"added": self.added,
|
||||
"removed": self.removed,
|
||||
"refreshed": self.refreshed,
|
||||
"missing": self.missing,
|
||||
"full_refresh": self.full_refresh,
|
||||
"warning": self.warning,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncSummary:
|
||||
started_at: int
|
||||
finished_at: int = 0
|
||||
folders: list[FolderSyncResult] = field(default_factory=list)
|
||||
blacklist_seeded: int = 0
|
||||
followers_updated: int = 0
|
||||
followers_failed: int = 0
|
||||
|
||||
@property
|
||||
def added(self) -> int:
|
||||
return sum(f.added for f in self.folders)
|
||||
|
||||
@property
|
||||
def removed(self) -> int:
|
||||
return sum(f.removed for f in self.folders)
|
||||
|
||||
@property
|
||||
def refreshed(self) -> int:
|
||||
return sum(f.refreshed for f in self.folders)
|
||||
|
||||
@property
|
||||
def missing(self) -> int:
|
||||
return sum(f.missing for f in self.folders)
|
||||
|
||||
@property
|
||||
def errors(self) -> list[str]:
|
||||
return [f"folder {f.media_id}: {f.error}" for f in self.folders if f.error]
|
||||
|
||||
@property
|
||||
def warnings(self) -> list[str]:
|
||||
return [f"folder {f.media_id}: {f.warning}" for f in self.folders if f.warning]
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return not self.errors
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"elapsed": max(0, self.finished_at - self.started_at),
|
||||
"added": self.added,
|
||||
"removed": self.removed,
|
||||
"refreshed": self.refreshed,
|
||||
"missing": self.missing,
|
||||
"blacklist_seeded": self.blacklist_seeded,
|
||||
"followers_updated": self.followers_updated,
|
||||
"followers_failed": self.followers_failed,
|
||||
"folders": [f.as_dict() for f in self.folders],
|
||||
"errors": self.errors,
|
||||
"warnings": self.warnings,
|
||||
"ok": self.ok,
|
||||
}
|
||||
|
||||
|
||||
class FolderSyncService:
|
||||
def __init__(
|
||||
self,
|
||||
config_store: ConfigStore,
|
||||
repository: Repository,
|
||||
client=None,
|
||||
*,
|
||||
client_provider=None,
|
||||
clock=time.time,
|
||||
logger=None,
|
||||
capacity_warn: int = CAPACITY_WARN,
|
||||
allow_empty_folders: bool = False,
|
||||
up_profile_service: UpProfileService | None = None,
|
||||
capture_followers: bool = True,
|
||||
) -> None:
|
||||
if client is None and client_provider is None:
|
||||
raise ValueError("必须提供 client 或 client_provider")
|
||||
self.config_store = config_store
|
||||
self.repo = repository
|
||||
self.client = client
|
||||
self.client_provider = client_provider
|
||||
self._clock = clock
|
||||
self.logger = logger
|
||||
self.capacity_warn = capacity_warn
|
||||
self.allow_empty_folders = allow_empty_folders
|
||||
self.capture_followers = capture_followers
|
||||
self.up_profile = up_profile_service or UpProfileService(
|
||||
repository, self._client, clock=clock, logger=logger
|
||||
)
|
||||
|
||||
def _client(self):
|
||||
if self.client_provider is not None:
|
||||
return self.client_provider.get()
|
||||
return self.client
|
||||
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
|
||||
def _log(self, level: str, event: str, **fields) -> None:
|
||||
if self.logger is not None:
|
||||
getattr(self.logger, level)(event, **fields)
|
||||
|
||||
def run(self, *, full_refresh: bool = False) -> SyncSummary:
|
||||
cfg = self.config_store.get()
|
||||
client = self._client()
|
||||
now = self._now()
|
||||
summary = SyncSummary(started_at=now)
|
||||
self._log(
|
||||
"info", "folder_sync.start", folders=len(cfg.folders), full=full_refresh
|
||||
)
|
||||
|
||||
for folder in cfg.folders:
|
||||
self.repo.ensure_folder(folder, now=now)
|
||||
self.repo.disable_folders_except(f.media_id for f in cfg.folders)
|
||||
if cfg.up_blacklist_seed:
|
||||
summary.blacklist_seeded = self.repo.seed_blacklist(
|
||||
cfg.up_blacklist_seed, now=now
|
||||
)
|
||||
|
||||
for folder in cfg.folders:
|
||||
try:
|
||||
result = self._sync_folder(
|
||||
client, cfg, folder, now=now, force_full=full_refresh
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 单个夹失败不阻塞其它夹
|
||||
result = FolderSyncResult(
|
||||
media_id=folder.media_id,
|
||||
year=folder.year,
|
||||
title=folder.title,
|
||||
error=str(exc),
|
||||
)
|
||||
self._log(
|
||||
"error",
|
||||
"folder_sync.folder_error",
|
||||
media_id=folder.media_id,
|
||||
error=str(exc),
|
||||
)
|
||||
else:
|
||||
self._log("info", "folder_sync.folder", **result.as_dict())
|
||||
if result.warning:
|
||||
self._log(
|
||||
"warning",
|
||||
"folder_sync.capacity",
|
||||
media_id=folder.media_id,
|
||||
warning=result.warning,
|
||||
)
|
||||
summary.folders.append(result)
|
||||
|
||||
if self.capture_followers:
|
||||
try:
|
||||
missing = self.repo.up_mids_missing_follower()
|
||||
if missing:
|
||||
capture = self.up_profile.capture(
|
||||
[int(row["up_mid"]) for row in missing],
|
||||
names={
|
||||
int(row["up_mid"]): str(row.get("up_name") or "")
|
||||
for row in missing
|
||||
},
|
||||
live_since=summary.started_at - 3600,
|
||||
)
|
||||
summary.followers_updated = capture.updated_videos
|
||||
summary.followers_failed = capture.failed
|
||||
except Exception as exc: # noqa: BLE001 粉丝采集失败不影响收藏夹同步
|
||||
summary.followers_failed += 1
|
||||
self._log(
|
||||
"warning", "folder_sync.followers_failed", error=str(exc)
|
||||
)
|
||||
|
||||
summary.finished_at = self._now()
|
||||
self._log(
|
||||
"info" if summary.ok else "error",
|
||||
"folder_sync.done",
|
||||
added=summary.added,
|
||||
removed=summary.removed,
|
||||
refreshed=summary.refreshed,
|
||||
missing=summary.missing,
|
||||
errors=len(summary.errors),
|
||||
elapsed=max(0, summary.finished_at - summary.started_at),
|
||||
)
|
||||
if summary.added or summary.removed or summary.refreshed or summary.followers_updated:
|
||||
self.repo.bump_cache_version("videos")
|
||||
return summary
|
||||
|
||||
def _sync_folder(
|
||||
self,
|
||||
client,
|
||||
cfg: AppConfig,
|
||||
folder: FolderConfig,
|
||||
*,
|
||||
now: int,
|
||||
force_full: bool,
|
||||
) -> FolderSyncResult:
|
||||
row = self.repo.get_folder(folder.media_id)
|
||||
full_due = (
|
||||
force_full
|
||||
or row is None
|
||||
or row.last_full_at is None
|
||||
or now - int(row.last_full_at)
|
||||
>= cfg.schedule.full_folder_refresh_hours * 3600
|
||||
)
|
||||
|
||||
resources = client.fav_ids(folder.media_id)
|
||||
remote_map = {item.aid: item for item in resources if item.type == 2}
|
||||
remote = set(remote_map)
|
||||
local = self.repo.folder_active_aids(folder.media_id)
|
||||
added = remote - local
|
||||
removed = local - remote
|
||||
|
||||
if not remote and local and not self.allow_empty_folders:
|
||||
raise RuntimeError(
|
||||
f"收藏夹 {folder.media_id} 远端返回 0 条，但本地仍有 {len(local)} 条；"
|
||||
f"疑似接口异常，已跳过差集。确认收藏夹确实被清空后，"
|
||||
f"可使用 --allow-empty-folder 重试"
|
||||
)
|
||||
|
||||
result = FolderSyncResult(
|
||||
media_id=folder.media_id,
|
||||
year=folder.year,
|
||||
title=folder.title or (row.title if row else ""),
|
||||
remote_count=len(remote),
|
||||
added=len(added),
|
||||
removed=len(removed),
|
||||
full_refresh=full_due,
|
||||
)
|
||||
|
||||
targets = set(added)
|
||||
if full_due:
|
||||
targets |= remote
|
||||
|
||||
if targets:
|
||||
infos = client.fav_infos(
|
||||
folder.media_id,
|
||||
sorted(targets),
|
||||
folder_mid=cfg.bilibili.read.mid,
|
||||
)
|
||||
for aid, resource in infos.items.items():
|
||||
if aid not in remote:
|
||||
continue
|
||||
outcome = self.repo.upsert_from_fav(
|
||||
resource, folder.media_id, was_added=aid in added, now=now
|
||||
)
|
||||
if aid in added:
|
||||
# 人工已加入收藏夹：候选池不再保留同一条视频
|
||||
self.repo.delete_candidate(aid)
|
||||
if outcome.moved_folder:
|
||||
self._log(
|
||||
"warning",
|
||||
"folder_sync.multi_folder",
|
||||
aid=aid,
|
||||
previous_folder_id=outcome.previous_folder_id,
|
||||
folder_id=folder.media_id,
|
||||
)
|
||||
for aid in infos.missing:
|
||||
source = remote_map.get(aid)
|
||||
self.repo.mark_missing_resource(
|
||||
aid,
|
||||
folder_id=folder.media_id,
|
||||
bvid=source.bvid if source else None,
|
||||
now=now,
|
||||
)
|
||||
result.refreshed = len(infos.items)
|
||||
result.missing = len(infos.missing)
|
||||
|
||||
for aid in removed:
|
||||
self.repo.mark_resource_removed(folder.media_id, aid, now=now)
|
||||
|
||||
self.repo.touch_folder_resources(folder.media_id, now=now)
|
||||
self.repo.finish_folder_sync(
|
||||
folder,
|
||||
media_count=len(remote),
|
||||
full_refresh=full_due,
|
||||
now=now,
|
||||
)
|
||||
|
||||
if len(remote) >= self.capacity_warn:
|
||||
result.warning = (
|
||||
f"收藏夹「{result.title or folder.media_id}」已有 {len(remote)} 条，"
|
||||
f"达到预警线 {self.capacity_warn}；请人工准备下一年度收藏夹并更新配置，"
|
||||
f"系统不会自动换夹"
|
||||
)
|
||||
return result
|
||||
Reference in new issue
Block a user