kcloud/re-xxm
2
0 Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor(sync): drop favorite-folder tracking and deal

Browse Source
This commit is contained in:
kairee committed 2026-11-01 13:59:10 +08:00
1 parent 62abbd7ea1
commit 6c4287b43a
21 files changed
+1390 -2766
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

reaction_board/bilibili/write_client.py
+4 -17
View File
@@ -1,5 +1,8 @@
# -*- coding: utf-8 -*-
"""B 站收藏夹写客户端（``deal`` / ``sort``）。
"""B 站收藏夹写客户端（``sort``）。
收藏夹重排序是本项目**唯一**使用 ``bilibili.write`` 凭据的写操作，且频率极低。
加入收藏夹（``deal``）等写操作已移除，收藏夹由人工维护。
只应使用 ``bilibili.write`` 凭据构造。
"""
@@ -39,22 +42,6 @@ class BilibiliWriteClient(BilibiliClient):
raise self._api_error(payload)
return payload
def deal(self, aid: int, folder_id: int) -> dict[str, Any]:
return self.post_form(
f"{BASE}/x/v3/fav/resource/deal",
{
"rid": int(aid),
"type": 2,
"add_media_ids": int(folder_id),
"del_media_ids": "",
"csrf": self.csrf,
},
referer=(
f"https://space.bilibili.com/{self.mid}/favlist"
f"?fid={int(folder_id)}&ftype=create"
),
)
def sort(self, media_id: int, sort_spec: str) -> dict[str, Any]:
return self.post_form(
f"{BASE}/x/v3/fav/resource/sort",
reaction_board/cli.py
+40 -96
View File
@@ -2,7 +2,7 @@
"""看板命令行入口。
python app.py init # 建库 / 迁移 / 写入黑名单种子（离线）
python app.py sync # 收藏夹只读同步（首次导入或增量）
python app.py ingest # 手动入库 B 站视频链接 / BV 号 / av 号（只写本地库）
python app.py snapshot # 采集当日七项快照与粉丝数
python app.py discover # 执行一次搜索发现并写入候选池
python app.py candidates # 查看候选池（离线）
@@ -10,6 +10,7 @@
python app.py run # 按配置周期常驻运行（--once 只跑一次）
python app.py status # 查看库内状态（离线）
收藏夹重排序使用独立的 ``favsort.py``（唯一使用 ``bilibili.write`` 的功能）。
P2 起 ``python app.py`` 将在此基础上启动本地看板服务。
"""
@@ -27,7 +28,6 @@ from .bilibili.auth_flow import AuthService
from .bilibili.client import BilibiliClient, Pacer
from .bilibili.provider import ClientProvider
from .bilibili.search import SearchClient
from .bilibili.write_client import BilibiliWriteClient
from .config import ConfigError, ConfigStore
from .database import Database
from .logging_json import JsonLogger
@@ -35,8 +35,7 @@ from .notify.telegram import TelegramClient
from .repository import Repository
from .scheduler import ScheduledTask, Scheduler
from .services.discovery import DiscoveryService
from .services.favorites import FavoriteActionService
from .services.folder_sync import FolderSyncService
from .services.library import LibraryService
from .services.snapshot import SnapshotService
from .services.telegram_poller import TelegramPoller
from .services.up_profile import UpProfileService
@@ -66,20 +65,6 @@ def _build_parser() -> argparse.ArgumentParser:
sub = parser.add_subparsers(dest="command", required=False)
sub.add_parser("init", parents=[common], help="建库、迁移并写入黑名单种子")
sync = sub.add_parser("sync", parents=[common], help="同步已配置的年度收藏夹")
sync.add_argument(
"--full", action="store_true", help="强制全量刷新元数据（忽略 6 小时周期）"
)
sync.add_argument(
"--allow-empty-folder",
action="store_true",
help="允许把远端 0 条视为真实清空（默认按异常跳过差集）",
)
sync.add_argument(
"--no-capture-followers",
action="store_true",
help="跳过同步末尾的 UP 粉丝采集（改用 backfill-followers 单独回填）",
)
snapshot = sub.add_parser(
"snapshot", parents=[common], help="采集当日七项快照与粉丝数"
)
@@ -103,6 +88,14 @@ def _build_parser() -> argparse.ArgumentParser:
default=None,
help="只查看指定决策状态的候选",
)
ingest = sub.add_parser(
"ingest", parents=[common],
help="手动入库：B 站视频链接、BV 号或 av 号（只写本地库）",
)
ingest.add_argument(
"refs", nargs="+", metavar="REF",
help="视频链接、BV 号（BV…）、av 号（av12345）或含它们的字符串",
)
auth = sub.add_parser("auth", parents=[common], help="扫码授权并更新指定账号 Cookie")
auth.add_argument("--account", choices=("read", "write"), default="read",
help="要授权的账号角色（默认 read）")
@@ -112,15 +105,10 @@ def _build_parser() -> argparse.ArgumentParser:
auth_check = sub.add_parser("auth-check", parents=[common], help="检查账号登录态")
auth_check.add_argument("--account", choices=("read", "write", "all"), default="read",
help="要检查的账号角色（默认 read）")
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行同步与快照")
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行发现与快照")
run.add_argument(
"--once", action="store_true", help="只执行一次到期任务后退出"
)
run.add_argument(
"--allow-empty-folder",
action="store_true",
help="允许把远端 0 条视为真实清空（默认按异常跳过差集）",
)
serve = sub.add_parser("serve", parents=[common], help="启动只读本地看板（不运行调度）")
serve.add_argument(
"--preview", action="store_true",
@@ -209,32 +197,30 @@ def cmd_init(args, logger: JsonLogger) -> int:
return 0
def cmd_sync(args, logger: JsonLogger) -> int:
store = ConfigStore(args.config, require_auth=True)
def cmd_ingest(args, logger: JsonLogger) -> int:
store = ConfigStore(args.config, require_auth=False)
cfg = store.get()
db = _open_db(cfg, logger)
try:
repo = Repository(db)
client = _client(cfg.bilibili.read, logger)
service = FolderSyncService(
store,
repo,
client,
logger=logger,
allow_empty_folders=args.allow_empty_folder,
capture_followers=not getattr(args, "no_capture_followers", False),
service = LibraryService(
store, repo, lambda: client, logger=logger
)
started = int(time.time())
summary = service.run(full_refresh=args.full)
payload = summary.as_dict()
text = " ".join(args.refs)
result = service.ingest_text(text)
payload = result.as_dict()
payload["database"] = str(cfg.storage.database_path)
payload["counts"] = repo.counts()
_record_task(
repo, "folder_sync", ok=summary.ok, started_at=started,
finished_at=int(time.time()), payload=payload,
repo, "ingest", ok=bool(result.results) and not any(
item.state == "error" for item in result.results
),
payload=payload,
)
_print(payload)
return 0 if summary.ok else 1
return 0 if result.results and not any(
item.state == "error" for item in result.results
) else 1
finally:
db.close()
@@ -403,7 +389,9 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
web_server = None
try:
repo = Repository(db)
folder_provider = ClientProvider(
if cfg.up_blacklist_seed:
repo.seed_blacklist(cfg.up_blacklist_seed)
read_provider = ClientProvider(
store, "read", lambda credentials: _client(credentials, logger)
)
snapshot_provider = ClientProvider(
@@ -413,34 +401,22 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
credentials, logger, pacer=Pacer(1.0, 1.6), retries=1
),
)
folder_service = FolderSyncService(
store,
repo,
client_provider=folder_provider,
logger=logger,
allow_empty_folders=args.allow_empty_folder,
)
snapshot_service = SnapshotService(
store, repo, client_provider=snapshot_provider, logger=logger
)
write_provider = ClientProvider(
up_profile_service = UpProfileService(repo, read_provider.get, logger=logger)
library_service = LibraryService(
store,
"write",
lambda credentials: BilibiliWriteClient(
cookie=credentials.cookie,
csrf=credentials.csrf,
mid=credentials.mid,
logger=logger,
),
)
favorite_service = FavoriteActionService(
store, repo, write_provider, logger=logger
repo,
read_provider,
up_profile_service=up_profile_service,
logger=logger,
)
notifier = _telegram(cfg, logger)
discovery_service = DiscoveryService(
store,
repo,
SearchClient(folder_provider.get),
SearchClient(read_provider.get),
notifier=notifier,
logger=logger,
)
@@ -513,29 +489,6 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
logger.info("snapshot.summary", **payload)
return summary
def run_sync():
started = int(time.time())
try:
summary = folder_service.run()
except Exception as exc: # noqa: BLE001
_record_task(
repo, "folder_sync", ok=False, started_at=started,
finished_at=int(time.time()), error=str(exc),
)
raise
payload = summary.as_dict()
_record_task(
repo, "folder_sync", ok=summary.ok, started_at=started,
finished_at=int(time.time()), payload=payload,
error=None if summary.ok else "；".join(summary.errors),
)
last["folder"] = summary
logger.info("folder_sync.summary", **payload)
# 新入库/恢复的视频立即补当天快照，而不是等到次日。
if summary.added > 0:
run_snapshot()
return summary
def run_config_watch():
before = store.revision
store.get()
@@ -547,11 +500,6 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
snapshot_at = f"{hh:02d}:{mm:02d}"
scheduler = Scheduler(
[
ScheduledTask(
"folder_sync",
float(cfg.schedule.folder_sync_minutes * 60),
run_sync,
),
ScheduledTask(
"snapshot",
86400.0,
@@ -579,7 +527,7 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
daemon=True,
).start()
if notifier is not None and not args.once:
poller = TelegramPoller(notifier, favorite_service, logger=logger)
poller = TelegramPoller(notifier, library_service, logger=logger)
threading.Thread(
target=poller.run_forever,
name="telegram-poller",
@@ -592,18 +540,15 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
else:
logger.info(
"scheduler.start",
folder_sync_seconds=cfg.schedule.folder_sync_minutes * 60,
snapshot_at=snapshot_at,
)
scheduler.run_forever()
folder_summary = last.get("folder")
snapshot_summary = last.get("snapshot")
discover_summary = last.get("discover")
folder_ok = folder_summary is None or getattr(folder_summary, "ok", True)
snapshot_ok = snapshot_summary is None or getattr(snapshot_summary, "completed", True)
discover_ok = discover_summary is None or not getattr(discover_summary, "errors", [])
return 0 if folder_ok and snapshot_ok and discover_ok else 1
return 0 if snapshot_ok and discover_ok else 1
finally:
if web_server is not None:
try:
@@ -703,16 +648,15 @@ def main(argv: Sequence[str] | None = None) -> int:
# 无子命令时启动完整服务：只读 Web + 调度 + Telegram。
args.command = "run"
args.once = False
args.allow_empty_folder = False
return cmd_run(args, logger, serve_web=True)
if args.command == "serve":
return cmd_serve(args, logger)
if args.command == "init":
return cmd_init(args, logger)
if args.command == "sync":
return cmd_sync(args, logger)
if args.command == "snapshot":
return cmd_snapshot(args, logger)
if args.command == "ingest":
return cmd_ingest(args, logger)
if args.command == "backfill-followers":
return cmd_backfill_followers(args, logger)
if args.command == "discover":
reaction_board/config.py
-14
View File
@@ -68,8 +68,6 @@ DEFAULT_CONFIG: dict[str, Any] = {
"up_blacklist_seed": [],
"schedule": {
"discover_minutes": 60,
"folder_sync_minutes": 60,
"full_folder_refresh_hours": 6,
"snapshot_at": "00:01",
"candidate_ttl_days": 7,
},
@@ -151,8 +149,6 @@ class FolderConfig:
@dataclass(frozen=True)
class ScheduleConfig:
discover_minutes: int
folder_sync_minutes: int
full_folder_refresh_hours: int
snapshot_at: str
candidate_ttl_days: int
@@ -370,16 +366,6 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
discover_minutes=_as_int(
sched_raw.get("discover_minutes", 60), "schedule.discover_minutes", minimum=1
),
folder_sync_minutes=_as_int(
sched_raw.get("folder_sync_minutes", 60),
"schedule.folder_sync_minutes",
minimum=1,
),
full_folder_refresh_hours=_as_int(
sched_raw.get("full_folder_refresh_hours", 6),
"schedule.full_folder_refresh_hours",
minimum=1,
),
snapshot_at=snapshot_at,
candidate_ttl_days=_as_int(
sched_raw.get("candidate_ttl_days", 7),
reaction_board/fav_position.py
-82
View File
@@ -1,82 +0,0 @@
# -*- coding: utf-8 -*-
"""新视频在收藏夹中的插入位置计算（纯函数）。"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Mapping, Sequence
@dataclass(frozen=True)
class VideoMeta:
up_mid: int | None
pubdate: int
@dataclass(frozen=True)
class InsertionPlan:
target_position: int
moves: tuple[tuple[int, int], ...] # (pred_aid, moved_aid)
@property
def needs_head_move(self) -> bool:
return len(self.moves) == 2
def plan_insertion(
current_ids: Sequence[int],
new_aid: int,
metadata: Mapping[int, VideoMeta],
new_meta: VideoMeta,
) -> InsertionPlan:
"""按“同 UP 连续、组间按最早投稿倒序”计算插入点。
``current_ids`` 可以包含 ``new_aid``（通常在末尾），函数会先排除它。
"""
ordered = [int(aid) for aid in current_ids if int(aid) != int(new_aid)]
if not ordered:
return InsertionPlan(target_position=1, moves=())
groups: list[dict] = []
for index, aid in enumerate(ordered):
meta = metadata.get(aid)
up_mid = meta.up_mid if meta else None
if up_mid is not None and groups and groups[-1]["up_mid"] == up_mid:
groups[-1]["aids"].append(aid)
else:
groups.append({"up_mid": up_mid, "aids": [aid], "start": index})
for group in groups:
keys = [
metadata[aid].pubdate
for aid in group["aids"]
if aid in metadata and metadata[aid].pubdate > 0
]
group["key"] = min(keys) if keys else 0
insert_index: int | None = None
if new_meta.up_mid is not None:
for group in groups:
if group["up_mid"] == new_meta.up_mid:
insert_index = group["start"]
break
if insert_index is None:
for group in groups:
if group["key"] < new_meta.pubdate:
insert_index = group["start"]
break
if insert_index is None:
insert_index = len(ordered)
target_position = insert_index + 1
if insert_index == 0:
first = ordered[0]
return InsertionPlan(
target_position=1,
moves=((first, int(new_aid)), (int(new_aid), first)),
)
predecessor = ordered[insert_index - 1]
return InsertionPlan(
target_position=target_position,
moves=((predecessor, int(new_aid)),),
)
reaction_board/migrations.py
+9
View File
@@ -207,6 +207,15 @@ MIGRATIONS: tuple[Migration, ...] = (
""",
),
),
Migration(
version=5,
name="stop favorite-folder tracking",
statements=(
# 不再追踪收藏夹后，“取消收藏”状态失去语义；历史 unfaved 行归一到 active。
# 表结构与历史数据保留，仅停止写入。
"UPDATE video SET status = 'active' WHERE status = 'unfaved'",
),
),
)
reaction_board/services/favorites.py
-237
View File
@@ -1,237 +0,0 @@
# -*- coding: utf-8 -*-
"""候选收藏、忽略、拉黑动作。"""
from __future__ import annotations
import time
from dataclasses import dataclass
from typing import Any
from ..bilibili.models import FavResource
from ..config import ConfigStore
from ..fav_position import VideoMeta, plan_insertion
from ..repository import Repository
from ..timeutil import shanghai_datetime
@dataclass
class ActionResult:
ok: bool
aid: int
action: str
state: str
position: int | None = None
already_in_folder: bool = False
message: str = ""
def as_dict(self) -> dict[str, Any]:
return {
"ok": self.ok,
"aid": self.aid,
"action": self.action,
"state": self.state,
"position": self.position,
"already_in_folder": self.already_in_folder,
"message": self.message,
}
class FavoriteActionService:
def __init__(
self,
config_store: ConfigStore,
repository: Repository,
write_provider,
*,
clock=time.time,
logger=None,
) -> None:
self.config_store = config_store
self.repo = repository
self.write_provider = write_provider
self._clock = clock
self.logger = logger
def _now(self) -> int:
return int(self._clock())
def _log(self, level: str, event: str, **fields) -> None:
if self.logger is not None:
getattr(self.logger, level)(event, **fields)
def _target_folder(self, candidate: dict[str, Any]):
cfg = self.config_store.get()
up_mid = candidate.get("up_mid")
if isinstance(up_mid, int) and not isinstance(up_mid, bool):
folder_id = self.repo.folder_for_up(up_mid)
if folder_id is not None:
row = self.repo.get_folder(folder_id)
if row is not None and row.enabled:
return row
year = shanghai_datetime(self._clock()).year
folder = cfg.folder_for_year(year)
if folder is not None:
return folder
latest = cfg.latest_folder()
if latest is not None:
return latest
raise RuntimeError("没有可用的目标收藏夹，请检查 folders 配置")
def favorite(self, aid: int) -> ActionResult:
aid = int(aid)
candidate = self.repo.get_candidate(aid)
if candidate is None:
return ActionResult(
ok=False, aid=aid, action="favorite", state="missing",
message="候选不存在或已处理",
)
try:
folder = self._target_folder(candidate)
except Exception as exc: # noqa: BLE001
return ActionResult(
ok=False, aid=aid, action="favorite", state="failed",
message=str(exc),
)
token, state = self.repo.begin_fav_action(
aid, folder.media_id, now=self._now()
)
if state == "done":
action = self.repo.get_fav_action(token) or {}
return ActionResult(
ok=True, aid=aid, action="favorite", state="done",
position=action.get("position"),
message="已在收藏夹",
)
if state == "busy":
return ActionResult(
ok=False, aid=aid, action="favorite", state="busy",
message="该候选正在处理中，请稍后重试",
)
already = False
try:
client = self.write_provider.get()
ids = client.fav_ids(folder.media_id)
if aid in {item.aid for item in ids}:
already = True
position = [item.aid for item in ids].index(aid) + 1
else:
client.deal(aid, folder.media_id)
ids = client.fav_ids(folder.media_id)
aid_order = [item.aid for item in ids]
if aid not in aid_order:
raise RuntimeError("deal 成功但回读收藏夹未找到该视频")
metadata = {
item_aid: VideoMeta(
up_mid=(info.get("up_mid") if info else None),
pubdate=int((info or {}).get("pubdate") or 0),
)
for item_aid, info in self.repo.video_metadata(aid_order).items()
}
plan = plan_insertion(
aid_order,
aid,
metadata,
VideoMeta(
up_mid=candidate.get("up_mid"),
pubdate=int(candidate.get("pubdate") or 0),
),
)
type_map = {item.aid: item.type for item in ids}
for predecessor, moved in plan.moves:
client.sort(
folder.media_id,
f"{predecessor}:{type_map.get(predecessor, 2)}:"
f"{moved}:{type_map.get(moved, 2)}",
)
final_order = [item.aid for item in client.fav_ids(folder.media_id)]
if aid not in final_order:
raise RuntimeError("排序后回读收藏夹未找到该视频")
position = final_order.index(aid) + 1
if position != plan.target_position:
raise RuntimeError(
f"回读位置不符：期望 #{plan.target_position}，实际 #{position}"
)
resource = FavResource(
aid=aid,
bvid=candidate.get("bvid"),
title=str(candidate.get("title") or "(无标题)"),
up_mid=candidate.get("up_mid"),
up_name=candidate.get("up_name"),
pubdate=int(candidate.get("pubdate") or 0),
duration=None,
cover=candidate.get("cover"),
attr=0,
fav_time=None,
)
self.repo.upsert_from_fav(
resource, folder.media_id, was_added=True, now=self._now()
)
self.repo.delete_candidate(aid)
self.repo.finish_fav_action(token, position=position, now=self._now())
self._log(
"info", "favorite.done",
aid=aid, folder_id=folder.media_id, position=position,
already=already,
)
self.repo.bump_cache_version("videos")
return ActionResult(
ok=True, aid=aid, action="favorite", state="done",
position=position, already_in_folder=already,
message="已在收藏夹" if already else "已加入收藏夹",
)
except Exception as exc: # noqa: BLE001 失败保留候选，允许重试
self.repo.fail_fav_action(token, error=str(exc), now=self._now())
self._log(
"error", "favorite.failed",
aid=aid, folder_id=folder.media_id, error=str(exc),
)
return ActionResult(
ok=False, aid=aid, action="favorite", state="failed",
message=str(exc),
)
def ignore(self, aid: int) -> ActionResult:
aid = int(aid)
candidate = self.repo.get_candidate(aid)
if candidate is None:
return ActionResult(
ok=False, aid=aid, action="ignore", state="missing",
message="候选不存在或已处理",
)
self.repo.mark_candidate_decision(aid, "rejected", now=self._now())
self.repo.bump_cache_version("videos")
return ActionResult(
ok=True, aid=aid, action="ignore", state="rejected",
message="已忽略",
)
def blacklist(self, aid: int) -> ActionResult:
aid = int(aid)
candidate = self.repo.get_candidate(aid)
if candidate is None:
return ActionResult(
ok=False, aid=aid, action="blacklist", state="missing",
message="候选不存在或已处理",
)
up_mid = candidate.get("up_mid")
if not isinstance(up_mid, int) or isinstance(up_mid, bool):
return ActionResult(
ok=False, aid=aid, action="blacklist", state="failed",
message="候选缺少 UP mid，无法拉黑",
)
self.repo.add_blacklist(
up_mid,
name=candidate.get("up_name"),
reason="telegram",
now=self._now(),
)
rejected = self.repo.reject_pending_for_up(up_mid, now=self._now())
self.repo.mark_candidate_decision(aid, "rejected", now=self._now())
self.repo.bump_cache_version("videos")
return ActionResult(
ok=True, aid=aid, action="blacklist", state="rejected",
message=f"已拉黑 UP，并忽略该 UP 的 {rejected} 条待决策候选",
)
reaction_board/services/folder_sync.py
-335
View File
@@ -1,335 +0,0 @@
# -*- coding: utf-8 -*-
"""收藏夹同步（库的真相源）。
每次运行：
1. 读取每个配置年度夹的全部 aid；
2. 与本地 ``folder_resource`` 做差集；
3. 新增资源按 200 条分块拉取 ``infos`` 并入库；
4. 从收藏夹移除的资源标记 ``unfaved``（不删除历史）；
5. 每 ``full_folder_refresh_hours`` 小时全量刷新一次元数据并识别失效资源；
6. 容量接近上限时预警，但绝不自动换夹。
黑名单只在发现流程生效，本服务不读取 ``up_blacklist``。
"""
from __future__ import annotations
import time
from dataclasses import dataclass, field
from typing import Any
from ..config import AppConfig, ConfigStore, FolderConfig
from ..repository import Repository
from .up_profile import UpProfileService
CAPACITY_WARN = 950
@dataclass
class FolderSyncResult:
media_id: int
year: int
title: str
remote_count: int = 0
added: int = 0
removed: int = 0
refreshed: int = 0
missing: int = 0
full_refresh: bool = False
warning: str | None = None
error: str | None = None
def as_dict(self) -> dict[str, Any]:
return {
"media_id": self.media_id,
"year": self.year,
"title": self.title,
"remote_count": self.remote_count,
"added": self.added,
"removed": self.removed,
"refreshed": self.refreshed,
"missing": self.missing,
"full_refresh": self.full_refresh,
"warning": self.warning,
"error": self.error,
}
@dataclass
class SyncSummary:
started_at: int
finished_at: int = 0
folders: list[FolderSyncResult] = field(default_factory=list)
blacklist_seeded: int = 0
followers_updated: int = 0
followers_failed: int = 0
@property
def added(self) -> int:
return sum(f.added for f in self.folders)
@property
def removed(self) -> int:
return sum(f.removed for f in self.folders)
@property
def refreshed(self) -> int:
return sum(f.refreshed for f in self.folders)
@property
def missing(self) -> int:
return sum(f.missing for f in self.folders)
@property
def errors(self) -> list[str]:
return [f"folder {f.media_id}: {f.error}" for f in self.folders if f.error]
@property
def warnings(self) -> list[str]:
return [f"folder {f.media_id}: {f.warning}" for f in self.folders if f.warning]
@property
def ok(self) -> bool:
return not self.errors
def as_dict(self) -> dict[str, Any]:
return {
"started_at": self.started_at,
"finished_at": self.finished_at,
"elapsed": max(0, self.finished_at - self.started_at),
"added": self.added,
"removed": self.removed,
"refreshed": self.refreshed,
"missing": self.missing,
"blacklist_seeded": self.blacklist_seeded,
"followers_updated": self.followers_updated,
"followers_failed": self.followers_failed,
"folders": [f.as_dict() for f in self.folders],
"errors": self.errors,
"warnings": self.warnings,
"ok": self.ok,
}
class FolderSyncService:
def __init__(
self,
config_store: ConfigStore,
repository: Repository,
client=None,
*,
client_provider=None,
clock=time.time,
logger=None,
capacity_warn: int = CAPACITY_WARN,
allow_empty_folders: bool = False,
up_profile_service: UpProfileService | None = None,
capture_followers: bool = True,
) -> None:
if client is None and client_provider is None:
raise ValueError("必须提供 client 或 client_provider")
self.config_store = config_store
self.repo = repository
self.client = client
self.client_provider = client_provider
self._clock = clock
self.logger = logger
self.capacity_warn = capacity_warn
self.allow_empty_folders = allow_empty_folders
self.capture_followers = capture_followers
self.up_profile = up_profile_service or UpProfileService(
repository, self._client, clock=clock, logger=logger
)
def _client(self):
if self.client_provider is not None:
return self.client_provider.get()
return self.client
def _now(self) -> int:
return int(self._clock())
def _log(self, level: str, event: str, **fields) -> None:
if self.logger is not None:
getattr(self.logger, level)(event, **fields)
def run(self, *, full_refresh: bool = False) -> SyncSummary:
cfg = self.config_store.get()
client = self._client()
now = self._now()
summary = SyncSummary(started_at=now)
self._log(
"info", "folder_sync.start", folders=len(cfg.folders), full=full_refresh
)
for folder in cfg.folders:
self.repo.ensure_folder(folder, now=now)
self.repo.disable_folders_except(f.media_id for f in cfg.folders)
if cfg.up_blacklist_seed:
summary.blacklist_seeded = self.repo.seed_blacklist(
cfg.up_blacklist_seed, now=now
)
for folder in cfg.folders:
try:
result = self._sync_folder(
client, cfg, folder, now=now, force_full=full_refresh
)
except Exception as exc: # noqa: BLE001 单个夹失败不阻塞其它夹
result = FolderSyncResult(
media_id=folder.media_id,
year=folder.year,
title=folder.title,
error=str(exc),
)
self._log(
"error",
"folder_sync.folder_error",
media_id=folder.media_id,
error=str(exc),
)
else:
self._log("info", "folder_sync.folder", **result.as_dict())
if result.warning:
self._log(
"warning",
"folder_sync.capacity",
media_id=folder.media_id,
warning=result.warning,
)
summary.folders.append(result)
if self.capture_followers:
try:
missing = self.repo.up_mids_missing_follower()
if missing:
capture = self.up_profile.capture(
[int(row["up_mid"]) for row in missing],
names={
int(row["up_mid"]): str(row.get("up_name") or "")
for row in missing
},
live_since=summary.started_at - 3600,
)
summary.followers_updated = capture.updated_videos
summary.followers_failed = capture.failed
except Exception as exc: # noqa: BLE001 粉丝采集失败不影响收藏夹同步
summary.followers_failed += 1
self._log(
"warning", "folder_sync.followers_failed", error=str(exc)
)
summary.finished_at = self._now()
self._log(
"info" if summary.ok else "error",
"folder_sync.done",
added=summary.added,
removed=summary.removed,
refreshed=summary.refreshed,
missing=summary.missing,
errors=len(summary.errors),
elapsed=max(0, summary.finished_at - summary.started_at),
)
if summary.added or summary.removed or summary.refreshed or summary.followers_updated:
self.repo.bump_cache_version("videos")
return summary
def _sync_folder(
self,
client,
cfg: AppConfig,
folder: FolderConfig,
*,
now: int,
force_full: bool,
) -> FolderSyncResult:
row = self.repo.get_folder(folder.media_id)
full_due = (
force_full
or row is None
or row.last_full_at is None
or now - int(row.last_full_at)
>= cfg.schedule.full_folder_refresh_hours * 3600
)
resources = client.fav_ids(folder.media_id)
remote_map = {item.aid: item for item in resources if item.type == 2}
remote = set(remote_map)
local = self.repo.folder_active_aids(folder.media_id)
added = remote - local
removed = local - remote
if not remote and local and not self.allow_empty_folders:
raise RuntimeError(
f"收藏夹 {folder.media_id} 远端返回 0 条，但本地仍有 {len(local)} 条；"
f"疑似接口异常，已跳过差集。确认收藏夹确实被清空后，"
f"可使用 --allow-empty-folder 重试"
)
result = FolderSyncResult(
media_id=folder.media_id,
year=folder.year,
title=folder.title or (row.title if row else ""),
remote_count=len(remote),
added=len(added),
removed=len(removed),
full_refresh=full_due,
)
targets = set(added)
if full_due:
targets |= remote
if targets:
infos = client.fav_infos(
folder.media_id,
sorted(targets),
folder_mid=cfg.bilibili.read.mid,
)
for aid, resource in infos.items.items():
if aid not in remote:
continue
outcome = self.repo.upsert_from_fav(
resource, folder.media_id, was_added=aid in added, now=now
)
if aid in added:
# 人工已加入收藏夹：候选池不再保留同一条视频
self.repo.delete_candidate(aid)
if outcome.moved_folder:
self._log(
"warning",
"folder_sync.multi_folder",
aid=aid,
previous_folder_id=outcome.previous_folder_id,
folder_id=folder.media_id,
)
for aid in infos.missing:
source = remote_map.get(aid)
self.repo.mark_missing_resource(
aid,
folder_id=folder.media_id,
bvid=source.bvid if source else None,
now=now,
)
result.refreshed = len(infos.items)
result.missing = len(infos.missing)
for aid in removed:
self.repo.mark_resource_removed(folder.media_id, aid, now=now)
self.repo.touch_folder_resources(folder.media_id, now=now)
self.repo.finish_folder_sync(
folder,
media_count=len(remote),
full_refresh=full_due,
now=now,
)
if len(remote) >= self.capacity_warn:
result.warning = (
f"收藏夹「{result.title or folder.media_id}」已有 {len(remote)} 条，"
f"达到预警线 {self.capacity_warn}；请人工准备下一年度收藏夹并更新配置，"
f"系统不会自动换夹"
)
return result
reaction_board/web/queries.py
-3
View File
@@ -1611,12 +1611,10 @@ class DashboardQueries:
now = self._now()
next_runs = {
"discover": int(now + cfg.schedule.discover_minutes * 60),
"folder_sync": int(now + cfg.schedule.folder_sync_minutes * 60),
"snapshot": int(next_daily_run(now, cfg.schedule.snapshot_at)),
}
for name, key, interval_minutes in (
("discover", "discovery", cfg.schedule.discover_minutes),
("folder_sync", "folder_sync", cfg.schedule.folder_sync_minutes),
):
record = tasks.get(key)
if isinstance(record, dict) and record.get("finished_at"):
@@ -1660,7 +1658,6 @@ class DashboardQueries:
"preview": self._kv(conn, "preview:seeded") == "1",
"schedule": {
"discover_minutes": cfg.schedule.discover_minutes,
"folder_sync_minutes": cfg.schedule.folder_sync_minutes,
"snapshot_at": cfg.schedule.snapshot_at,
"candidate_ttl_days": cfg.schedule.candidate_ttl_days,
},
reaction_board/web/static/views/candidates.js
+2 -2
View File
@@ -30,7 +30,7 @@ function candidateCard(item) {
</div>
<div class="candidate-actions">
${item.url ? `<a class="button ghost" href="${escapeAttr(item.url)}" target="_blank" rel="noreferrer">打开 B 站</a>` : ""}
<span class="readonly-note">请在 Telegram 中收藏 / 忽略 / 拉黑</span>
<span class="readonly-note">请在 Telegram 中入库 / 忽略 / 拉黑</span>
</div>
</div>
</article>`;
@@ -51,7 +51,7 @@ export async function renderCandidates(root) {
${data.items.length ? `<div class="notice-row">
<strong>候选处理</strong>
<span>${data.telegram_enabled ? "请在 Telegram 消息中收藏、忽略或拉黑。" : "Telegram 未启用，请使用 CLI 或数据库处理候选。"}</span>
<span>${data.telegram_enabled ? "请在 Telegram 消息中入库、忽略或拉黑。" : "Telegram 未启用，请使用 CLI 或数据库处理候选。"}</span>
</div>` : ""}
<section class="panel">
reaction_board/web/static/views/home.js
+3 -3
View File
@@ -30,7 +30,8 @@ function heatStrip(items) {
function capacity(folders) {
if (!folders.length) return "";
return `<section class="panel capacity-panel">
${sectionHeader("年度收藏夹")}
${sectionHeader("历史收藏夹")}
<p class="section-note">系统已不再同步收藏夹；下列统计仅保留历史来源。</p>
${folders.map((folder) => `
<div class="capacity-row">
<div class="capacity-label">
@@ -39,7 +40,6 @@ function capacity(folders) {
</div>
<div class="capacity-track"><span style="width:${Math.min(100, folder.ratio * 100)}%"
class="${folder.warning ? "warning" : ""}"></span></div>
${folder.warning ? `<p class="warning-text">接近容量上限，请准备新建年度收藏夹。</p>` : ""}
</div>`).join("")}
</section>`;
}
@@ -95,7 +95,7 @@ export async function renderHome(root) {
${addedNote}
<div class="video-list">
${added.items.length ? added.items.map((item) => videoCard(item)).join("")
: emptyState("最近 7 天没有新入库视频", "收藏夹同步或搜索确认后会出现在这里。")}
: emptyState("最近 7 天没有新入库视频", "搜索发现确认或 Telegram 私聊手动入库后会出现在这里。")}
</div>
</section>
reaction_board/web/static/views/system.js
+5 -6
View File
@@ -47,7 +47,6 @@ export async function renderSystem(root) {
<div><span>采集成功</span><strong>${formatCount(health.ok)}</strong></div>
<div><span>采集失败</span><strong class="${health.failed ? "danger" : ""}">${formatCount(health.failed)}</strong></div>
<div><span>在库</span><strong>${formatCount(counts.active)}</strong></div>
<div><span>已取消收藏</span><strong>${formatCount(counts.unfaved)}</strong></div>
<div><span>已失效</span><strong>${formatCount(counts.removed)}</strong></div>
<div><span>已移出库</span><strong>${formatCount(counts.dropped)}</strong></div>
</div>
@@ -60,16 +59,16 @@ export async function renderSystem(root) {
<section class="panel">
${sectionHeader("预计下次运行")}
<div class="status-grid">
<div><span>收藏夹同步</span><strong>${escapeHtml(formatDateTime(next.folder_sync))}</strong></div>
<div><span>搜索发现</span><strong>${escapeHtml(formatDateTime(next.discover))}</strong></div>
<div><span>每日快照</span><strong>${escapeHtml(formatDateTime(next.snapshot))}</strong></div>
<div><span>调度参数</span><strong>${system.schedule.discover_minutes} / ${system.schedule.folder_sync_minutes} 分钟</strong></div>
<div><span>发现周期</span><strong>${system.schedule.discover_minutes} 分钟</strong></div>
</div>
<p class="section-note">预计时间按配置和最近任务完成时间推算；实际以常驻进程为准。</p>
</section>
<section class="panel">
${sectionHeader("收藏夹容量")}
${sectionHeader("历史收藏夹")}
<p class="section-note">系统已不再同步收藏夹；下列统计仅保留历史来源。收藏夹由人工维护，重排序使用 favsort.py。</p>
${(system.folders || []).map((folder) => `
<div class="capacity-row">
<div class="capacity-label">
@@ -78,8 +77,8 @@ export async function renderSystem(root) {
</div>
<div class="capacity-track"><span style="width:${Math.min(100, folder.ratio * 100)}%"
class="${folder.warning ? "warning" : ""}"></span></div>
<p class="section-note">最近同步 ${escapeHtml(formatDateTime(folder.last_ids_at))}</p>
</div>`).join("") || emptyState("未配置收藏夹")}
<p class="section-note">历史快照 ${escapeHtml(formatDateTime(folder.last_ids_at))}</p>
</div>`).join("") || emptyState("无历史收藏夹记录")}
</section>
<section class="panel">