refactor(sync): drop favorite-folder tracking and deal
This commit is contained in:
kairee committed
1 parent
62abbd7ea1
commit
6c4287b43a
21 files changed
+115
-1491
No files matched your search
@@ -1,5 +1,8 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""B 站收藏夹写客户端（``deal`` / ``sort``）。
|
||||
"""B 站收藏夹写客户端（``sort``）。
|
||||
|
||||
收藏夹重排序是本项目**唯一**使用 ``bilibili.write`` 凭据的写操作，且频率极低。
|
||||
加入收藏夹（``deal``）等写操作已移除，收藏夹由人工维护。
|
||||
|
||||
只应使用 ``bilibili.write`` 凭据构造。
|
||||
"""
|
||||
@@ -39,22 +42,6 @@ class BilibiliWriteClient(BilibiliClient):
|
||||
raise self._api_error(payload)
|
||||
return payload
|
||||
|
||||
def deal(self, aid: int, folder_id: int) -> dict[str, Any]:
|
||||
return self.post_form(
|
||||
f"{BASE}/x/v3/fav/resource/deal",
|
||||
{
|
||||
"rid": int(aid),
|
||||
"type": 2,
|
||||
"add_media_ids": int(folder_id),
|
||||
"del_media_ids": "",
|
||||
"csrf": self.csrf,
|
||||
},
|
||||
referer=(
|
||||
f"https://space.bilibili.com/{self.mid}/favlist"
|
||||
f"?fid={int(folder_id)}&ftype=create"
|
||||
),
|
||||
)
|
||||
|
||||
def sort(self, media_id: int, sort_spec: str) -> dict[str, Any]:
|
||||
return self.post_form(
|
||||
f"{BASE}/x/v3/fav/resource/sort",
|
||||
|
||||
+40
-96
@@ -2,7 +2,7 @@
|
||||
"""看板命令行入口。
|
||||
|
||||
python app.py init # 建库 / 迁移 / 写入黑名单种子（离线）
|
||||
python app.py sync # 收藏夹只读同步（首次导入或增量）
|
||||
python app.py ingest # 手动入库 B 站视频链接 / BV 号 / av 号（只写本地库）
|
||||
python app.py snapshot # 采集当日七项快照与粉丝数
|
||||
python app.py discover # 执行一次搜索发现并写入候选池
|
||||
python app.py candidates # 查看候选池（离线）
|
||||
@@ -10,6 +10,7 @@
|
||||
python app.py run # 按配置周期常驻运行（--once 只跑一次）
|
||||
python app.py status # 查看库内状态（离线）
|
||||
|
||||
收藏夹重排序使用独立的 ``favsort.py``（唯一使用 ``bilibili.write`` 的功能）。
|
||||
P2 起 ``python app.py`` 将在此基础上启动本地看板服务。
|
||||
"""
|
||||
|
||||
@@ -27,7 +28,6 @@ from .bilibili.auth_flow import AuthService
|
||||
from .bilibili.client import BilibiliClient, Pacer
|
||||
from .bilibili.provider import ClientProvider
|
||||
from .bilibili.search import SearchClient
|
||||
from .bilibili.write_client import BilibiliWriteClient
|
||||
from .config import ConfigError, ConfigStore
|
||||
from .database import Database
|
||||
from .logging_json import JsonLogger
|
||||
@@ -35,8 +35,7 @@ from .notify.telegram import TelegramClient
|
||||
from .repository import Repository
|
||||
from .scheduler import ScheduledTask, Scheduler
|
||||
from .services.discovery import DiscoveryService
|
||||
from .services.favorites import FavoriteActionService
|
||||
from .services.folder_sync import FolderSyncService
|
||||
from .services.library import LibraryService
|
||||
from .services.snapshot import SnapshotService
|
||||
from .services.telegram_poller import TelegramPoller
|
||||
from .services.up_profile import UpProfileService
|
||||
@@ -66,20 +65,6 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
sub = parser.add_subparsers(dest="command", required=False)
|
||||
|
||||
sub.add_parser("init", parents=[common], help="建库、迁移并写入黑名单种子")
|
||||
sync = sub.add_parser("sync", parents=[common], help="同步已配置的年度收藏夹")
|
||||
sync.add_argument(
|
||||
"--full", action="store_true", help="强制全量刷新元数据（忽略 6 小时周期）"
|
||||
)
|
||||
sync.add_argument(
|
||||
"--allow-empty-folder",
|
||||
action="store_true",
|
||||
help="允许把远端 0 条视为真实清空（默认按异常跳过差集）",
|
||||
)
|
||||
sync.add_argument(
|
||||
"--no-capture-followers",
|
||||
action="store_true",
|
||||
help="跳过同步末尾的 UP 粉丝采集（改用 backfill-followers 单独回填）",
|
||||
)
|
||||
snapshot = sub.add_parser(
|
||||
"snapshot", parents=[common], help="采集当日七项快照与粉丝数"
|
||||
)
|
||||
@@ -103,6 +88,14 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
default=None,
|
||||
help="只查看指定决策状态的候选",
|
||||
)
|
||||
ingest = sub.add_parser(
|
||||
"ingest", parents=[common],
|
||||
help="手动入库：B 站视频链接、BV 号或 av 号（只写本地库）",
|
||||
)
|
||||
ingest.add_argument(
|
||||
"refs", nargs="+", metavar="REF",
|
||||
help="视频链接、BV 号（BV…）、av 号（av12345）或含它们的字符串",
|
||||
)
|
||||
auth = sub.add_parser("auth", parents=[common], help="扫码授权并更新指定账号 Cookie")
|
||||
auth.add_argument("--account", choices=("read", "write"), default="read",
|
||||
help="要授权的账号角色（默认 read）")
|
||||
@@ -112,15 +105,10 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
auth_check = sub.add_parser("auth-check", parents=[common], help="检查账号登录态")
|
||||
auth_check.add_argument("--account", choices=("read", "write", "all"), default="read",
|
||||
help="要检查的账号角色（默认 read）")
|
||||
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行同步与快照")
|
||||
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行发现与快照")
|
||||
run.add_argument(
|
||||
"--once", action="store_true", help="只执行一次到期任务后退出"
|
||||
)
|
||||
run.add_argument(
|
||||
"--allow-empty-folder",
|
||||
action="store_true",
|
||||
help="允许把远端 0 条视为真实清空（默认按异常跳过差集）",
|
||||
)
|
||||
serve = sub.add_parser("serve", parents=[common], help="启动只读本地看板（不运行调度）")
|
||||
serve.add_argument(
|
||||
"--preview", action="store_true",
|
||||
@@ -209,32 +197,30 @@ def cmd_init(args, logger: JsonLogger) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_sync(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=True)
|
||||
def cmd_ingest(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=False)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
client = _client(cfg.bilibili.read, logger)
|
||||
service = FolderSyncService(
|
||||
store,
|
||||
repo,
|
||||
client,
|
||||
logger=logger,
|
||||
allow_empty_folders=args.allow_empty_folder,
|
||||
capture_followers=not getattr(args, "no_capture_followers", False),
|
||||
service = LibraryService(
|
||||
store, repo, lambda: client, logger=logger
|
||||
)
|
||||
started = int(time.time())
|
||||
summary = service.run(full_refresh=args.full)
|
||||
payload = summary.as_dict()
|
||||
text = " ".join(args.refs)
|
||||
result = service.ingest_text(text)
|
||||
payload = result.as_dict()
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
payload["counts"] = repo.counts()
|
||||
_record_task(
|
||||
repo, "folder_sync", ok=summary.ok, started_at=started,
|
||||
finished_at=int(time.time()), payload=payload,
|
||||
repo, "ingest", ok=bool(result.results) and not any(
|
||||
item.state == "error" for item in result.results
|
||||
),
|
||||
payload=payload,
|
||||
)
|
||||
_print(payload)
|
||||
return 0 if summary.ok else 1
|
||||
return 0 if result.results and not any(
|
||||
item.state == "error" for item in result.results
|
||||
) else 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -403,7 +389,9 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
web_server = None
|
||||
try:
|
||||
repo = Repository(db)
|
||||
folder_provider = ClientProvider(
|
||||
if cfg.up_blacklist_seed:
|
||||
repo.seed_blacklist(cfg.up_blacklist_seed)
|
||||
read_provider = ClientProvider(
|
||||
store, "read", lambda credentials: _client(credentials, logger)
|
||||
)
|
||||
snapshot_provider = ClientProvider(
|
||||
@@ -413,34 +401,22 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
credentials, logger, pacer=Pacer(1.0, 1.6), retries=1
|
||||
),
|
||||
)
|
||||
folder_service = FolderSyncService(
|
||||
store,
|
||||
repo,
|
||||
client_provider=folder_provider,
|
||||
logger=logger,
|
||||
allow_empty_folders=args.allow_empty_folder,
|
||||
)
|
||||
snapshot_service = SnapshotService(
|
||||
store, repo, client_provider=snapshot_provider, logger=logger
|
||||
)
|
||||
write_provider = ClientProvider(
|
||||
up_profile_service = UpProfileService(repo, read_provider.get, logger=logger)
|
||||
library_service = LibraryService(
|
||||
store,
|
||||
"write",
|
||||
lambda credentials: BilibiliWriteClient(
|
||||
cookie=credentials.cookie,
|
||||
csrf=credentials.csrf,
|
||||
mid=credentials.mid,
|
||||
logger=logger,
|
||||
),
|
||||
)
|
||||
favorite_service = FavoriteActionService(
|
||||
store, repo, write_provider, logger=logger
|
||||
repo,
|
||||
read_provider,
|
||||
up_profile_service=up_profile_service,
|
||||
logger=logger,
|
||||
)
|
||||
notifier = _telegram(cfg, logger)
|
||||
discovery_service = DiscoveryService(
|
||||
store,
|
||||
repo,
|
||||
SearchClient(folder_provider.get),
|
||||
SearchClient(read_provider.get),
|
||||
notifier=notifier,
|
||||
logger=logger,
|
||||
)
|
||||
@@ -513,29 +489,6 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
logger.info("snapshot.summary", **payload)
|
||||
return summary
|
||||
|
||||
def run_sync():
|
||||
started = int(time.time())
|
||||
try:
|
||||
summary = folder_service.run()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_record_task(
|
||||
repo, "folder_sync", ok=False, started_at=started,
|
||||
finished_at=int(time.time()), error=str(exc),
|
||||
)
|
||||
raise
|
||||
payload = summary.as_dict()
|
||||
_record_task(
|
||||
repo, "folder_sync", ok=summary.ok, started_at=started,
|
||||
finished_at=int(time.time()), payload=payload,
|
||||
error=None if summary.ok else "；".join(summary.errors),
|
||||
)
|
||||
last["folder"] = summary
|
||||
logger.info("folder_sync.summary", **payload)
|
||||
# 新入库/恢复的视频立即补当天快照，而不是等到次日。
|
||||
if summary.added > 0:
|
||||
run_snapshot()
|
||||
return summary
|
||||
|
||||
def run_config_watch():
|
||||
before = store.revision
|
||||
store.get()
|
||||
@@ -547,11 +500,6 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
snapshot_at = f"{hh:02d}:{mm:02d}"
|
||||
scheduler = Scheduler(
|
||||
[
|
||||
ScheduledTask(
|
||||
"folder_sync",
|
||||
float(cfg.schedule.folder_sync_minutes * 60),
|
||||
run_sync,
|
||||
),
|
||||
ScheduledTask(
|
||||
"snapshot",
|
||||
86400.0,
|
||||
@@ -579,7 +527,7 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
daemon=True,
|
||||
).start()
|
||||
if notifier is not None and not args.once:
|
||||
poller = TelegramPoller(notifier, favorite_service, logger=logger)
|
||||
poller = TelegramPoller(notifier, library_service, logger=logger)
|
||||
threading.Thread(
|
||||
target=poller.run_forever,
|
||||
name="telegram-poller",
|
||||
@@ -592,18 +540,15 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
else:
|
||||
logger.info(
|
||||
"scheduler.start",
|
||||
folder_sync_seconds=cfg.schedule.folder_sync_minutes * 60,
|
||||
snapshot_at=snapshot_at,
|
||||
)
|
||||
scheduler.run_forever()
|
||||
|
||||
folder_summary = last.get("folder")
|
||||
snapshot_summary = last.get("snapshot")
|
||||
discover_summary = last.get("discover")
|
||||
folder_ok = folder_summary is None or getattr(folder_summary, "ok", True)
|
||||
snapshot_ok = snapshot_summary is None or getattr(snapshot_summary, "completed", True)
|
||||
discover_ok = discover_summary is None or not getattr(discover_summary, "errors", [])
|
||||
return 0 if folder_ok and snapshot_ok and discover_ok else 1
|
||||
return 0 if snapshot_ok and discover_ok else 1
|
||||
finally:
|
||||
if web_server is not None:
|
||||
try:
|
||||
@@ -703,16 +648,15 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
# 无子命令时启动完整服务：只读 Web + 调度 + Telegram。
|
||||
args.command = "run"
|
||||
args.once = False
|
||||
args.allow_empty_folder = False
|
||||
return cmd_run(args, logger, serve_web=True)
|
||||
if args.command == "serve":
|
||||
return cmd_serve(args, logger)
|
||||
if args.command == "init":
|
||||
return cmd_init(args, logger)
|
||||
if args.command == "sync":
|
||||
return cmd_sync(args, logger)
|
||||
if args.command == "snapshot":
|
||||
return cmd_snapshot(args, logger)
|
||||
if args.command == "ingest":
|
||||
return cmd_ingest(args, logger)
|
||||
if args.command == "backfill-followers":
|
||||
return cmd_backfill_followers(args, logger)
|
||||
if args.command == "discover":
|
||||
|
||||
@@ -68,8 +68,6 @@ DEFAULT_CONFIG: dict[str, Any] = {
|
||||
"up_blacklist_seed": [],
|
||||
"schedule": {
|
||||
"discover_minutes": 60,
|
||||
"folder_sync_minutes": 60,
|
||||
"full_folder_refresh_hours": 6,
|
||||
"snapshot_at": "00:01",
|
||||
"candidate_ttl_days": 7,
|
||||
},
|
||||
@@ -151,8 +149,6 @@ class FolderConfig:
|
||||
@dataclass(frozen=True)
|
||||
class ScheduleConfig:
|
||||
discover_minutes: int
|
||||
folder_sync_minutes: int
|
||||
full_folder_refresh_hours: int
|
||||
snapshot_at: str
|
||||
candidate_ttl_days: int
|
||||
|
||||
@@ -370,16 +366,6 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
||||
discover_minutes=_as_int(
|
||||
sched_raw.get("discover_minutes", 60), "schedule.discover_minutes", minimum=1
|
||||
),
|
||||
folder_sync_minutes=_as_int(
|
||||
sched_raw.get("folder_sync_minutes", 60),
|
||||
"schedule.folder_sync_minutes",
|
||||
minimum=1,
|
||||
),
|
||||
full_folder_refresh_hours=_as_int(
|
||||
sched_raw.get("full_folder_refresh_hours", 6),
|
||||
"schedule.full_folder_refresh_hours",
|
||||
minimum=1,
|
||||
),
|
||||
snapshot_at=snapshot_at,
|
||||
candidate_ttl_days=_as_int(
|
||||
sched_raw.get("candidate_ttl_days", 7),
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""新视频在收藏夹中的插入位置计算（纯函数）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Mapping, Sequence
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VideoMeta:
|
||||
up_mid: int | None
|
||||
pubdate: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InsertionPlan:
|
||||
target_position: int
|
||||
moves: tuple[tuple[int, int], ...] # (pred_aid, moved_aid)
|
||||
|
||||
@property
|
||||
def needs_head_move(self) -> bool:
|
||||
return len(self.moves) == 2
|
||||
|
||||
|
||||
def plan_insertion(
|
||||
current_ids: Sequence[int],
|
||||
new_aid: int,
|
||||
metadata: Mapping[int, VideoMeta],
|
||||
new_meta: VideoMeta,
|
||||
) -> InsertionPlan:
|
||||
"""按“同 UP 连续、组间按最早投稿倒序”计算插入点。
|
||||
|
||||
``current_ids`` 可以包含 ``new_aid``（通常在末尾），函数会先排除它。
|
||||
"""
|
||||
ordered = [int(aid) for aid in current_ids if int(aid) != int(new_aid)]
|
||||
if not ordered:
|
||||
return InsertionPlan(target_position=1, moves=())
|
||||
|
||||
groups: list[dict] = []
|
||||
for index, aid in enumerate(ordered):
|
||||
meta = metadata.get(aid)
|
||||
up_mid = meta.up_mid if meta else None
|
||||
if up_mid is not None and groups and groups[-1]["up_mid"] == up_mid:
|
||||
groups[-1]["aids"].append(aid)
|
||||
else:
|
||||
groups.append({"up_mid": up_mid, "aids": [aid], "start": index})
|
||||
|
||||
for group in groups:
|
||||
keys = [
|
||||
metadata[aid].pubdate
|
||||
for aid in group["aids"]
|
||||
if aid in metadata and metadata[aid].pubdate > 0
|
||||
]
|
||||
group["key"] = min(keys) if keys else 0
|
||||
|
||||
insert_index: int | None = None
|
||||
if new_meta.up_mid is not None:
|
||||
for group in groups:
|
||||
if group["up_mid"] == new_meta.up_mid:
|
||||
insert_index = group["start"]
|
||||
break
|
||||
if insert_index is None:
|
||||
for group in groups:
|
||||
if group["key"] < new_meta.pubdate:
|
||||
insert_index = group["start"]
|
||||
break
|
||||
if insert_index is None:
|
||||
insert_index = len(ordered)
|
||||
|
||||
target_position = insert_index + 1
|
||||
if insert_index == 0:
|
||||
first = ordered[0]
|
||||
return InsertionPlan(
|
||||
target_position=1,
|
||||
moves=((first, int(new_aid)), (int(new_aid), first)),
|
||||
)
|
||||
predecessor = ordered[insert_index - 1]
|
||||
return InsertionPlan(
|
||||
target_position=target_position,
|
||||
moves=((predecessor, int(new_aid)),),
|
||||
)
|
||||
@@ -207,6 +207,15 @@ MIGRATIONS: tuple[Migration, ...] = (
|
||||
""",
|
||||
),
|
||||
),
|
||||
Migration(
|
||||
version=5,
|
||||
name="stop favorite-folder tracking",
|
||||
statements=(
|
||||
# 不再追踪收藏夹后，“取消收藏”状态失去语义；历史 unfaved 行归一到 active。
|
||||
# 表结构与历史数据保留，仅停止写入。
|
||||
"UPDATE video SET status = 'active' WHERE status = 'unfaved'",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,237 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""候选收藏、忽略、拉黑动作。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from ..bilibili.models import FavResource
|
||||
from ..config import ConfigStore
|
||||
from ..fav_position import VideoMeta, plan_insertion
|
||||
from ..repository import Repository
|
||||
from ..timeutil import shanghai_datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActionResult:
|
||||
ok: bool
|
||||
aid: int
|
||||
action: str
|
||||
state: str
|
||||
position: int | None = None
|
||||
already_in_folder: bool = False
|
||||
message: str = ""
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"ok": self.ok,
|
||||
"aid": self.aid,
|
||||
"action": self.action,
|
||||
"state": self.state,
|
||||
"position": self.position,
|
||||
"already_in_folder": self.already_in_folder,
|
||||
"message": self.message,
|
||||
}
|
||||
|
||||
|
||||
class FavoriteActionService:
|
||||
def __init__(
|
||||
self,
|
||||
config_store: ConfigStore,
|
||||
repository: Repository,
|
||||
write_provider,
|
||||
*,
|
||||
clock=time.time,
|
||||
logger=None,
|
||||
) -> None:
|
||||
self.config_store = config_store
|
||||
self.repo = repository
|
||||
self.write_provider = write_provider
|
||||
self._clock = clock
|
||||
self.logger = logger
|
||||
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
|
||||
def _log(self, level: str, event: str, **fields) -> None:
|
||||
if self.logger is not None:
|
||||
getattr(self.logger, level)(event, **fields)
|
||||
|
||||
def _target_folder(self, candidate: dict[str, Any]):
|
||||
cfg = self.config_store.get()
|
||||
up_mid = candidate.get("up_mid")
|
||||
if isinstance(up_mid, int) and not isinstance(up_mid, bool):
|
||||
folder_id = self.repo.folder_for_up(up_mid)
|
||||
if folder_id is not None:
|
||||
row = self.repo.get_folder(folder_id)
|
||||
if row is not None and row.enabled:
|
||||
return row
|
||||
year = shanghai_datetime(self._clock()).year
|
||||
folder = cfg.folder_for_year(year)
|
||||
if folder is not None:
|
||||
return folder
|
||||
latest = cfg.latest_folder()
|
||||
if latest is not None:
|
||||
return latest
|
||||
raise RuntimeError("没有可用的目标收藏夹，请检查 folders 配置")
|
||||
|
||||
def favorite(self, aid: int) -> ActionResult:
|
||||
aid = int(aid)
|
||||
candidate = self.repo.get_candidate(aid)
|
||||
if candidate is None:
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="favorite", state="missing",
|
||||
message="候选不存在或已处理",
|
||||
)
|
||||
try:
|
||||
folder = self._target_folder(candidate)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="favorite", state="failed",
|
||||
message=str(exc),
|
||||
)
|
||||
|
||||
token, state = self.repo.begin_fav_action(
|
||||
aid, folder.media_id, now=self._now()
|
||||
)
|
||||
if state == "done":
|
||||
action = self.repo.get_fav_action(token) or {}
|
||||
return ActionResult(
|
||||
ok=True, aid=aid, action="favorite", state="done",
|
||||
position=action.get("position"),
|
||||
message="已在收藏夹",
|
||||
)
|
||||
if state == "busy":
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="favorite", state="busy",
|
||||
message="该候选正在处理中，请稍后重试",
|
||||
)
|
||||
|
||||
already = False
|
||||
try:
|
||||
client = self.write_provider.get()
|
||||
ids = client.fav_ids(folder.media_id)
|
||||
if aid in {item.aid for item in ids}:
|
||||
already = True
|
||||
position = [item.aid for item in ids].index(aid) + 1
|
||||
else:
|
||||
client.deal(aid, folder.media_id)
|
||||
ids = client.fav_ids(folder.media_id)
|
||||
aid_order = [item.aid for item in ids]
|
||||
if aid not in aid_order:
|
||||
raise RuntimeError("deal 成功但回读收藏夹未找到该视频")
|
||||
metadata = {
|
||||
item_aid: VideoMeta(
|
||||
up_mid=(info.get("up_mid") if info else None),
|
||||
pubdate=int((info or {}).get("pubdate") or 0),
|
||||
)
|
||||
for item_aid, info in self.repo.video_metadata(aid_order).items()
|
||||
}
|
||||
plan = plan_insertion(
|
||||
aid_order,
|
||||
aid,
|
||||
metadata,
|
||||
VideoMeta(
|
||||
up_mid=candidate.get("up_mid"),
|
||||
pubdate=int(candidate.get("pubdate") or 0),
|
||||
),
|
||||
)
|
||||
type_map = {item.aid: item.type for item in ids}
|
||||
for predecessor, moved in plan.moves:
|
||||
client.sort(
|
||||
folder.media_id,
|
||||
f"{predecessor}:{type_map.get(predecessor, 2)}:"
|
||||
f"{moved}:{type_map.get(moved, 2)}",
|
||||
)
|
||||
final_order = [item.aid for item in client.fav_ids(folder.media_id)]
|
||||
if aid not in final_order:
|
||||
raise RuntimeError("排序后回读收藏夹未找到该视频")
|
||||
position = final_order.index(aid) + 1
|
||||
if position != plan.target_position:
|
||||
raise RuntimeError(
|
||||
f"回读位置不符：期望 #{plan.target_position}，实际 #{position}"
|
||||
)
|
||||
|
||||
resource = FavResource(
|
||||
aid=aid,
|
||||
bvid=candidate.get("bvid"),
|
||||
title=str(candidate.get("title") or "(无标题)"),
|
||||
up_mid=candidate.get("up_mid"),
|
||||
up_name=candidate.get("up_name"),
|
||||
pubdate=int(candidate.get("pubdate") or 0),
|
||||
duration=None,
|
||||
cover=candidate.get("cover"),
|
||||
attr=0,
|
||||
fav_time=None,
|
||||
)
|
||||
self.repo.upsert_from_fav(
|
||||
resource, folder.media_id, was_added=True, now=self._now()
|
||||
)
|
||||
self.repo.delete_candidate(aid)
|
||||
self.repo.finish_fav_action(token, position=position, now=self._now())
|
||||
self._log(
|
||||
"info", "favorite.done",
|
||||
aid=aid, folder_id=folder.media_id, position=position,
|
||||
already=already,
|
||||
)
|
||||
self.repo.bump_cache_version("videos")
|
||||
return ActionResult(
|
||||
ok=True, aid=aid, action="favorite", state="done",
|
||||
position=position, already_in_folder=already,
|
||||
message="已在收藏夹" if already else "已加入收藏夹",
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 失败保留候选，允许重试
|
||||
self.repo.fail_fav_action(token, error=str(exc), now=self._now())
|
||||
self._log(
|
||||
"error", "favorite.failed",
|
||||
aid=aid, folder_id=folder.media_id, error=str(exc),
|
||||
)
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="favorite", state="failed",
|
||||
message=str(exc),
|
||||
)
|
||||
|
||||
def ignore(self, aid: int) -> ActionResult:
|
||||
aid = int(aid)
|
||||
candidate = self.repo.get_candidate(aid)
|
||||
if candidate is None:
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="ignore", state="missing",
|
||||
message="候选不存在或已处理",
|
||||
)
|
||||
self.repo.mark_candidate_decision(aid, "rejected", now=self._now())
|
||||
self.repo.bump_cache_version("videos")
|
||||
return ActionResult(
|
||||
ok=True, aid=aid, action="ignore", state="rejected",
|
||||
message="已忽略",
|
||||
)
|
||||
|
||||
def blacklist(self, aid: int) -> ActionResult:
|
||||
aid = int(aid)
|
||||
candidate = self.repo.get_candidate(aid)
|
||||
if candidate is None:
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="blacklist", state="missing",
|
||||
message="候选不存在或已处理",
|
||||
)
|
||||
up_mid = candidate.get("up_mid")
|
||||
if not isinstance(up_mid, int) or isinstance(up_mid, bool):
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="blacklist", state="failed",
|
||||
message="候选缺少 UP mid，无法拉黑",
|
||||
)
|
||||
self.repo.add_blacklist(
|
||||
up_mid,
|
||||
name=candidate.get("up_name"),
|
||||
reason="telegram",
|
||||
now=self._now(),
|
||||
)
|
||||
rejected = self.repo.reject_pending_for_up(up_mid, now=self._now())
|
||||
self.repo.mark_candidate_decision(aid, "rejected", now=self._now())
|
||||
self.repo.bump_cache_version("videos")
|
||||
return ActionResult(
|
||||
ok=True, aid=aid, action="blacklist", state="rejected",
|
||||
message=f"已拉黑 UP，并忽略该 UP 的 {rejected} 条待决策候选",
|
||||
)
|
||||
@@ -1,335 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""收藏夹同步（库的真相源）。
|
||||
|
||||
每次运行：
|
||||
|
||||
1. 读取每个配置年度夹的全部 aid；
|
||||
2. 与本地 ``folder_resource`` 做差集；
|
||||
3. 新增资源按 200 条分块拉取 ``infos`` 并入库；
|
||||
4. 从收藏夹移除的资源标记 ``unfaved``（不删除历史）；
|
||||
5. 每 ``full_folder_refresh_hours`` 小时全量刷新一次元数据并识别失效资源；
|
||||
6. 容量接近上限时预警，但绝不自动换夹。
|
||||
|
||||
黑名单只在发现流程生效，本服务不读取 ``up_blacklist``。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from ..config import AppConfig, ConfigStore, FolderConfig
|
||||
from ..repository import Repository
|
||||
from .up_profile import UpProfileService
|
||||
|
||||
CAPACITY_WARN = 950
|
||||
|
||||
|
||||
@dataclass
|
||||
class FolderSyncResult:
|
||||
media_id: int
|
||||
year: int
|
||||
title: str
|
||||
remote_count: int = 0
|
||||
added: int = 0
|
||||
removed: int = 0
|
||||
refreshed: int = 0
|
||||
missing: int = 0
|
||||
full_refresh: bool = False
|
||||
warning: str | None = None
|
||||
error: str | None = None
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"media_id": self.media_id,
|
||||
"year": self.year,
|
||||
"title": self.title,
|
||||
"remote_count": self.remote_count,
|
||||
"added": self.added,
|
||||
"removed": self.removed,
|
||||
"refreshed": self.refreshed,
|
||||
"missing": self.missing,
|
||||
"full_refresh": self.full_refresh,
|
||||
"warning": self.warning,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncSummary:
|
||||
started_at: int
|
||||
finished_at: int = 0
|
||||
folders: list[FolderSyncResult] = field(default_factory=list)
|
||||
blacklist_seeded: int = 0
|
||||
followers_updated: int = 0
|
||||
followers_failed: int = 0
|
||||
|
||||
@property
|
||||
def added(self) -> int:
|
||||
return sum(f.added for f in self.folders)
|
||||
|
||||
@property
|
||||
def removed(self) -> int:
|
||||
return sum(f.removed for f in self.folders)
|
||||
|
||||
@property
|
||||
def refreshed(self) -> int:
|
||||
return sum(f.refreshed for f in self.folders)
|
||||
|
||||
@property
|
||||
def missing(self) -> int:
|
||||
return sum(f.missing for f in self.folders)
|
||||
|
||||
@property
|
||||
def errors(self) -> list[str]:
|
||||
return [f"folder {f.media_id}: {f.error}" for f in self.folders if f.error]
|
||||
|
||||
@property
|
||||
def warnings(self) -> list[str]:
|
||||
return [f"folder {f.media_id}: {f.warning}" for f in self.folders if f.warning]
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return not self.errors
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"elapsed": max(0, self.finished_at - self.started_at),
|
||||
"added": self.added,
|
||||
"removed": self.removed,
|
||||
"refreshed": self.refreshed,
|
||||
"missing": self.missing,
|
||||
"blacklist_seeded": self.blacklist_seeded,
|
||||
"followers_updated": self.followers_updated,
|
||||
"followers_failed": self.followers_failed,
|
||||
"folders": [f.as_dict() for f in self.folders],
|
||||
"errors": self.errors,
|
||||
"warnings": self.warnings,
|
||||
"ok": self.ok,
|
||||
}
|
||||
|
||||
|
||||
class FolderSyncService:
|
||||
def __init__(
|
||||
self,
|
||||
config_store: ConfigStore,
|
||||
repository: Repository,
|
||||
client=None,
|
||||
*,
|
||||
client_provider=None,
|
||||
clock=time.time,
|
||||
logger=None,
|
||||
capacity_warn: int = CAPACITY_WARN,
|
||||
allow_empty_folders: bool = False,
|
||||
up_profile_service: UpProfileService | None = None,
|
||||
capture_followers: bool = True,
|
||||
) -> None:
|
||||
if client is None and client_provider is None:
|
||||
raise ValueError("必须提供 client 或 client_provider")
|
||||
self.config_store = config_store
|
||||
self.repo = repository
|
||||
self.client = client
|
||||
self.client_provider = client_provider
|
||||
self._clock = clock
|
||||
self.logger = logger
|
||||
self.capacity_warn = capacity_warn
|
||||
self.allow_empty_folders = allow_empty_folders
|
||||
self.capture_followers = capture_followers
|
||||
self.up_profile = up_profile_service or UpProfileService(
|
||||
repository, self._client, clock=clock, logger=logger
|
||||
)
|
||||
|
||||
def _client(self):
|
||||
if self.client_provider is not None:
|
||||
return self.client_provider.get()
|
||||
return self.client
|
||||
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
|
||||
def _log(self, level: str, event: str, **fields) -> None:
|
||||
if self.logger is not None:
|
||||
getattr(self.logger, level)(event, **fields)
|
||||
|
||||
def run(self, *, full_refresh: bool = False) -> SyncSummary:
|
||||
cfg = self.config_store.get()
|
||||
client = self._client()
|
||||
now = self._now()
|
||||
summary = SyncSummary(started_at=now)
|
||||
self._log(
|
||||
"info", "folder_sync.start", folders=len(cfg.folders), full=full_refresh
|
||||
)
|
||||
|
||||
for folder in cfg.folders:
|
||||
self.repo.ensure_folder(folder, now=now)
|
||||
self.repo.disable_folders_except(f.media_id for f in cfg.folders)
|
||||
if cfg.up_blacklist_seed:
|
||||
summary.blacklist_seeded = self.repo.seed_blacklist(
|
||||
cfg.up_blacklist_seed, now=now
|
||||
)
|
||||
|
||||
for folder in cfg.folders:
|
||||
try:
|
||||
result = self._sync_folder(
|
||||
client, cfg, folder, now=now, force_full=full_refresh
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 单个夹失败不阻塞其它夹
|
||||
result = FolderSyncResult(
|
||||
media_id=folder.media_id,
|
||||
year=folder.year,
|
||||
title=folder.title,
|
||||
error=str(exc),
|
||||
)
|
||||
self._log(
|
||||
"error",
|
||||
"folder_sync.folder_error",
|
||||
media_id=folder.media_id,
|
||||
error=str(exc),
|
||||
)
|
||||
else:
|
||||
self._log("info", "folder_sync.folder", **result.as_dict())
|
||||
if result.warning:
|
||||
self._log(
|
||||
"warning",
|
||||
"folder_sync.capacity",
|
||||
media_id=folder.media_id,
|
||||
warning=result.warning,
|
||||
)
|
||||
summary.folders.append(result)
|
||||
|
||||
if self.capture_followers:
|
||||
try:
|
||||
missing = self.repo.up_mids_missing_follower()
|
||||
if missing:
|
||||
capture = self.up_profile.capture(
|
||||
[int(row["up_mid"]) for row in missing],
|
||||
names={
|
||||
int(row["up_mid"]): str(row.get("up_name") or "")
|
||||
for row in missing
|
||||
},
|
||||
live_since=summary.started_at - 3600,
|
||||
)
|
||||
summary.followers_updated = capture.updated_videos
|
||||
summary.followers_failed = capture.failed
|
||||
except Exception as exc: # noqa: BLE001 粉丝采集失败不影响收藏夹同步
|
||||
summary.followers_failed += 1
|
||||
self._log(
|
||||
"warning", "folder_sync.followers_failed", error=str(exc)
|
||||
)
|
||||
|
||||
summary.finished_at = self._now()
|
||||
self._log(
|
||||
"info" if summary.ok else "error",
|
||||
"folder_sync.done",
|
||||
added=summary.added,
|
||||
removed=summary.removed,
|
||||
refreshed=summary.refreshed,
|
||||
missing=summary.missing,
|
||||
errors=len(summary.errors),
|
||||
elapsed=max(0, summary.finished_at - summary.started_at),
|
||||
)
|
||||
if summary.added or summary.removed or summary.refreshed or summary.followers_updated:
|
||||
self.repo.bump_cache_version("videos")
|
||||
return summary
|
||||
|
||||
def _sync_folder(
|
||||
self,
|
||||
client,
|
||||
cfg: AppConfig,
|
||||
folder: FolderConfig,
|
||||
*,
|
||||
now: int,
|
||||
force_full: bool,
|
||||
) -> FolderSyncResult:
|
||||
row = self.repo.get_folder(folder.media_id)
|
||||
full_due = (
|
||||
force_full
|
||||
or row is None
|
||||
or row.last_full_at is None
|
||||
or now - int(row.last_full_at)
|
||||
>= cfg.schedule.full_folder_refresh_hours * 3600
|
||||
)
|
||||
|
||||
resources = client.fav_ids(folder.media_id)
|
||||
remote_map = {item.aid: item for item in resources if item.type == 2}
|
||||
remote = set(remote_map)
|
||||
local = self.repo.folder_active_aids(folder.media_id)
|
||||
added = remote - local
|
||||
removed = local - remote
|
||||
|
||||
if not remote and local and not self.allow_empty_folders:
|
||||
raise RuntimeError(
|
||||
f"收藏夹 {folder.media_id} 远端返回 0 条，但本地仍有 {len(local)} 条；"
|
||||
f"疑似接口异常，已跳过差集。确认收藏夹确实被清空后，"
|
||||
f"可使用 --allow-empty-folder 重试"
|
||||
)
|
||||
|
||||
result = FolderSyncResult(
|
||||
media_id=folder.media_id,
|
||||
year=folder.year,
|
||||
title=folder.title or (row.title if row else ""),
|
||||
remote_count=len(remote),
|
||||
added=len(added),
|
||||
removed=len(removed),
|
||||
full_refresh=full_due,
|
||||
)
|
||||
|
||||
targets = set(added)
|
||||
if full_due:
|
||||
targets |= remote
|
||||
|
||||
if targets:
|
||||
infos = client.fav_infos(
|
||||
folder.media_id,
|
||||
sorted(targets),
|
||||
folder_mid=cfg.bilibili.read.mid,
|
||||
)
|
||||
for aid, resource in infos.items.items():
|
||||
if aid not in remote:
|
||||
continue
|
||||
outcome = self.repo.upsert_from_fav(
|
||||
resource, folder.media_id, was_added=aid in added, now=now
|
||||
)
|
||||
if aid in added:
|
||||
# 人工已加入收藏夹：候选池不再保留同一条视频
|
||||
self.repo.delete_candidate(aid)
|
||||
if outcome.moved_folder:
|
||||
self._log(
|
||||
"warning",
|
||||
"folder_sync.multi_folder",
|
||||
aid=aid,
|
||||
previous_folder_id=outcome.previous_folder_id,
|
||||
folder_id=folder.media_id,
|
||||
)
|
||||
for aid in infos.missing:
|
||||
source = remote_map.get(aid)
|
||||
self.repo.mark_missing_resource(
|
||||
aid,
|
||||
folder_id=folder.media_id,
|
||||
bvid=source.bvid if source else None,
|
||||
now=now,
|
||||
)
|
||||
result.refreshed = len(infos.items)
|
||||
result.missing = len(infos.missing)
|
||||
|
||||
for aid in removed:
|
||||
self.repo.mark_resource_removed(folder.media_id, aid, now=now)
|
||||
|
||||
self.repo.touch_folder_resources(folder.media_id, now=now)
|
||||
self.repo.finish_folder_sync(
|
||||
folder,
|
||||
media_count=len(remote),
|
||||
full_refresh=full_due,
|
||||
now=now,
|
||||
)
|
||||
|
||||
if len(remote) >= self.capacity_warn:
|
||||
result.warning = (
|
||||
f"收藏夹「{result.title or folder.media_id}」已有 {len(remote)} 条，"
|
||||
f"达到预警线 {self.capacity_warn}；请人工准备下一年度收藏夹并更新配置，"
|
||||
f"系统不会自动换夹"
|
||||
)
|
||||
return result
|
||||
@@ -1611,12 +1611,10 @@ class DashboardQueries:
|
||||
now = self._now()
|
||||
next_runs = {
|
||||
"discover": int(now + cfg.schedule.discover_minutes * 60),
|
||||
"folder_sync": int(now + cfg.schedule.folder_sync_minutes * 60),
|
||||
"snapshot": int(next_daily_run(now, cfg.schedule.snapshot_at)),
|
||||
}
|
||||
for name, key, interval_minutes in (
|
||||
("discover", "discovery", cfg.schedule.discover_minutes),
|
||||
("folder_sync", "folder_sync", cfg.schedule.folder_sync_minutes),
|
||||
):
|
||||
record = tasks.get(key)
|
||||
if isinstance(record, dict) and record.get("finished_at"):
|
||||
@@ -1660,7 +1658,6 @@ class DashboardQueries:
|
||||
"preview": self._kv(conn, "preview:seeded") == "1",
|
||||
"schedule": {
|
||||
"discover_minutes": cfg.schedule.discover_minutes,
|
||||
"folder_sync_minutes": cfg.schedule.folder_sync_minutes,
|
||||
"snapshot_at": cfg.schedule.snapshot_at,
|
||||
"candidate_ttl_days": cfg.schedule.candidate_ttl_days,
|
||||
},
|
||||
|
||||
@@ -30,7 +30,7 @@ function candidateCard(item) {
|
||||
</div>
|
||||
<div class="candidate-actions">
|
||||
${item.url ? `<a class="button ghost" href="${escapeAttr(item.url)}" target="_blank" rel="noreferrer">打开 B 站</a>` : ""}
|
||||
<span class="readonly-note">请在 Telegram 中收藏 / 忽略 / 拉黑</span>
|
||||
<span class="readonly-note">请在 Telegram 中入库 / 忽略 / 拉黑</span>
|
||||
</div>
|
||||
</div>
|
||||
</article>`;
|
||||
@@ -51,7 +51,7 @@ export async function renderCandidates(root) {
|
||||
|
||||
${data.items.length ? `<div class="notice-row">
|
||||
<strong>候选处理</strong>
|
||||
<span>${data.telegram_enabled ? "请在 Telegram 消息中收藏、忽略或拉黑。" : "Telegram 未启用，请使用 CLI 或数据库处理候选。"}</span>
|
||||
<span>${data.telegram_enabled ? "请在 Telegram 消息中入库、忽略或拉黑。" : "Telegram 未启用，请使用 CLI 或数据库处理候选。"}</span>
|
||||
</div>` : ""}
|
||||
|
||||
<section class="panel">
|
||||
|
||||
@@ -30,7 +30,8 @@ function heatStrip(items) {
|
||||
function capacity(folders) {
|
||||
if (!folders.length) return "";
|
||||
return `<section class="panel capacity-panel">
|
||||
${sectionHeader("年度收藏夹")}
|
||||
${sectionHeader("历史收藏夹")}
|
||||
<p class="section-note">系统已不再同步收藏夹；下列统计仅保留历史来源。</p>
|
||||
${folders.map((folder) => `
|
||||
<div class="capacity-row">
|
||||
<div class="capacity-label">
|
||||
@@ -39,7 +40,6 @@ function capacity(folders) {
|
||||
</div>
|
||||
<div class="capacity-track"><span style="width:${Math.min(100, folder.ratio * 100)}%"
|
||||
class="${folder.warning ? "warning" : ""}"></span></div>
|
||||
${folder.warning ? `<p class="warning-text">接近容量上限，请准备新建年度收藏夹。</p>` : ""}
|
||||
</div>`).join("")}
|
||||
</section>`;
|
||||
}
|
||||
@@ -95,7 +95,7 @@ export async function renderHome(root) {
|
||||
${addedNote}
|
||||
<div class="video-list">
|
||||
${added.items.length ? added.items.map((item) => videoCard(item)).join("")
|
||||
: emptyState("最近 7 天没有新入库视频", "收藏夹同步或搜索确认后会出现在这里。")}
|
||||
: emptyState("最近 7 天没有新入库视频", "搜索发现确认或 Telegram 私聊手动入库后会出现在这里。")}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
@@ -47,7 +47,6 @@ export async function renderSystem(root) {
|
||||
<div><span>采集成功</span><strong>${formatCount(health.ok)}</strong></div>
|
||||
<div><span>采集失败</span><strong class="${health.failed ? "danger" : ""}">${formatCount(health.failed)}</strong></div>
|
||||
<div><span>在库</span><strong>${formatCount(counts.active)}</strong></div>
|
||||
<div><span>已取消收藏</span><strong>${formatCount(counts.unfaved)}</strong></div>
|
||||
<div><span>已失效</span><strong>${formatCount(counts.removed)}</strong></div>
|
||||
<div><span>已移出库</span><strong>${formatCount(counts.dropped)}</strong></div>
|
||||
</div>
|
||||
@@ -60,16 +59,16 @@ export async function renderSystem(root) {
|
||||
<section class="panel">
|
||||
${sectionHeader("预计下次运行")}
|
||||
<div class="status-grid">
|
||||
<div><span>收藏夹同步</span><strong>${escapeHtml(formatDateTime(next.folder_sync))}</strong></div>
|
||||
<div><span>搜索发现</span><strong>${escapeHtml(formatDateTime(next.discover))}</strong></div>
|
||||
<div><span>每日快照</span><strong>${escapeHtml(formatDateTime(next.snapshot))}</strong></div>
|
||||
<div><span>调度参数</span><strong>${system.schedule.discover_minutes} / ${system.schedule.folder_sync_minutes} 分钟</strong></div>
|
||||
<div><span>发现周期</span><strong>${system.schedule.discover_minutes} 分钟</strong></div>
|
||||
</div>
|
||||
<p class="section-note">预计时间按配置和最近任务完成时间推算；实际以常驻进程为准。</p>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
${sectionHeader("收藏夹容量")}
|
||||
${sectionHeader("历史收藏夹")}
|
||||
<p class="section-note">系统已不再同步收藏夹；下列统计仅保留历史来源。收藏夹由人工维护，重排序使用 favsort.py。</p>
|
||||
${(system.folders || []).map((folder) => `
|
||||
<div class="capacity-row">
|
||||
<div class="capacity-label">
|
||||
@@ -78,8 +77,8 @@ export async function renderSystem(root) {
|
||||
</div>
|
||||
<div class="capacity-track"><span style="width:${Math.min(100, folder.ratio * 100)}%"
|
||||
class="${folder.warning ? "warning" : ""}"></span></div>
|
||||
<p class="section-note">最近同步 ${escapeHtml(formatDateTime(folder.last_ids_at))}</p>
|
||||
</div>`).join("") || emptyState("未配置收藏夹")}
|
||||
<p class="section-note">历史快照 ${escapeHtml(formatDateTime(folder.last_ids_at))}</p>
|
||||
</div>`).join("") || emptyState("无历史收藏夹记录")}
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
|
||||
+12
-12
@@ -51,7 +51,7 @@ def test_init_and_status():
|
||||
path = write_config(tmp)
|
||||
code, output = run_cli(["init", "--config", str(path)])
|
||||
assert code == 0
|
||||
assert '"schema_version": 4' in output
|
||||
assert '"schema_version": 5' in output
|
||||
|
||||
db = Database(Path(tmp) / "app.db")
|
||||
db.initialize()
|
||||
@@ -79,14 +79,14 @@ def test_missing_config():
|
||||
assert "缺少配置文件" in payload["error"]
|
||||
|
||||
|
||||
def test_sync_requires_auth_without_network():
|
||||
def test_ingest_without_refs_is_offline():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = write_config(tmp, cookie="", csrf="")
|
||||
code, output = run_cli(["sync", "--config", str(path)])
|
||||
assert code == 2
|
||||
payload = json.loads(output.strip().splitlines()[-1])
|
||||
assert "cookie" in payload["error"]
|
||||
assert not (Path(tmp) / "app.db").exists()
|
||||
code, output = run_cli(["ingest", "--config", str(path), "没有视频"])
|
||||
assert code == 1
|
||||
data = json.loads(output[output.rindex("{\n"):])
|
||||
assert data["refs"] == 0
|
||||
assert data["results"] == []
|
||||
|
||||
|
||||
def test_snapshot_offline_empty_db():
|
||||
@@ -180,15 +180,15 @@ def test_run_once_offline():
|
||||
cli._client = original
|
||||
cli.DiscoveryService = original_discovery
|
||||
assert code == 0
|
||||
assert '"name": "folder_sync"' in output
|
||||
assert '"name": "snapshot"' in output
|
||||
assert '"name": "folder_sync"' not in output
|
||||
assert selected and all(credentials.mid == 1 for credentials in selected)
|
||||
|
||||
db = Database(Path(tmp) / "app.db")
|
||||
db.initialize()
|
||||
try:
|
||||
repo = Repository(db)
|
||||
for name in ("folder_sync", "snapshot", "discovery", "candidate_cleanup"):
|
||||
for name in ("snapshot", "discovery", "candidate_cleanup"):
|
||||
assert repo.kv_get(f"task:last:{name}"), name
|
||||
finally:
|
||||
db.close()
|
||||
@@ -229,7 +229,7 @@ def test_default_command_starts_web_and_scheduler():
|
||||
original = cli.cmd_run
|
||||
|
||||
def fake_run(args, logger, *, serve_web=False):
|
||||
calls.append((args.command, serve_web, args.once, args.allow_empty_folder))
|
||||
calls.append((args.command, serve_web, args.once))
|
||||
return 0
|
||||
|
||||
cli.cmd_run = fake_run
|
||||
@@ -238,7 +238,7 @@ def test_default_command_starts_web_and_scheduler():
|
||||
finally:
|
||||
cli.cmd_run = original
|
||||
assert code == 0
|
||||
assert calls == [("run", True, False, False)]
|
||||
assert calls == [("run", True, False)]
|
||||
|
||||
|
||||
def test_snapshot_backup_failure_returns_error_and_retries():
|
||||
@@ -335,7 +335,7 @@ def test_backfill_followers_dry_run():
|
||||
def main_test():
|
||||
test_init_and_status()
|
||||
test_missing_config()
|
||||
test_sync_requires_auth_without_network()
|
||||
test_ingest_without_refs_is_offline()
|
||||
test_snapshot_offline_empty_db()
|
||||
test_auth_check_without_cookie()
|
||||
test_auth_check_all_and_auth_dispatch()
|
||||
|
||||
@@ -24,7 +24,7 @@ def test_write_read_transaction():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db = make_db(tmp)
|
||||
try:
|
||||
assert db.schema_version == 4
|
||||
assert db.schema_version == 5
|
||||
db.write(lambda c: c.execute("INSERT INTO kv(k, v) VALUES ('a', '1')"))
|
||||
assert db.read(lambda c: c.execute("SELECT v FROM kv WHERE k='a'").fetchone()[0]) == "1"
|
||||
|
||||
@@ -94,8 +94,8 @@ def test_close_and_reinitialize():
|
||||
|
||||
db2 = make_db(tmp)
|
||||
try:
|
||||
assert db2.schema_version == 4
|
||||
assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 4
|
||||
assert db2.schema_version == 5
|
||||
assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 5
|
||||
finally:
|
||||
db2.close()
|
||||
|
||||
@@ -123,7 +123,7 @@ def test_migration_backup_on_existing_db():
|
||||
db = Database(path, backup_dir=Path(tmp) / "backup")
|
||||
try:
|
||||
applied = db.initialize()
|
||||
assert applied == [1, 2, 3, 4]
|
||||
assert applied == [1, 2, 3, 4, 5]
|
||||
assert len(list((Path(tmp) / "backup").glob("app-pre-v0-*.db"))) == 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""新视频插入位置纯函数测试: uv run python tests/test_fav_position.py"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
from reaction_board.fav_position import VideoMeta, plan_insertion # noqa: E402
|
||||
|
||||
|
||||
def meta(up_mid, pubdate):
|
||||
return VideoMeta(up_mid=up_mid, pubdate=pubdate)
|
||||
|
||||
|
||||
def test_empty_and_single():
|
||||
assert plan_insertion([], 9, {}, meta(100, 200)).moves == ()
|
||||
assert plan_insertion([], 9, {}, meta(100, 200)).target_position == 1
|
||||
plan = plan_insertion([1], 9, {1: meta(200, 50)}, meta(100, 200))
|
||||
assert plan.target_position == 1
|
||||
assert plan.moves == ((1, 9), (9, 1))
|
||||
|
||||
|
||||
def test_same_up_group():
|
||||
current = [1, 2, 3, 4]
|
||||
metadata = {
|
||||
1: meta(100, 100),
|
||||
2: meta(100, 90),
|
||||
3: meta(200, 80),
|
||||
4: meta(200, 70),
|
||||
}
|
||||
# 同 UP 组在中间：插到组头
|
||||
plan = plan_insertion(current, 9, metadata, meta(200, 95))
|
||||
assert plan.target_position == 3
|
||||
assert plan.moves == ((2, 9),)
|
||||
|
||||
# 同 UP 组在首位：两次移动
|
||||
plan = plan_insertion(current, 9, metadata, meta(100, 200))
|
||||
assert plan.target_position == 1
|
||||
assert plan.moves == ((1, 9), (9, 1))
|
||||
|
||||
|
||||
def test_new_group_by_earliest_pubdate():
|
||||
current = [1, 2, 3, 4]
|
||||
metadata = {
|
||||
1: meta(100, 100),
|
||||
2: meta(100, 90),
|
||||
3: meta(200, 80),
|
||||
4: meta(200, 70),
|
||||
}
|
||||
# 新 UP 的组键 85，应插到组键 70 的组之前
|
||||
plan = plan_insertion(current, 9, metadata, meta(300, 85))
|
||||
assert plan.target_position == 3
|
||||
assert plan.moves == ((2, 9),)
|
||||
|
||||
# 比所有组都旧：追加到末尾
|
||||
plan = plan_insertion(current, 9, metadata, meta(300, 60))
|
||||
assert plan.target_position == 5
|
||||
assert plan.moves == ((4, 9),)
|
||||
|
||||
|
||||
def test_excludes_new_aid_if_present():
|
||||
current = [1, 9, 2]
|
||||
metadata = {1: meta(100, 100), 2: meta(100, 90)}
|
||||
plan = plan_insertion(current, 9, metadata, meta(100, 95))
|
||||
assert plan.target_position == 1
|
||||
assert plan.moves == ((1, 9), (9, 1))
|
||||
|
||||
|
||||
def main():
|
||||
test_empty_and_single()
|
||||
test_same_up_group()
|
||||
test_new_group_by_earliest_pubdate()
|
||||
test_excludes_new_aid_if_present()
|
||||
print("OK: 新视频插入位置计算测试全部通过")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,211 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""收藏/忽略/拉黑动作测试（假写客户端，不联网）: uv run python tests/test_favorites.py"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
from reaction_board.bilibili.client import BilibiliAPIError # noqa: E402
|
||||
from reaction_board.bilibili.models import FavResource # noqa: E402
|
||||
from reaction_board.config import ConfigStore # noqa: E402
|
||||
from reaction_board.database import Database # noqa: E402
|
||||
from reaction_board.repository import Repository # noqa: E402
|
||||
from reaction_board.services.favorites import FavoriteActionService # noqa: E402
|
||||
|
||||
NOW = 1_793_444_393
|
||||
FOLDER_ID = 3700023351
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeFavId:
|
||||
aid: int
|
||||
type: int = 2
|
||||
|
||||
|
||||
class FakeWriteClient:
|
||||
def __init__(self, order, *, fail_deal=False):
|
||||
self.order = list(order)
|
||||
self.fail_deal = fail_deal
|
||||
self.dealt = []
|
||||
self.sorts = []
|
||||
|
||||
def fav_ids(self, media_id):
|
||||
return [FakeFavId(aid) for aid in self.order]
|
||||
|
||||
def deal(self, aid, folder_id):
|
||||
if self.fail_deal:
|
||||
raise BilibiliAPIError(-1, "收藏夹已满")
|
||||
self.dealt.append((aid, folder_id))
|
||||
self.order.append(aid)
|
||||
|
||||
def sort(self, media_id, spec):
|
||||
predecessor, _, moved, _ = (int(part) for part in spec.split(":"))
|
||||
self.sorts.append(spec)
|
||||
self.order.remove(moved)
|
||||
index = self.order.index(predecessor)
|
||||
self.order.insert(index + 1, moved)
|
||||
|
||||
|
||||
class FakeProvider:
|
||||
def __init__(self, client):
|
||||
self.client = client
|
||||
|
||||
def get(self):
|
||||
return self.client
|
||||
|
||||
|
||||
def write_config(tmp):
|
||||
data = {
|
||||
"bilibili": {
|
||||
"read": {"mid": 1, "csrf": "r", "cookie": "read-cookie"},
|
||||
"write": {"mid": 2, "csrf": "w", "cookie": "write-cookie"},
|
||||
},
|
||||
"storage": {
|
||||
"database_path": str(Path(tmp) / "app.db"),
|
||||
"backup_dir": str(Path(tmp) / "backup"),
|
||||
},
|
||||
"folders": [{"year": 2026, "media_id": FOLDER_ID, "title": "2026"}],
|
||||
"discover": {"keywords": ["x"], "pages": 1, "window_days": 14},
|
||||
}
|
||||
path = Path(tmp) / "config.json"
|
||||
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def add_video(repo, aid, up_mid, pubdate):
|
||||
repo.upsert_from_fav(
|
||||
FavResource(
|
||||
aid=aid, bvid=f"BV{aid}", title=f"视频{aid}", up_mid=up_mid,
|
||||
up_name=f"UP{up_mid}", pubdate=pubdate, duration=600,
|
||||
cover=None, attr=0, fav_time=None,
|
||||
),
|
||||
FOLDER_ID,
|
||||
was_added=True,
|
||||
)
|
||||
|
||||
|
||||
def add_candidate(repo, aid, up_mid=100, pubdate=NOW - 3600):
|
||||
return repo.upsert_candidate(
|
||||
aid=aid,
|
||||
bvid=f"BV{aid}",
|
||||
title=f"咻咻满 reaction {aid}",
|
||||
up_mid=up_mid,
|
||||
up_name=f"UP{up_mid}",
|
||||
pubdate=pubdate,
|
||||
cover="https://example/cover.jpg",
|
||||
score=3,
|
||||
channel="search:test#p1",
|
||||
now=NOW,
|
||||
)
|
||||
|
||||
|
||||
def setup(tmp, order, *, fail_deal=False):
|
||||
path = write_config(tmp)
|
||||
store = ConfigStore(path, env={})
|
||||
db = Database(Path(tmp) / "app.db", backup_dir=Path(tmp) / "backup")
|
||||
db.initialize()
|
||||
repo = Repository(db, clock=lambda: NOW)
|
||||
add_video(repo, 1, 100, 1_792_000_000)
|
||||
add_video(repo, 2, 200, 1_791_000_000)
|
||||
add_candidate(repo, 9, up_mid=100, pubdate=1_793_000_000)
|
||||
client = FakeWriteClient(order, fail_deal=fail_deal)
|
||||
service = FavoriteActionService(
|
||||
store, repo, FakeProvider(client), clock=lambda: NOW
|
||||
)
|
||||
return db, repo, client, service
|
||||
|
||||
|
||||
def test_favorite_inserts_and_positions():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, repo, client, service = setup(tmp, [1, 2])
|
||||
try:
|
||||
result = service.favorite(9)
|
||||
assert result.ok is True
|
||||
assert result.position == 1
|
||||
assert result.already_in_folder is False
|
||||
assert client.dealt == [(9, FOLDER_ID)]
|
||||
assert client.sorts == ["1:2:9:2", "9:2:1:2"]
|
||||
assert client.order == [9, 1, 2]
|
||||
assert repo.get_candidate(9) is None
|
||||
video = repo.get_video(9)
|
||||
assert video["status"] == "active"
|
||||
assert video["folder_id"] == FOLDER_ID
|
||||
action = repo.get_fav_action("fav:9")
|
||||
assert action["state"] == "done"
|
||||
assert action["position"] == 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_favorite_already_in_folder_skips_writes():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, repo, client, service = setup(tmp, [1, 9, 2])
|
||||
try:
|
||||
result = service.favorite(9)
|
||||
assert result.ok is True
|
||||
assert result.already_in_folder is True
|
||||
assert result.position == 2
|
||||
assert client.dealt == []
|
||||
assert client.sorts == []
|
||||
assert repo.get_candidate(9) is None
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_favorite_failure_keeps_candidate_and_allows_retry():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, repo, client, service = setup(tmp, [1, 2], fail_deal=True)
|
||||
try:
|
||||
result = service.favorite(9)
|
||||
assert result.ok is False
|
||||
assert "收藏夹已满" in result.message
|
||||
assert repo.get_candidate(9) is not None
|
||||
action = repo.get_fav_action("fav:9")
|
||||
assert action["state"] == "failed"
|
||||
assert "收藏夹已满" in action["err"]
|
||||
|
||||
client.fail_deal = False
|
||||
retry = service.favorite(9)
|
||||
assert retry.ok is True
|
||||
assert retry.position == 1
|
||||
assert repo.get_candidate(9) is None
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_ignore_and_blacklist():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, repo, client, service = setup(tmp, [1, 2])
|
||||
try:
|
||||
add_candidate(repo, 11, up_mid=300)
|
||||
add_candidate(repo, 12, up_mid=300)
|
||||
ignored = service.ignore(11)
|
||||
assert ignored.ok is True
|
||||
assert repo.get_candidate(11)["decision"] == "rejected"
|
||||
|
||||
result = service.blacklist(12)
|
||||
assert result.ok is True
|
||||
assert repo.is_blacklisted(300)
|
||||
assert repo.get_candidate(12)["decision"] == "rejected"
|
||||
# 已有视频不受拉黑影响
|
||||
assert repo.get_video(1)["status"] == "active"
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def main():
|
||||
test_favorite_inserts_and_positions()
|
||||
test_favorite_already_in_folder_skips_writes()
|
||||
test_favorite_failure_keeps_candidate_and_allows_retry()
|
||||
test_ignore_and_blacklist()
|
||||
print("OK: 收藏、忽略、拉黑与失败重试测试全部通过")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,373 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""收藏夹同步集成测试（用真实接口形状的 transport Mock，不联网）:
|
||||
uv run python tests/test_folder_sync.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from fixtures import info_for # noqa: E402
|
||||
from reaction_board.bilibili.client import ( # noqa: E402
|
||||
BilibiliClient,
|
||||
HTTPResponse,
|
||||
Pacer,
|
||||
TransportError,
|
||||
)
|
||||
from reaction_board.config import ConfigStore # noqa: E402
|
||||
from reaction_board.database import Database # noqa: E402
|
||||
from reaction_board.repository import Repository # noqa: E402
|
||||
from reaction_board.services.folder_sync import FolderSyncService # noqa: E402
|
||||
|
||||
F1 = 4451053851
|
||||
F2 = 3700023351
|
||||
|
||||
|
||||
def http_json(payload):
|
||||
return HTTPResponse(
|
||||
200, {"content-type": "application/json"},
|
||||
json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||||
)
|
||||
|
||||
|
||||
class FakeFavTransport:
|
||||
def __init__(self, folders, infos, *, fail_media=None, omit_ids_data=None):
|
||||
self.folders = {k: list(v) for k, v in folders.items()}
|
||||
self.infos = dict(infos)
|
||||
self.fail_media = fail_media
|
||||
self.omit_ids_data = omit_ids_data
|
||||
self.ids_calls = []
|
||||
self.info_calls = []
|
||||
self.relation_calls = []
|
||||
|
||||
def request(self, method, url, headers, data=None, timeout=30):
|
||||
query = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
|
||||
if "/x/relation/stat" in url:
|
||||
mid = int(query["vmid"][0])
|
||||
self.relation_calls.append(mid)
|
||||
return http_json({
|
||||
"code": 0, "message": "0", "ttl": 1,
|
||||
"data": {"mid": mid, "following": 5, "follower": 1000 + mid},
|
||||
})
|
||||
if "/x/v3/fav/resource/ids" in url:
|
||||
media_id = int(query["media_id"][0])
|
||||
self.ids_calls.append(media_id)
|
||||
if self.fail_media == media_id:
|
||||
raise TransportError("ids 网络错误")
|
||||
if self.omit_ids_data == media_id:
|
||||
return http_json({"code": 0, "message": "OK", "ttl": 1})
|
||||
data_rows = [
|
||||
{"id": aid, "type": 2, "bv_id": f"BV{aid}", "bvid": f"BV{aid}"}
|
||||
for aid in self.folders.get(media_id, [])
|
||||
]
|
||||
return http_json({"code": 0, "message": "0", "ttl": 1, "data": data_rows})
|
||||
if "/x/v3/fav/resource/infos" in url:
|
||||
media_id = int(query["folder_id"][0])
|
||||
resources = query["resources"][0]
|
||||
aids = [int(part.split(":")[0]) for part in resources.split(",")]
|
||||
self.info_calls.append((media_id, aids))
|
||||
rows = [self.infos[aid] for aid in aids if aid in self.infos]
|
||||
return http_json({"code": 0, "message": "0", "ttl": 1, "data": rows})
|
||||
if "/x/v3/fav/folder/created/list-all" in url:
|
||||
return http_json({
|
||||
"code": 0, "message": "0", "ttl": 1,
|
||||
"data": {"count": 0, "list": []},
|
||||
})
|
||||
raise AssertionError(f"未预期的 URL: {url}")
|
||||
|
||||
|
||||
def write_config(tmp, folders):
|
||||
data = {
|
||||
"bilibili": {
|
||||
"read": {"mid": 28802351, "csrf": "csrf-secret", "cookie": "SESSDATA=secret; bili_jct=csrf-secret; buvid3=device"},
|
||||
"write": {"mid": 2, "csrf": "write-csrf", "cookie": "write-cookie"},
|
||||
},
|
||||
"storage": {
|
||||
"database_path": str(Path(tmp) / "app.db"),
|
||||
"backup_dir": str(Path(tmp) / "backup"),
|
||||
},
|
||||
"folders": folders,
|
||||
"up_blacklist_seed": [999],
|
||||
"schedule": {
|
||||
"discover_minutes": 60,
|
||||
"folder_sync_minutes": 60,
|
||||
"full_folder_refresh_hours": 6,
|
||||
"snapshot_at": "00:01",
|
||||
"candidate_ttl_days": 7,
|
||||
},
|
||||
}
|
||||
path = Path(tmp) / "config.json"
|
||||
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def make_service(tmp, transport, clock, *, capacity_warn=950):
|
||||
folders = [
|
||||
{"year": 2025, "media_id": F1, "title": "2025"},
|
||||
{"year": 2026, "media_id": F2, "title": "2026"},
|
||||
]
|
||||
path = write_config(tmp, folders)
|
||||
store = ConfigStore(path, env={}, require_auth=True)
|
||||
db = Database(Path(tmp) / "app.db", backup_dir=Path(tmp) / "backup")
|
||||
db.initialize()
|
||||
repo = Repository(db, clock=lambda: clock[0])
|
||||
client = BilibiliClient(
|
||||
cookie="SESSDATA=secret; bvid3=device",
|
||||
csrf="csrf",
|
||||
mid=28802351,
|
||||
transport=transport,
|
||||
pacer=Pacer(0, 0),
|
||||
retries=1,
|
||||
)
|
||||
service = FolderSyncService(
|
||||
store, repo, client, clock=lambda: clock[0], capacity_warn=capacity_warn
|
||||
)
|
||||
return db, repo, service
|
||||
|
||||
|
||||
def valid(aid, *, up_mid=100, title=None):
|
||||
return info_for(aid, title=title or f"视频{aid}", up_mid=up_mid, up_name="UP")
|
||||
|
||||
|
||||
def test_initial_import_and_incremental_skip():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = FakeFavTransport(
|
||||
{F1: [1101, 1102, 1103, 1104], F2: []},
|
||||
{
|
||||
1101: valid(1101),
|
||||
1102: valid(1102, up_mid=999), # 黑名单 UP 也必须正常入库
|
||||
1103: valid(1103, title="已失效", up_mid=100),
|
||||
},
|
||||
)
|
||||
# 1103 标记为 attr=9；1104 静默缺失
|
||||
transport.infos[1103]["attr"] = 9
|
||||
clock = [1_800_000_000]
|
||||
db, repo, service = make_service(tmp, transport, clock)
|
||||
try:
|
||||
repo.upsert_candidate(
|
||||
aid=1101, bvid="BV1101", title="咻咻满 reaction",
|
||||
up_mid=100, up_name="UP", pubdate=1_799_000_000,
|
||||
cover=None, score=3, channel="search:test#p1",
|
||||
now=1_799_999_000,
|
||||
)
|
||||
summary = service.run()
|
||||
assert summary.ok
|
||||
assert summary.added == 4
|
||||
assert summary.missing == 1
|
||||
assert summary.refreshed == 3
|
||||
# 人工加入收藏夹后，同 aid 候选自动移除
|
||||
assert repo.get_candidate(1101) is None
|
||||
assert repo.seed_blacklist([999], now=0) == 0 # 已由 service 写入
|
||||
|
||||
assert repo.get_video(1101)["status"] == "active"
|
||||
assert repo.get_video(1102)["status"] == "active" # 黑名单不影响收藏夹导入
|
||||
assert repo.get_video(1103)["status"] == "removed"
|
||||
assert repo.get_video(1104)["status"] == "removed"
|
||||
assert repo.get_video(1104)["bvid"] == "BV1104"
|
||||
assert repo.folder_active_aids(F1) == {1101, 1102, 1103, 1104}
|
||||
assert repo.is_blacklisted(999)
|
||||
# 新入库视频记录入库时粉丝数；失效视频不再采集
|
||||
assert summary.followers_updated == 2
|
||||
assert summary.followers_failed == 0
|
||||
assert repo.get_video(1101)["up_follower"] == 1100
|
||||
assert repo.get_video(1101)["up_follower_backfilled"] == 0
|
||||
assert repo.get_video(1102)["up_follower"] == 1999
|
||||
assert sorted(transport.relation_calls) == [100, 999]
|
||||
|
||||
folder = repo.get_folder(F1)
|
||||
assert folder.media_count == 4
|
||||
assert folder.last_full_at == 1_800_000_000
|
||||
|
||||
# 6 小时内再次运行：没有新增，不请求 infos
|
||||
clock[0] += 60
|
||||
info_before = len(transport.info_calls)
|
||||
relation_before = len(transport.relation_calls)
|
||||
summary2 = service.run()
|
||||
assert summary2.added == 0
|
||||
assert summary2.refreshed == 0
|
||||
assert len(transport.info_calls) == info_before
|
||||
assert len(transport.relation_calls) == relation_before
|
||||
assert summary2.folders[0].full_refresh is False
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_diff_add_and_remove():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = FakeFavTransport(
|
||||
{F1: [1101, 1102], F2: []},
|
||||
{1101: valid(1101), 1102: valid(1102)},
|
||||
)
|
||||
clock = [1_800_000_000]
|
||||
db, repo, service = make_service(tmp, transport, clock)
|
||||
try:
|
||||
service.run()
|
||||
assert repo.get_video(1101)["status"] == "active"
|
||||
assert repo.get_video(1102)["status"] == "active"
|
||||
|
||||
transport.folders[F1] = [1102, 1103]
|
||||
transport.infos[1103] = valid(1103)
|
||||
clock[0] += 3600
|
||||
summary = service.run()
|
||||
assert summary.added == 1
|
||||
assert summary.removed == 1
|
||||
assert summary.refreshed == 1
|
||||
assert repo.get_video(1101)["status"] == "unfaved"
|
||||
assert repo.get_video(1102)["status"] == "active"
|
||||
assert repo.get_video(1103)["status"] == "active"
|
||||
assert repo.folder_active_aids(F1) == {1102, 1103}
|
||||
assert repo.get_folder(F1).media_count == 2
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_full_refresh_marks_invalid_and_missing():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = FakeFavTransport(
|
||||
{F1: [1101, 1102], F2: []},
|
||||
{1101: valid(1101), 1102: valid(1102)},
|
||||
)
|
||||
clock = [1_800_000_000]
|
||||
db, repo, service = make_service(tmp, transport, clock)
|
||||
try:
|
||||
service.run()
|
||||
assert repo.get_video(1101)["status"] == "active"
|
||||
assert repo.get_video(1102)["status"] == "active"
|
||||
|
||||
# 6 小时后：1101 失效，1102 静默缺失
|
||||
clock[0] += 6 * 3600 + 1
|
||||
transport.infos[1101]["attr"] = 9
|
||||
del transport.infos[1102]
|
||||
summary = service.run()
|
||||
assert summary.folders[0].full_refresh is True
|
||||
assert summary.missing == 1
|
||||
assert repo.get_video(1101)["status"] == "removed"
|
||||
assert repo.get_video(1102)["status"] == "removed"
|
||||
assert repo.get_folder(F1).last_full_at == clock[0]
|
||||
# 全量刷新请求了夹内全部 aid
|
||||
assert set(transport.info_calls[-1][1]) == {1101, 1102}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_capacity_warning_does_not_move_folder():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = FakeFavTransport(
|
||||
{F1: [1, 2, 3], F2: []},
|
||||
{i: valid(i) for i in (1, 2, 3)},
|
||||
)
|
||||
clock = [1_800_000_000]
|
||||
db, repo, service = make_service(tmp, transport, clock, capacity_warn=3)
|
||||
try:
|
||||
summary = service.run()
|
||||
assert summary.folders[0].warning
|
||||
assert "预警线 3" in summary.folders[0].warning
|
||||
assert "不会自动换夹" in summary.folders[0].warning
|
||||
# 视频仍在原夹，不被移动
|
||||
assert all(repo.get_video(i)["folder_id"] == F1 for i in (1, 2, 3))
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_folder_error_does_not_block_other_folders():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = FakeFavTransport(
|
||||
{F1: [1101], F2: [2201]},
|
||||
{1101: valid(1101), 2201: valid(2201)},
|
||||
fail_media=F1,
|
||||
)
|
||||
clock = [1_800_000_000]
|
||||
db, repo, service = make_service(tmp, transport, clock)
|
||||
try:
|
||||
summary = service.run()
|
||||
assert summary.ok is False
|
||||
assert len(summary.errors) == 1 and str(F1) in summary.errors[0]
|
||||
assert summary.folders[0].error
|
||||
assert summary.folders[1].error is None
|
||||
assert repo.get_video(2201)["status"] == "active"
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_blacklist_seed_only_once():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = FakeFavTransport({F1: [1101], F2: []}, {1101: valid(1101)})
|
||||
clock = [1_800_000_000]
|
||||
db, repo, service = make_service(tmp, transport, clock)
|
||||
try:
|
||||
first = service.run()
|
||||
assert first.blacklist_seeded == 1
|
||||
clock[0] += 60
|
||||
second = service.run()
|
||||
assert second.blacklist_seeded == 0
|
||||
assert len(repo.blacklist()) == 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_empty_remote_requires_explicit_opt_in():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = FakeFavTransport(
|
||||
{F1: [1101, 1102], F2: []},
|
||||
{1101: valid(1101), 1102: valid(1102)},
|
||||
)
|
||||
clock = [1_800_000_000]
|
||||
db, repo, service = make_service(tmp, transport, clock)
|
||||
try:
|
||||
service.run()
|
||||
transport.folders[F1] = []
|
||||
clock[0] += 3600
|
||||
summary = service.run()
|
||||
assert summary.ok is False
|
||||
assert "疑似接口异常" in summary.folders[0].error
|
||||
assert repo.get_video(1101)["status"] == "active"
|
||||
assert repo.folder_active_aids(F1) == {1101, 1102}
|
||||
|
||||
service.allow_empty_folders = True
|
||||
summary = service.run()
|
||||
assert summary.ok is True
|
||||
assert repo.get_video(1101)["status"] == "unfaved"
|
||||
assert repo.folder_active_aids(F1) == set()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_missing_ids_data_aborts_folder():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
transport = FakeFavTransport(
|
||||
{F1: [1101], F2: []},
|
||||
{1101: valid(1101)},
|
||||
omit_ids_data=F1,
|
||||
)
|
||||
clock = [1_800_000_000]
|
||||
db, repo, service = make_service(tmp, transport, clock)
|
||||
try:
|
||||
summary = service.run()
|
||||
assert summary.ok is False
|
||||
assert "data" in summary.folders[0].error
|
||||
assert repo.get_video(1101) is None
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def main():
|
||||
test_initial_import_and_incremental_skip()
|
||||
test_diff_add_and_remove()
|
||||
test_full_refresh_marks_invalid_and_missing()
|
||||
test_capacity_warning_does_not_move_folder()
|
||||
test_folder_error_does_not_block_other_folders()
|
||||
test_blacklist_seed_only_once()
|
||||
test_empty_remote_requires_explicit_opt_in()
|
||||
test_missing_ids_data_aborts_folder()
|
||||
print("OK: 收藏夹初始导入、差集同步、失效识别与容量预警测试全部通过")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -37,8 +37,8 @@ def test_fresh_and_idempotent():
|
||||
conn = sqlite3.connect(Path(tmp) / "app.db")
|
||||
try:
|
||||
applied = run_migrations(conn)
|
||||
assert applied == [1, 2, 3, 4]
|
||||
assert current_version(conn) == 4
|
||||
assert applied == [1, 2, 3, 4, 5]
|
||||
assert current_version(conn) == 5
|
||||
assert EXPECTED_TABLES <= table_names(conn)
|
||||
assert "schema_version" in table_names(conn)
|
||||
columns = {
|
||||
@@ -66,7 +66,7 @@ def test_fresh_and_idempotent():
|
||||
# 再跑一次不应重复应用
|
||||
assert run_migrations(conn) == []
|
||||
row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone()
|
||||
assert row[0] == 4
|
||||
assert row[0] == 5
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@@ -81,15 +81,15 @@ def test_backup_before_upgrade():
|
||||
conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')")
|
||||
conn.commit()
|
||||
|
||||
v5 = Migration(5, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
|
||||
v6 = Migration(6, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
|
||||
applied = run_migrations(
|
||||
conn,
|
||||
backup_dir=backup_dir,
|
||||
now=1_800_000_000,
|
||||
migrations=MIGRATIONS + (v5,),
|
||||
migrations=MIGRATIONS + (v6,),
|
||||
)
|
||||
assert applied == [5]
|
||||
backups = list(backup_dir.glob("app-pre-v4-*.db"))
|
||||
assert applied == [6]
|
||||
backups = list(backup_dir.glob("app-pre-v5-*.db"))
|
||||
assert len(backups) == 1
|
||||
check = sqlite3.connect(backups[0])
|
||||
try:
|
||||
@@ -159,12 +159,35 @@ def test_validate_migration_order():
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_v5_normalizes_unfaved():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
conn = sqlite3.connect(Path(tmp) / "app.db")
|
||||
try:
|
||||
run_migrations(conn, migrations=MIGRATIONS[:4])
|
||||
conn.execute(
|
||||
"INSERT INTO video(aid, title, pubdate, status, first_seen) "
|
||||
"VALUES (1, 't', 1, 'unfaved', 1)"
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO video(aid, title, pubdate, status, first_seen) "
|
||||
"VALUES (2, 't', 1, 'dropped', 1)"
|
||||
)
|
||||
conn.commit()
|
||||
applied = run_migrations(conn)
|
||||
assert applied == [5]
|
||||
rows = dict(conn.execute("SELECT aid, status FROM video").fetchall())
|
||||
assert rows == {1: "active", 2: "dropped"}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def main():
|
||||
test_fresh_and_idempotent()
|
||||
test_backup_before_upgrade()
|
||||
test_failed_migration_rolls_back()
|
||||
test_prune_backups()
|
||||
test_validate_migration_order()
|
||||
test_v5_normalizes_unfaved()
|
||||
print("OK: SQLite 迁移、备份与回滚测试全部通过")
|
||||
|
||||
|
||||
|
||||
@@ -342,7 +342,6 @@ def test_fans_candidates_system_and_export():
|
||||
assert system["errors"] == []
|
||||
assert system["preview"] is False
|
||||
assert system["next_runs"]["snapshot"] > 0
|
||||
assert system["next_runs"]["folder_sync"] > FIXED_NOW
|
||||
assert system["next_runs"]["discover"] > FIXED_NOW
|
||||
finally:
|
||||
db.close()
|
||||
@@ -455,7 +454,7 @@ def test_about_payload():
|
||||
assert about["snapshot_days"] == 3
|
||||
assert about["snapshot_first_date"] == DATES[0]
|
||||
assert about["fans_start_date"] == DATES[0]
|
||||
assert about["schema_version"] == 4
|
||||
assert about["schema_version"] == 5
|
||||
assert about["analytics"] == {
|
||||
"min_duration_seconds": 600,
|
||||
"maturity_days": 2,
|
||||
|
||||
@@ -75,7 +75,7 @@ def test_readonly_routes_and_static():
|
||||
assert status == 200 and payload["ok"] is True
|
||||
status, _, payload = running.json("/api/about")
|
||||
assert status == 200
|
||||
assert payload["data"]["schema_version"] == 4
|
||||
assert payload["data"]["schema_version"] == 5
|
||||
assert payload["data"]["analytics"]["min_duration_seconds"] == 600
|
||||
|
||||
status, _, overview = running.json("/api/overview")
|
||||
|
||||
@@ -48,8 +48,6 @@ def write_config(
|
||||
"up_blacklist_seed": [],
|
||||
"schedule": {
|
||||
"discover_minutes": 60,
|
||||
"folder_sync_minutes": 60,
|
||||
"full_folder_refresh_hours": 6,
|
||||
"snapshot_at": "00:01",
|
||||
"candidate_ttl_days": 7,
|
||||
},
|
||||
|
||||
@@ -271,12 +271,12 @@ def _seed_task_records(
|
||||
"backup_path": f"data/backup/app-{latest_day.strftime('%Y%m%d')}.db",
|
||||
},
|
||||
},
|
||||
"folder_sync": {
|
||||
"name": "folder_sync", "ok": True,
|
||||
"ingest": {
|
||||
"name": "ingest", "ok": True,
|
||||
"started_at": int(base + 20 * 3600),
|
||||
"finished_at": int(base + 20 * 3600 + 8),
|
||||
"finished_at": int(base + 20 * 3600 + 6),
|
||||
"error": None,
|
||||
"summary": {"folders": 2, "added": 65, "removed": 0, "ok": True},
|
||||
"summary": {"refs": 2, "added": 2, "ok": True},
|
||||
},
|
||||
"discovery": {
|
||||
"name": "discovery", "ok": True,
|
||||
|
||||
Reference in new issue
Block a user