feat(p3): add discovery, telegram notifications, and one-click favorites
Implement WBI video search with legacy fallback, classification scoring, candidate pool, permanent seen-aid dedupe, blacklist filtering, and batched Telegram notifications. Add write client for fav deal/sort, target-folder selection, insertion position calculation, read-back verification, fav_action locking, retry on failure, ignore and blacklist actions. Wire discovery, candidate cleanup, Telegram long polling, and new discover/candidates CLI commands into the scheduler. Add migration v3 for candidate duration and index stats.
This commit is contained in:
kairee committed
1 parent
6a21b5d905
commit
d971e718e9
31 files changed
+2886
-55
No files matched your search
@@ -0,0 +1,224 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""每小时搜索发现与候选池写入。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from ..bilibili.client import BilibiliError
|
||||
from ..bilibili.search import SearchClient, SearchItem
|
||||
from ..classify import passes_required_title, score_item
|
||||
from ..config import ConfigStore
|
||||
from ..repository import Repository
|
||||
|
||||
|
||||
@dataclass
|
||||
class DiscoverySummary:
|
||||
started_at: int
|
||||
finished_at: int = 0
|
||||
searches: int = 0
|
||||
items: int = 0
|
||||
known: int = 0
|
||||
old: int = 0
|
||||
blacklisted: int = 0
|
||||
title_miss: int = 0
|
||||
scored: int = 0
|
||||
candidates: int = 0
|
||||
notified: int = 0
|
||||
notify_failed: int = 0
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"elapsed": max(0, self.finished_at - self.started_at),
|
||||
"searches": self.searches,
|
||||
"items": self.items,
|
||||
"known": self.known,
|
||||
"old": self.old,
|
||||
"blacklisted": self.blacklisted,
|
||||
"title_miss": self.title_miss,
|
||||
"scored": self.scored,
|
||||
"candidates": self.candidates,
|
||||
"notified": self.notified,
|
||||
"notify_failed": self.notify_failed,
|
||||
"errors": self.errors,
|
||||
}
|
||||
|
||||
|
||||
class DiscoveryService:
|
||||
def __init__(
|
||||
self,
|
||||
config_store: ConfigStore,
|
||||
repository: Repository,
|
||||
search_client: SearchClient,
|
||||
*,
|
||||
notifier=None,
|
||||
clock=time.time,
|
||||
sleeper=time.sleep,
|
||||
rng: random.Random | None = None,
|
||||
logger=None,
|
||||
) -> None:
|
||||
self.config_store = config_store
|
||||
self.repo = repository
|
||||
self.search_client = search_client
|
||||
self.notifier = notifier
|
||||
self._clock = clock
|
||||
self._sleep = sleeper
|
||||
self._rng = rng or random.Random()
|
||||
self.logger = logger
|
||||
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
|
||||
def _log(self, level: str, event: str, **fields) -> None:
|
||||
if self.logger is not None:
|
||||
getattr(self.logger, level)(event, **fields)
|
||||
|
||||
def run(self) -> DiscoverySummary:
|
||||
cfg = self.config_store.get()
|
||||
now = self._now()
|
||||
summary = DiscoverySummary(started_at=now)
|
||||
pending: list[dict[str, Any]] = []
|
||||
first = True
|
||||
|
||||
for keyword in cfg.discover.keywords:
|
||||
for page in range(1, cfg.discover.pages + 1):
|
||||
if not first:
|
||||
self._sleep(self._rng.uniform(1.5, 2.5))
|
||||
first = False
|
||||
channel = f"search:{keyword}#p{page}"
|
||||
try:
|
||||
items = self.search_client.search(keyword, page)
|
||||
except BilibiliError as exc:
|
||||
summary.errors.append(f"{channel}: {exc}")
|
||||
self._log(
|
||||
"error", "discovery.search_failed",
|
||||
channel=channel, error=str(exc),
|
||||
)
|
||||
continue
|
||||
summary.searches += 1
|
||||
for item in items:
|
||||
self._handle_item(item, channel, cfg, summary, pending, now)
|
||||
|
||||
self._notify(pending, summary, now)
|
||||
summary.finished_at = self._now()
|
||||
self._log(
|
||||
"info" if not summary.errors else "warning",
|
||||
"discovery.done",
|
||||
**summary.as_dict(),
|
||||
)
|
||||
return summary
|
||||
|
||||
def _handle_item(
|
||||
self,
|
||||
item: SearchItem,
|
||||
channel: str,
|
||||
cfg,
|
||||
summary: DiscoverySummary,
|
||||
pending: list[dict[str, Any]],
|
||||
now: int,
|
||||
) -> None:
|
||||
summary.items += 1
|
||||
existing = self.repo.get_candidate(item.aid)
|
||||
if existing is not None:
|
||||
summary.known += 1
|
||||
if (
|
||||
existing.get("notified_at") is None
|
||||
and existing.get("decision") is None
|
||||
and float(existing.get("score") or 0) >= cfg.notify.min_score
|
||||
):
|
||||
self._queue_notify(pending, existing)
|
||||
return
|
||||
if self.repo.known_aid(item.aid):
|
||||
summary.known += 1
|
||||
return
|
||||
if item.pubdate and now - item.pubdate > cfg.discover.window_days * 86400:
|
||||
summary.old += 1
|
||||
return
|
||||
if not passes_required_title(item.title, cfg.classify["require_title"]):
|
||||
summary.title_miss += 1
|
||||
return
|
||||
if item.up_mid is not None and self.repo.is_blacklisted(item.up_mid):
|
||||
summary.blacklisted += 1
|
||||
return
|
||||
|
||||
self.repo.record_seen(item.aid, source=channel, now=now)
|
||||
score = score_item(
|
||||
title=item.title,
|
||||
up_mid=item.up_mid,
|
||||
duration_seconds=item.duration_seconds,
|
||||
classify=cfg.classify,
|
||||
)
|
||||
summary.scored += 1
|
||||
if score < 0:
|
||||
return
|
||||
|
||||
candidate = self.repo.upsert_candidate(
|
||||
aid=item.aid,
|
||||
bvid=item.bvid,
|
||||
title=item.title,
|
||||
up_mid=item.up_mid,
|
||||
up_name=item.up_name,
|
||||
pubdate=item.pubdate,
|
||||
cover=item.cover,
|
||||
score=score,
|
||||
channel=channel,
|
||||
duration=item.duration_seconds,
|
||||
play=item.play,
|
||||
like=item.like,
|
||||
favorite=item.favorite,
|
||||
danmaku=item.danmaku,
|
||||
review=item.review,
|
||||
now=now,
|
||||
)
|
||||
summary.candidates += 1
|
||||
if (
|
||||
score >= cfg.notify.min_score
|
||||
and candidate.get("notified_at") is None
|
||||
and candidate.get("decision") is None
|
||||
):
|
||||
self._queue_notify(pending, candidate)
|
||||
|
||||
@staticmethod
|
||||
def _queue_notify(
|
||||
pending: list[dict[str, Any]], candidate: dict[str, Any]
|
||||
) -> None:
|
||||
"""同一次扫描中同一 aid 可能出现在多个关键词/页，按 aid 去重。"""
|
||||
if all(item["aid"] != candidate["aid"] for item in pending):
|
||||
pending.append(candidate)
|
||||
|
||||
def _notify(
|
||||
self, pending: list[dict[str, Any]], summary: DiscoverySummary, now: int
|
||||
) -> None:
|
||||
if not pending or self.notifier is None:
|
||||
return
|
||||
batch_id = f"{now}-{self._rng.randrange(1_000_000):06d}"
|
||||
error: str | None = None
|
||||
ok = False
|
||||
try:
|
||||
ok = bool(self.notifier.send_candidates(pending))
|
||||
except Exception as exc: # noqa: BLE001 通知失败不回滚候选
|
||||
error = str(exc)
|
||||
for candidate in pending:
|
||||
self.repo.mark_notified(
|
||||
candidate["aid"],
|
||||
channel="telegram",
|
||||
batch_id=batch_id,
|
||||
ok=ok,
|
||||
err=error,
|
||||
now=now,
|
||||
)
|
||||
if ok:
|
||||
summary.notified = len(pending)
|
||||
else:
|
||||
summary.notify_failed = len(pending)
|
||||
summary.errors.append(f"telegram: {error or '发送失败'}")
|
||||
self._log(
|
||||
"error", "discovery.notify_failed",
|
||||
count=len(pending), error=error or "发送失败",
|
||||
)
|
||||
@@ -0,0 +1,234 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""候选收藏、忽略、拉黑动作。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from ..bilibili.models import FavResource
|
||||
from ..config import ConfigStore
|
||||
from ..fav_position import VideoMeta, plan_insertion
|
||||
from ..repository import Repository
|
||||
from ..timeutil import shanghai_datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActionResult:
|
||||
ok: bool
|
||||
aid: int
|
||||
action: str
|
||||
state: str
|
||||
position: int | None = None
|
||||
already_in_folder: bool = False
|
||||
message: str = ""
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"ok": self.ok,
|
||||
"aid": self.aid,
|
||||
"action": self.action,
|
||||
"state": self.state,
|
||||
"position": self.position,
|
||||
"already_in_folder": self.already_in_folder,
|
||||
"message": self.message,
|
||||
}
|
||||
|
||||
|
||||
class FavoriteActionService:
|
||||
def __init__(
|
||||
self,
|
||||
config_store: ConfigStore,
|
||||
repository: Repository,
|
||||
write_provider,
|
||||
*,
|
||||
clock=time.time,
|
||||
logger=None,
|
||||
) -> None:
|
||||
self.config_store = config_store
|
||||
self.repo = repository
|
||||
self.write_provider = write_provider
|
||||
self._clock = clock
|
||||
self.logger = logger
|
||||
|
||||
def _now(self) -> int:
|
||||
return int(self._clock())
|
||||
|
||||
def _log(self, level: str, event: str, **fields) -> None:
|
||||
if self.logger is not None:
|
||||
getattr(self.logger, level)(event, **fields)
|
||||
|
||||
def _target_folder(self, candidate: dict[str, Any]):
|
||||
cfg = self.config_store.get()
|
||||
up_mid = candidate.get("up_mid")
|
||||
if isinstance(up_mid, int) and not isinstance(up_mid, bool):
|
||||
folder_id = self.repo.folder_for_up(up_mid)
|
||||
if folder_id is not None:
|
||||
row = self.repo.get_folder(folder_id)
|
||||
if row is not None and row.enabled:
|
||||
return row
|
||||
year = shanghai_datetime(self._clock()).year
|
||||
folder = cfg.folder_for_year(year)
|
||||
if folder is not None:
|
||||
return folder
|
||||
latest = cfg.latest_folder()
|
||||
if latest is not None:
|
||||
return latest
|
||||
raise RuntimeError("没有可用的目标收藏夹，请检查 folders 配置")
|
||||
|
||||
def favorite(self, aid: int) -> ActionResult:
|
||||
aid = int(aid)
|
||||
candidate = self.repo.get_candidate(aid)
|
||||
if candidate is None:
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="favorite", state="missing",
|
||||
message="候选不存在或已处理",
|
||||
)
|
||||
try:
|
||||
folder = self._target_folder(candidate)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="favorite", state="failed",
|
||||
message=str(exc),
|
||||
)
|
||||
|
||||
token, state = self.repo.begin_fav_action(
|
||||
aid, folder.media_id, now=self._now()
|
||||
)
|
||||
if state == "done":
|
||||
action = self.repo.get_fav_action(token) or {}
|
||||
return ActionResult(
|
||||
ok=True, aid=aid, action="favorite", state="done",
|
||||
position=action.get("position"),
|
||||
message="已在收藏夹",
|
||||
)
|
||||
if state == "busy":
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="favorite", state="busy",
|
||||
message="该候选正在处理中，请稍后重试",
|
||||
)
|
||||
|
||||
already = False
|
||||
try:
|
||||
client = self.write_provider.get()
|
||||
ids = client.fav_ids(folder.media_id)
|
||||
if aid in {item.aid for item in ids}:
|
||||
already = True
|
||||
position = [item.aid for item in ids].index(aid) + 1
|
||||
else:
|
||||
client.deal(aid, folder.media_id)
|
||||
ids = client.fav_ids(folder.media_id)
|
||||
aid_order = [item.aid for item in ids]
|
||||
if aid not in aid_order:
|
||||
raise RuntimeError("deal 成功但回读收藏夹未找到该视频")
|
||||
metadata = {
|
||||
item_aid: VideoMeta(
|
||||
up_mid=(info.get("up_mid") if info else None),
|
||||
pubdate=int((info or {}).get("pubdate") or 0),
|
||||
)
|
||||
for item_aid, info in self.repo.video_metadata(aid_order).items()
|
||||
}
|
||||
plan = plan_insertion(
|
||||
aid_order,
|
||||
aid,
|
||||
metadata,
|
||||
VideoMeta(
|
||||
up_mid=candidate.get("up_mid"),
|
||||
pubdate=int(candidate.get("pubdate") or 0),
|
||||
),
|
||||
)
|
||||
type_map = {item.aid: item.type for item in ids}
|
||||
for predecessor, moved in plan.moves:
|
||||
client.sort(
|
||||
folder.media_id,
|
||||
f"{predecessor}:{type_map.get(predecessor, 2)}:"
|
||||
f"{moved}:{type_map.get(moved, 2)}",
|
||||
)
|
||||
final_order = [item.aid for item in client.fav_ids(folder.media_id)]
|
||||
if aid not in final_order:
|
||||
raise RuntimeError("排序后回读收藏夹未找到该视频")
|
||||
position = final_order.index(aid) + 1
|
||||
if position != plan.target_position:
|
||||
raise RuntimeError(
|
||||
f"回读位置不符：期望 #{plan.target_position}，实际 #{position}"
|
||||
)
|
||||
|
||||
resource = FavResource(
|
||||
aid=aid,
|
||||
bvid=candidate.get("bvid"),
|
||||
title=str(candidate.get("title") or "(无标题)"),
|
||||
up_mid=candidate.get("up_mid"),
|
||||
up_name=candidate.get("up_name"),
|
||||
pubdate=int(candidate.get("pubdate") or 0),
|
||||
duration=None,
|
||||
cover=candidate.get("cover"),
|
||||
attr=0,
|
||||
fav_time=None,
|
||||
)
|
||||
self.repo.upsert_from_fav(
|
||||
resource, folder.media_id, was_added=True, now=self._now()
|
||||
)
|
||||
self.repo.delete_candidate(aid)
|
||||
self.repo.finish_fav_action(token, position=position, now=self._now())
|
||||
self._log(
|
||||
"info", "favorite.done",
|
||||
aid=aid, folder_id=folder.media_id, position=position,
|
||||
already=already,
|
||||
)
|
||||
return ActionResult(
|
||||
ok=True, aid=aid, action="favorite", state="done",
|
||||
position=position, already_in_folder=already,
|
||||
message="已在收藏夹" if already else "已加入收藏夹",
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 失败保留候选，允许重试
|
||||
self.repo.fail_fav_action(token, error=str(exc), now=self._now())
|
||||
self._log(
|
||||
"error", "favorite.failed",
|
||||
aid=aid, folder_id=folder.media_id, error=str(exc),
|
||||
)
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="favorite", state="failed",
|
||||
message=str(exc),
|
||||
)
|
||||
|
||||
def ignore(self, aid: int) -> ActionResult:
|
||||
aid = int(aid)
|
||||
candidate = self.repo.get_candidate(aid)
|
||||
if candidate is None:
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="ignore", state="missing",
|
||||
message="候选不存在或已处理",
|
||||
)
|
||||
self.repo.mark_candidate_decision(aid, "rejected", now=self._now())
|
||||
return ActionResult(
|
||||
ok=True, aid=aid, action="ignore", state="rejected",
|
||||
message="已忽略",
|
||||
)
|
||||
|
||||
def blacklist(self, aid: int) -> ActionResult:
|
||||
aid = int(aid)
|
||||
candidate = self.repo.get_candidate(aid)
|
||||
if candidate is None:
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="blacklist", state="missing",
|
||||
message="候选不存在或已处理",
|
||||
)
|
||||
up_mid = candidate.get("up_mid")
|
||||
if not isinstance(up_mid, int) or isinstance(up_mid, bool):
|
||||
return ActionResult(
|
||||
ok=False, aid=aid, action="blacklist", state="failed",
|
||||
message="候选缺少 UP mid，无法拉黑",
|
||||
)
|
||||
self.repo.add_blacklist(
|
||||
up_mid,
|
||||
name=candidate.get("up_name"),
|
||||
reason="telegram",
|
||||
now=self._now(),
|
||||
)
|
||||
rejected = self.repo.reject_pending_for_up(up_mid, now=self._now())
|
||||
self.repo.mark_candidate_decision(aid, "rejected", now=self._now())
|
||||
return ActionResult(
|
||||
ok=True, aid=aid, action="blacklist", state="rejected",
|
||||
message=f"已拉黑 UP，并忽略该 UP 的 {rejected} 条待决策候选",
|
||||
)
|
||||
@@ -260,6 +260,9 @@ class FolderSyncService:
|
||||
outcome = self.repo.upsert_from_fav(
|
||||
resource, folder.media_id, was_added=aid in added, now=now
|
||||
)
|
||||
if aid in added:
|
||||
# 人工已加入收藏夹：候选池不再保留同一条视频
|
||||
self.repo.delete_candidate(aid)
|
||||
if outcome.moved_folder:
|
||||
self._log(
|
||||
"warning",
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Telegram getUpdates long polling 与按钮回调分发。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any, Mapping
|
||||
|
||||
|
||||
class TelegramPoller:
|
||||
def __init__(self, bot, actions, *, logger=None, sleeper=time.sleep) -> None:
|
||||
self.bot = bot
|
||||
self.actions = actions
|
||||
self.logger = logger
|
||||
self._sleep = sleeper
|
||||
|
||||
def _log(self, level: str, event: str, **fields) -> None:
|
||||
if self.logger is not None:
|
||||
getattr(self.logger, level)(event, **fields)
|
||||
|
||||
def run_once(self, offset: int | None = None) -> int | None:
|
||||
updates = self.bot.get_updates(offset)
|
||||
for update in updates:
|
||||
update_id = update.get("update_id")
|
||||
if isinstance(update_id, int) and not isinstance(update_id, bool):
|
||||
offset = max(offset or 0, update_id + 1)
|
||||
callback = update.get("callback_query")
|
||||
if isinstance(callback, Mapping):
|
||||
self.handle_callback(callback)
|
||||
return offset
|
||||
|
||||
def handle_callback(self, callback: Mapping[str, Any]) -> None:
|
||||
callback_id = str(callback.get("id") or "")
|
||||
message = callback.get("message") or {}
|
||||
chat_id = (message.get("chat") or {}).get("id")
|
||||
from_id = (callback.get("from") or {}).get("id")
|
||||
if chat_id != self.bot.chat_id or from_id != self.bot.chat_id:
|
||||
self._log(
|
||||
"warning", "telegram.unauthorized",
|
||||
chat_id=chat_id, from_id=from_id,
|
||||
)
|
||||
self._answer(callback_id, "无权限")
|
||||
return
|
||||
|
||||
action, _, raw_aid = str(callback.get("data") or "").partition(":")
|
||||
try:
|
||||
aid = int(raw_aid)
|
||||
except (TypeError, ValueError):
|
||||
self._answer(callback_id, "未知操作")
|
||||
return
|
||||
|
||||
if action == "fav":
|
||||
result = self.actions.favorite(aid)
|
||||
elif action == "ign":
|
||||
result = self.actions.ignore(aid)
|
||||
elif action == "blk":
|
||||
result = self.actions.blacklist(aid)
|
||||
else:
|
||||
self._answer(callback_id, "未知操作")
|
||||
return
|
||||
|
||||
text = result.message
|
||||
if result.ok and result.position is not None:
|
||||
text = f"{text}（位置 #{result.position}）"
|
||||
self._log(
|
||||
"info", "telegram.action",
|
||||
action=action, aid=aid, ok=result.ok, state=result.state,
|
||||
position=result.position,
|
||||
)
|
||||
self._answer(callback_id, text)
|
||||
|
||||
def _answer(self, callback_id: str, text: str) -> None:
|
||||
try:
|
||||
self.bot.answer_callback(callback_id, text)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._log("error", "telegram.answer_failed", error=str(exc))
|
||||
|
||||
def run_forever(self, stop_event=None) -> None:
|
||||
offset: int | None = None
|
||||
while stop_event is None or not stop_event.is_set():
|
||||
try:
|
||||
offset = self.run_once(offset)
|
||||
except Exception as exc: # noqa: BLE001 长轮询失败后退避重试
|
||||
self._log("error", "telegram.poll_failed", error=str(exc))
|
||||
self._sleep(3)
|
||||
Reference in new issue
Block a user