From 742c67a1c3c4c26a01381918aa6c63480fc49c12 Mon Sep 17 00:00:00 2001 From: kaireewu Date: Sun, 1 Nov 2026 19:00:48 +0800 Subject: [PATCH] feat(classify): tighten scoring with reaction co-occurrence and date negatives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 强正向 reaction 需标题同时含 require_title；移除听/看等宽泛词，新增弱正向组；charge 命中>=2 记+3、==1 记+1；负向补充录播/录像并新增标题带日期强负向；时长仅在已命中 reaction/charge 时加权；discover 关键词改为单个咻咻满、pages=3、window_days=7，notify.min_score=4；同步示例配置、project.md/README 与相关测试。 --- README.md | 2 +- config.docker.example.json | 35 +++++++-------- config.example.json | 29 ++++-------- project.md | 38 ++++++++++------ reaction_board/classify.py | 85 +++++++++++++++++++++++++++-------- reaction_board/config.py | 25 ++++++----- tests/test_classify.py | 91 +++++++++++++++++++++++++++++--------- tests/test_config.py | 6 ++- tests/test_discovery.py | 6 ++- 9 files changed, 210 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index 1b4f07f..47e24cf 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ | 视频入库 | 可用 | 搜索候选经 Telegram 确认入库；私聊发送视频链接、BV 号、av 号或 `b23.tv` 短链手动入库。只写本地库。 | | 视频日快照 | 可用（P1） | 逐条采集播放、点赞、投币、收藏、转发、评论、弹幕。 | | 粉丝追踪 | 可用（P1） | 每日记录咻咻满等配置 UP 的粉丝数。 | -| 搜索发现与候选池 | 可用（P3） | 每小时搜索前 2 页，分类打分后写入候选池。 | +| 搜索发现与候选池 | 可用（P3） | 每小时按 `discover` 配置搜索并分类打分后写入候选池。 | | Telegram 通知与入库 | 可用（P3） | 入库 / 忽略 / 拉黑按钮；私聊消息手动入库。 | | 本地数据看板 | 可用（P2/P9） | 只读总览、视频榜、UP 主榜、内容效率、视频/UP 详情、粉丝曲线、数据说明、候选池和系统状态。 | | 访问统计（PV/UV） | 可用 | 前端埋点上报，按客户端 IP 与页面去重；先落内存、每分钟批量入库；系统页展示近 30 天。 | diff --git a/config.docker.example.json b/config.docker.example.json index e8bae5b..7a3be25 100644 --- a/config.docker.example.json +++ b/config.docker.example.json @@ -67,33 +67,30 @@ "candidate_ttl_days": 7 }, "discover": { - "keywords": ["咻咻满 reaction", "咻咻满 充电"], - "pages": 2, - "window_days": 14 + "keywords": ["咻咻满"], + "pages": 3, + "window_days": 7 }, "notify": { - "min_score": 2, + "min_score": 4, "max_items_per_message": 10 }, "classify": { "require_title": "咻咻满", "self_mid": 37754047, - "positive": [ - "reaction", - "re", - "反应", - "看", - "听", - "初听", - "首次", - "点评", - "锐评", - "路人", - "震惊", - "双人" + "reaction": ["reaction", "反应"], + "charge": ["加急", "特急", "充电", "专场", "转场"], + "weak_positive": ["双人", "路人", "第一次", "首次"], + "negative": [ + "歌切", + "切片", + "纯享", + "搬运", + "完整版", + "录播", + "录像" ], - "charge": ["充电", "加急", "专场"], - "negative": ["歌切", "切片", "纯享", "搬运", "完整版"], + "date_negative": true, "duration_boost": { "gt_30min": 1, "gt_60min": 2 } }, "heat_weights": { diff --git a/config.example.json b/config.example.json index a108047..94f7fb0 100644 --- a/config.example.json +++ b/config.example.json @@ -77,33 +77,22 @@ "candidate_ttl_days": 7 }, "discover": { - "keywords": ["咻咻满 reaction", "咻咻满 充电"], - "pages": 2, - "window_days": 14 + "keywords": ["咻咻满"], + "pages": 3, + "window_days": 7 }, "notify": { - "min_score": 2, + "min_score": 4, "max_items_per_message": 10 }, "classify": { "require_title": "咻咻满", "self_mid": 37754047, - "positive": [ - "reaction", - "re", - "反应", - "看", - "听", - "初听", - "首次", - "点评", - "锐评", - "路人", - "震惊", - "双人" - ], - "charge": ["充电", "加急", "专场"], - "negative": ["歌切", "切片", "纯享", "搬运", "完整版"], + "reaction": ["reaction", "反应"], + "charge": ["加急", "特急", "充电", "专场", "转场"], + "weak_positive": ["双人", "路人", "第一次", "首次"], + "negative": ["歌切", "切片", "纯享", "搬运", "完整版", "录播", "录像"], + "date_negative": true, "duration_boost": { "gt_30min": 1, "gt_60min": 2 } }, "heat_weights": { diff --git a/project.md b/project.md index 7cdde9b..0c91163 100644 --- a/project.md +++ b/project.md @@ -118,12 +118,16 @@ Telegram 私聊发链接/BV 号 ───┘ ], "schedule": { "discover_minutes": 60, "snapshot_at": "00:01", "candidate_ttl_days": 7 }, - "notify": { "min_score": 2, "max_items_per_message": 10 }, + "discover": { "keywords": ["咻咻满"], "pages": 3, "window_days": 7 }, + "notify": { "min_score": 4, "max_items_per_message": 10 }, "classify": { "require_title": "咻咻满", - "positive": ["reaction", "re", "反应", "看", "听", "初听", "首次", "点评", "锐评", "路人", "震惊", "双人"], - "charge": ["充电", "加急", "专场"], - "negative": ["歌切", "切片", "纯享", "搬运", "完整版"], + "self_mid": 37754047, + "reaction": ["reaction", "反应"], + "charge": ["加急", "特急", "充电", "专场", "转场"], + "weak_positive": ["双人", "路人", "第一次", "首次"], + "negative": ["歌切", "切片", "纯享", "搬运", "完整版", "录播", "录像"], + "date_negative": true, "duration_boost": { "gt_30min": 1, "gt_60min": 2 } }, "heat_weights": { "view": 0.30, "like": 0.20, "coin": 0.20, @@ -257,24 +261,30 @@ title 含 "咻咻满"（全半角/大小写归一化后），否则直接丢弃 up_mid 命中黑名单 → 直接排除（永不通知，连候选池都不进） score = 0 -+2 标题含 reaction|re|反应|看|听|初听|首次|点评|锐评|路人|震惊|双人 -+2 标题含 充电|加急|专场 -+2 时长 > 60 分钟 -+1 时长 > 30 分钟（与上一条不叠加，取大） --5 标题含 歌切|切片|纯享|搬运|完整版 ++3 标题同时含 "咻咻满" 与 reaction|反应（强正向；只含 reaction 不成立） ++3 标题含 加急|特急|充电|专场|转场 中任意 >=2 个 ++1 标题含其中恰好 1 个 ++1 标题含 双人|路人|第一次|首次（弱正向） ++2 时长 > 60 分钟（仅当已命中 reaction 或 charge 时计入） ++1 时长 > 30 分钟（同上，与上一条不叠加，取大） +-6 标题含 歌切|切片|纯享|搬运|完整版|录播|录像 +-6 标题带日期（2026-10-28 / 2026/10/25 / 2026.10.26 / 2024年11月24日 / 10月25日） -3 UP 为 咻咻满本人（mid=37754047） -通知阈值 = 2 -0 <= score < 2 且含"咻咻满" → 进看板"候选池"（7 天后删除），不通知 +通知阈值 = 4 +0 <= score < 4 且含"咻咻满" → 进看板"候选池"（7 天后删除），不通知 score < 0 → 丢弃 ``` **几条设计依据**： -- 对现有 664 条收藏回测，**92.6% 的标题含 reaction 类词**，规则召回足够；误判代价只是"多发/少发一条通知"，最终由人点击决定； -- **"直播回放"不再算负向词**：确实有人先直播看、再发直播回放，这类仍是 reaction；改由"UP 黑名单"处理专发回放/切片的号； -- **时长是强正向信号**：实测这批 reaction 视频时长普遍 100~336 分钟（`105:05`、`158:24`、`336:31`），30/60 分钟阈值能把大多数切片（十几分钟）区分开； +- **强正向必须"咻咻满 + reaction"同现**：B 站搜索会命中标题根本不含关键词的结果（简介/标签/相似内容），所以"标题含 reaction"本身不足以判定为高置信，必须与 `require_title` 同时成立； +- **"听/看"等宽泛词已移除**：几乎所有 reaction 标题都含，无区分度； +- **时长只在已命中 reaction/charge 时加权**：避免"完整录播/直播录像"这类既无 reaction 也无 charge 的长视频仅靠时长进池； +- **带日期的标题按强负向处理**：`2026-10-28`、`10月25日` 这类通常对应某场直播的录播/回放/录像/自用收藏；实测对真 reaction 标题零误伤，代价是极少数"真 reaction 恰好写日期"会被漏掉； +- **`录播`/`录像` 计负向**：`完整版` 覆盖不到 `完整录播`；`录像` 覆盖 `直播录像`。裸 `回放` 不列入，因为 `【回放/咻咻满reaction】` 之类确为 reaction； - 时长直接取**搜索结果自带的 `duration` 字段**（格式是 `分钟:秒`，超过 60 分钟写作 `105:5`），**不需要额外请求**； +- 关键词用单个 `咻咻满`（`discover.pages=3` 覆盖 7 天窗口），因此 `require_title` 是过滤非相关结果的关键兜底； - UP 黑名单在配置文件里维护，可热加载，后续从"误通知"记录里一键加入。 ### 3.4 通知内容 diff --git a/reaction_board/classify.py b/reaction_board/classify.py index e0c851f..b1ef8a8 100644 --- a/reaction_board/classify.py +++ b/reaction_board/classify.py @@ -47,11 +47,16 @@ def parse_duration(value: Any) -> int: return 0 -def contains_any(title: str, words: Any) -> bool: +def matched_words(title: str, words: Any) -> list[str]: + """返回标题命中（归一化子串）的词，保持 ``words`` 顺序、跳过空词。""" normalized = normalize_title(title) if not isinstance(words, (list, tuple)): - return False - return any(normalize_title(str(word)) in normalized for word in words if word) + return [] + return [str(word) for word in words if word and normalize_title(str(word)) in normalized] + + +def contains_any(title: str, words: Any) -> bool: + return bool(matched_words(title, words)) def passes_required_title(title: str, require_title: str) -> bool: @@ -60,6 +65,30 @@ def passes_required_title(title: str, require_title: str) -> bool: return normalize_title(require_title) in normalize_title(title) +# 标题带日期通常意味着录播/回放/录像（按具体某场直播命名），作为强负向信号。 +# 覆盖 `2026-10-28` / `2026/10/25` / `2026.10.26` / `2024年11月24日` / `10月25日`。 +_DATE_PATTERNS = ( + r"\d{4}\s*[-/.]\s*\d{1,2}\s*[-/.]\s*\d{1,2}", + r"\d{4}\s*年\s*\d{1,2}\s*月(?:\s*\d{1,2}\s*日)?", + r"\d{1,2}\s*月\s*\d{1,2}\s*日", +) +_DATE_RE = re.compile("|".join(_DATE_PATTERNS)) + +# 各信号的权重（见 project.md §3.3）。 +REACTION_SCORE = 3 +CHARGE_MULTI_SCORE = 3 +CHARGE_SINGLE_SCORE = 1 +WEAK_SCORE = 1 +NEGATIVE_SCORE = 6 +DATE_SCORE = 6 +SELF_SCORE = 3 + + +def date_hits(title: str) -> list[str]: + """返回标题中的日期片段，用于判定「带日期≈录播/回放」。""" + return _DATE_RE.findall(title or "") + + def score_item( *, title: str, @@ -67,23 +96,43 @@ def score_item( duration_seconds: int, classify: Mapping[str, Any], ) -> int: - """按 project.md §3.3 计算候选分数。""" + """按 project.md §3.3 计算候选分数。 + + 强正向 ``reaction`` 只在标题同时命中 ``require_title`` 时成立；时长只作为 + 已存在 reaction / charge 信号时的加权，避免纯长录像仅靠时长进池。 + """ score = 0 - if contains_any(title, classify.get("positive")): - score += 2 - if contains_any(title, classify.get("charge")): - score += 2 + require_title = str(classify.get("require_title") or "") - boost = classify.get("duration_boost") or {} - gt_60 = int(boost.get("gt_60min", 2)) - gt_30 = int(boost.get("gt_30min", 1)) - if duration_seconds > 60 * 60: - score += gt_60 - elif duration_seconds > 30 * 60: - score += gt_30 + reaction_matched = bool(matched_words(title, classify.get("reaction"))) + reaction_hit = reaction_matched and ( + not require_title or passes_required_title(title, require_title) + ) + if reaction_hit: + score += REACTION_SCORE - if contains_any(title, classify.get("negative")): - score -= 5 + charge_hits = matched_words(title, classify.get("charge")) + if len(charge_hits) >= 2: + score += CHARGE_MULTI_SCORE + elif len(charge_hits) == 1: + score += CHARGE_SINGLE_SCORE + + if matched_words(title, classify.get("weak_positive")): + score += WEAK_SCORE + + if reaction_hit or charge_hits: + boost = classify.get("duration_boost") or {} + gt_60 = int(boost.get("gt_60min", 2)) + gt_30 = int(boost.get("gt_30min", 1)) + if duration_seconds > 60 * 60: + score += gt_60 + elif duration_seconds > 30 * 60: + score += gt_30 + + if matched_words(title, classify.get("negative")): + score -= NEGATIVE_SCORE + if classify.get("date_negative", True) and date_hits(title): + score -= DATE_SCORE if up_mid is not None and int(up_mid) == int(classify.get("self_mid", 37754047)): - score -= 3 + score -= SELF_SCORE return score diff --git a/reaction_board/config.py b/reaction_board/config.py index b6455c1..46fbbce 100644 --- a/reaction_board/config.py +++ b/reaction_board/config.py @@ -25,12 +25,11 @@ from .client_ip import ( DEFAULT_CLASSIFY: dict[str, Any] = { "require_title": "咻咻满", "self_mid": 37754047, - "positive": [ - "reaction", "re", "反应", "看", "听", "初听", "首次", - "点评", "锐评", "路人", "震惊", "双人", - ], - "charge": ["充电", "加急", "专场"], - "negative": ["歌切", "切片", "纯享", "搬运", "完整版"], + "reaction": ["reaction", "反应"], + "charge": ["加急", "特急", "充电", "专场", "转场"], + "weak_positive": ["双人", "路人", "第一次", "首次"], + "negative": ["歌切", "切片", "纯享", "搬运", "完整版", "录播", "录像"], + "date_negative": True, "duration_boost": {"gt_30min": 1, "gt_60min": 2}, } @@ -87,11 +86,11 @@ DEFAULT_CONFIG: dict[str, Any] = { "candidate_ttl_days": 7, }, "discover": { - "keywords": ["咻咻满 reaction", "咻咻满 充电"], - "pages": 2, - "window_days": 14, + "keywords": ["咻咻满"], + "pages": 3, + "window_days": 7, }, - "notify": {"min_score": 2, "max_items_per_message": 10}, + "notify": {"min_score": 4, "max_items_per_message": 10}, "classify": DEFAULT_CLASSIFY, "heat_weights": DEFAULT_HEAT_WEIGHTS, "analytics": DEFAULT_ANALYTICS, @@ -500,8 +499,12 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig: classify["self_mid"] = _as_int( classify.get("self_mid", 37754047), "classify.self_mid", minimum=1 ) - for key in ("positive", "charge", "negative"): + for key in ("reaction", "charge", "weak_positive", "negative"): classify[key] = _as_str_list(classify.get(key, []), f"classify.{key}") + date_negative = classify.get("date_negative", True) + if not isinstance(date_negative, bool): + raise ConfigError("classify.date_negative 必须是布尔值") + classify["date_negative"] = date_negative boost = classify.get("duration_boost", {}) if not isinstance(boost, Mapping): raise ConfigError("classify.duration_boost 必须是对象") diff --git a/tests/test_classify.py b/tests/test_classify.py index 5803bc2..742707e 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -10,6 +10,8 @@ sys.stdout.reconfigure(encoding="utf-8") from reaction_board.classify import ( # noqa: E402 clean_title, contains_any, + date_hits, + matched_words, normalize_title, parse_duration, passes_required_title, @@ -19,13 +21,22 @@ from reaction_board.classify import ( # noqa: E402 CLASSIFY = { "require_title": "咻咻满", "self_mid": 37754047, - "positive": ["reaction", "re", "反应", "看", "听", "初听", "首次", "点评", "锐评", "路人", "震惊", "双人"], - "charge": ["充电", "加急", "专场"], - "negative": ["歌切", "切片", "纯享", "搬运", "完整版"], + "reaction": ["reaction", "反应"], + "charge": ["加急", "特急", "充电", "专场", "转场"], + "weak_positive": ["双人", "路人", "第一次", "首次"], + "negative": ["歌切", "切片", "纯享", "搬运", "完整版", "录播", "录像"], + "date_negative": True, "duration_boost": {"gt_30min": 1, "gt_60min": 2}, } +def score(title, up_mid=100, duration=0, classify=CLASSIFY): + return score_item( + title=title, up_mid=up_mid, + duration_seconds=duration, classify=classify, + ) + + def test_title_helpers(): assert normalize_title("咻咻满 REACTION") == "咻咻满 reaction" assert clean_title('咻咻满 reaction') == "咻咻满 reaction" @@ -34,6 +45,8 @@ def test_title_helpers(): assert not passes_required_title("其他 UP 的 reaction", "咻咻满") assert contains_any("咻咻满 REACTION", ["reaction"]) assert not contains_any("咻咻满", []) + assert matched_words("咻咻满 REACTION 反应", ["reaction", "反应"]) == ["reaction", "反应"] + assert matched_words("咻咻满", []) == [] def test_parse_duration(): @@ -46,32 +59,70 @@ def test_parse_duration(): assert parse_duration(-5) == 0 -def test_score_rules(): - def score(title, up_mid=100, duration=0): - return score_item( - title=title, up_mid=up_mid, - duration_seconds=duration, classify=CLASSIFY, - ) +def test_date_hits(): + assert date_hits("咻咻满-2026-10-28-素颜满") == ["2026-10-28"] + assert date_hits("【咻咻满】10月25日直播片段") == ["10月25日"] + assert date_hits("咻咻满 ☆ 2026.10.26") == ["2026.10.26"] + assert date_hits("【咻咻满】2026/10/25直播回放") == ["2026/10/25"] + assert date_hits("咻咻满：《发如雪》2024年10月5日直播歌切") == ["2024年10月5日"] + assert date_hits("咻咻满 reaction") == [] - assert score("咻咻满 reaction", duration=60) == 2 - assert score("咻咻满 充电", duration=60) == 2 + +def test_score_strong_reaction_requires_title(): + # 强正向 reaction 只在标题含 require_title 时成立。 + assert score("咻咻满 reaction", duration=60) == 3 + assert score("咻咻满 反应", duration=60) == 3 + assert score("其他 UP 的 reaction", duration=61 * 60) == 0 + + +def test_score_charge_levels(): + assert score("咻咻满 充电", duration=60) == 1 + assert score("咻咻满 加急专场", duration=60) == 3 assert score("咻咻满 reaction 充电", duration=60) == 4 - assert score("咻咻满 reaction", duration=61 * 60) == 4 - assert score("咻咻满 reaction", duration=31 * 60) == 3 - assert score("咻咻满 reaction", duration=30 * 60) == 2 - assert score("咻咻满 切片", duration=60) == -5 + assert score("咻咻满 reaction 加急充电专场", duration=60) == 6 + + +def test_score_weak_positive(): + assert score("咻咻满 双人", duration=60) == 1 + assert score("咻咻满 reaction 双人", duration=60) == 4 + + +def test_duration_only_counts_with_signal(): + assert score("咻咻满 reaction", duration=61 * 60) == 5 + assert score("咻咻满 reaction", duration=31 * 60) == 4 + assert score("咻咻满 reaction", duration=30 * 60) == 3 + # 无 reaction / charge 信号时，时长不给分（避免纯长录像进池）。 + assert score("咻咻满", duration=61 * 60) == 0 + assert score("咻咻满 双人", duration=61 * 60) == 1 + + +def test_score_negative_and_self(): + assert score("咻咻满 切片", duration=60) == -6 assert score("咻咻满 reaction 切片", duration=60) == -3 - assert score("咻咻满 reaction", up_mid=37754047, duration=60) == -1 - assert score("咻咻满", duration=0) == 0 - assert score("咻咻满", duration=61 * 60) == 2 + assert score("咻咻满 reaction 直播录像", duration=60) == -3 + assert score("咻咻满 reaction", up_mid=37754047, duration=60) == 0 + + +def test_score_date_negative(): + assert score("咻咻满 reaction 2026-10-28 完整录播", duration=60) == -9 + assert score("咻咻满 reaction 2026-10-28", duration=60) == -3 + assert score("咻咻满 充电 10月25日", duration=60) == -5 + disabled = dict(CLASSIFY, date_negative=False) + assert score("咻咻满 reaction 2026-10-28", duration=60, classify=disabled) == 3 def main(): test_title_helpers() test_parse_duration() - test_score_rules() + test_date_hits() + test_score_strong_reaction_requires_title() + test_score_charge_levels() + test_score_weak_positive() + test_duration_only_counts_with_signal() + test_score_negative_and_self() + test_score_date_negative() print("OK: 候选标题归一化、时长解析与分类打分测试全部通过") if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/tests/test_config.py b/tests/test_config.py index 5066b91..91d771a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -54,9 +54,11 @@ def test_dual_accounts_and_env_override(): assert (cfg.bilibili.write.mid, cfg.bilibili.write.csrf, cfg.bilibili.write.cookie) == (202, "env-write", "env-write-cookie") assert cfg.storage.database_path == Path(tmp) / "custom.db" assert cfg.up_blacklist_seed == (1, 2) - assert cfg.discover.keywords == ("咻咻满 reaction", "咻咻满 充电") - assert cfg.discover.pages == 2 + assert cfg.discover.keywords == ("咻咻满",) + assert cfg.discover.pages == 3 assert cfg.classify["self_mid"] == 37754047 + assert cfg.classify["reaction"] == ["reaction", "反应"] + assert cfg.classify["date_negative"] is True def test_flat_and_missing_account_rejected(): diff --git a/tests/test_discovery.py b/tests/test_discovery.py index dd1a53a..70121b2 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -40,12 +40,14 @@ def write_config(tmp, **discover): "classify": { "require_title": "咻咻满", "self_mid": 37754047, - "positive": ["reaction"], + "reaction": ["reaction"], "charge": ["充电"], + "weak_positive": [], "negative": ["切片"], + "date_negative": True, "duration_boost": {"gt_30min": 1, "gt_60min": 2}, }, - "notify": {"min_score": 2, "max_items_per_message": 10}, + "notify": {"min_score": 3, "max_items_per_message": 10}, } path = Path(tmp) / "config.json" path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")