diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 262d571..af854d2 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -602,12 +602,14 @@ interaction_rate = (like + coin + favorite) / max(view, 1) ```text 内容小时 = duration / 3600 -内容效率 = view / 内容小时 -互动效率 = (like + coin + favorite) / 内容小时 +内容效率 = (view + κ_v·μ₀) / (内容小时 + κ_v) +互动效率 = (like + coin + favorite + κ_v·μ₀ᵢ) / (内容小时 + κ_v) UP 内容效率 = (Σview + κ·μ₀) / (Σ内容分钟 + κ) * 60 ``` - 分母只用总时长，不用分 P 数（B 站播放记在 bvid 上，`view/pages` 无统计含义）； +- 视频级 EB 收缩 `κ_v = analytics.video_shrink_minutes / 60`（默认 60 分钟），`μ₀` 为全库播放/小时；详情页展示原始播放/小时； +- 效率榜与综合热度榜要求播放 ≥ `analytics.min_view_for_ranking`（默认 2000）； - 效率榜要求 `duration >= analytics.min_duration_seconds`（默认 600 秒）且发布满 `analytics.maturity_days`（默认 2 天）； - 不引入 `log(age)` 等无界年龄项，避免系统性惩罚老视频； - `duration` 缺失或非正数不参与效率排序； @@ -735,7 +737,7 @@ uv run python tests/test_moves.py - 收藏夹同步结束时采集缺失 UP 的粉丝数（`relation/stat`，匿名可用）；`first_seen >= 运行开始 - 1h` 记为实时，更早的记为回填； - `app.py backfill-followers`：一次性回填历史视频粉丝数，支持 `--limit` / `--dry-run`； - 新增内容效率/互动效率榜与 UP 内容效率（Gamma–Poisson EB 收缩），过滤 < 600 秒与未满 2 天的视频； -- `analytics` 配置段：`min_duration_seconds`、`maturity_days`、`min_view_for_efficiency`、`shrink_minutes`； +- `analytics` 配置段：`min_duration_seconds`、`maturity_days`、`min_view_for_ranking`、`video_shrink_minutes`、`shrink_minutes`； - 主导航拆分：`/videos`（视频）与 `/ups`（UP主）独立；旧 `/rank` 保留兼容； - 新增 `/about` 数据说明页与 `/api/about`； - 卡片精简：列表只保留主指标与互动率，明细移到详情页； diff --git a/README.md b/README.md index ef77a09..d5cb27f 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ uv run python app.py serve --preview --config data/preview/config.json - 首页默认“最新动态”：今日增量、最新入库、最近发布、增长动态、热度速览、收藏夹容量与采集状态； - 「视频」与「UP主」是两个独立入口：视频榜支持内容效率（默认）、综合热度、播放/投币/收藏/评论、互动率、播放增量/播放增速；UP 榜展示视频数、总时长、总播放、播放·小时、互动·小时等，可分别排序； -- 内容效率 = 播放 / 内容小时，只统计时长 ≥ 600 秒且发布满 2 天的视频；UP 内容效率用经验贝叶斯收缩，避免极小样本冲榜； +- 内容效率 = 播放 / 内容小时；单条视频用 60 分钟先验暴露量向全库均值收缩，只统计时长 ≥ 600 秒、发布满 2 天、播放 ≥ 2000 的视频（综合热度榜同样要求播放 ≥ 2000）；UP 内容效率用经验贝叶斯收缩，避免极小样本冲榜； - 「数据说明」（`/about`）说明数据来源、发布时间/入库时间/快照日期、指标公式、筛选口径与已知局限，数值由 `/api/about` 动态读取； - 视频详情提供分 P 数、七项指标、内容/互动效率、入库时粉丝数、单指标趋势、快照表和同 UP 对比；UP 页提供聚合趋势、单视频对比与入库时/当前粉丝数； - 粉丝页只绘制粉丝曲线，不叠加视频数据；支持 7/30/90/全部与自选日期，范围超过一年时按每月峰值日期聚合；最近记录默认 14 天并显示较上一成功日的增量； diff --git a/config.docker.example.json b/config.docker.example.json index 50df8b1..5b992c7 100644 --- a/config.docker.example.json +++ b/config.docker.example.json @@ -92,7 +92,8 @@ "analytics": { "min_duration_seconds": 600, "maturity_days": 2, - "min_view_for_efficiency": 0, + "min_view_for_ranking": 2000, + "video_shrink_minutes": 60, "shrink_minutes": null } } \ No newline at end of file diff --git a/config.example.json b/config.example.json index bc47643..9c13357 100644 --- a/config.example.json +++ b/config.example.json @@ -102,7 +102,8 @@ "analytics": { "min_duration_seconds": 600, "maturity_days": 2, - "min_view_for_efficiency": 0, + "min_view_for_ranking": 2000, + "video_shrink_minutes": 60, "shrink_minutes": null } } diff --git a/project.md b/project.md index 67fea3e..8984683 100644 --- a/project.md +++ b/project.md @@ -676,12 +676,15 @@ Reaction 视频时长差异极大（100~336 分钟），而播放是累计计数 ``` 内容小时 = duration / 3600 # 多 P 的 duration 已是各 P 之和 -内容效率 = view / 内容小时 -互动效率 = (like + coin + favorite) / 内容小时 +内容效率 = (view + κ_v·μ₀) / (内容小时 + κ_v) # 榜单口径（收缩后） +原始效率 = view / 内容小时 # 详情页口径 +互动效率 = (互动 + κ_v·μ₀ᵢ) / (内容小时 + κ_v) UP 内容效率 = EB 收缩后的 Σview / Σ内容小时 ``` - 分母只用总时长，**不用分 P 数**：B 站播放记在 bvid 上，`view / pages` 没有统计含义；`pages` 仅用于详情展示与包装诊断； +- **视频级也做 EB 收缩**：`κ_v` 默认 60 分钟（配置 `analytics.video_shrink_minutes`），`μ₀` 为全库播放/小时。短时长视频的暴露量小、速率估计噪声大，收缩后不再虚高；2–3 小时的长视频几乎不受影响； +- 效率榜与综合热度榜只统计**播放 ≥ `analytics.min_view_for_ranking`（默认 2000）** 的视频；播放过低的视频多为小样本噪声，不上榜； - 效率榜只统计 `duration >= analytics.min_duration_seconds`（默认 600 秒）且发布满 `analytics.maturity_days`（默认 2 天）的视频；不足者标注"样本不足"； - **不用无界年龄项**：reaction 视频通常在发布前两天完成数据积累、两周内接近平台期，`log(age)` 会系统性惩罚老视频，故采用"成熟窗口筛选"而非年龄回归； - `duration` 为空或 ≤ 0（通常只在失效视频出现）不进入效率榜，不插补； @@ -839,6 +842,7 @@ storage：SQLite WAL，单写线程 | 39 | **粉丝快照回填（真实性）** | 预览库 488 个 UP、796/796 视频补齐 `up_follower`，0 失败；历史视频标记 `backfilled=1` | | 40 | **导航拆分与数据说明页** | `/videos`、`/ups`、`/about` 均返回 200；`/api/about` 返回 schema 4、当前阈值、权重与粉丝覆盖度 | | 41 | **全量离线测试** | 32 个 `tests/test_*.py` 全部通过（含新增 `test_up_profile.py` 与迁移/数据库断言更新） | +| 42 | **内容效率噪声修正** | 预览库实测：视频级 EB 收缩（κ=60 分钟）+ 播放 ≥ 2000 后，效率榜 Top10 全为 ≥110 分钟视频；「什么哦吼吼」4 条不再入榜；综合热度榜同样应用 2000 门槛（637 条） | **未验证（需授权或外部配合）**： diff --git a/reaction_board/config.py b/reaction_board/config.py index f23ef40..72362cc 100644 --- a/reaction_board/config.py +++ b/reaction_board/config.py @@ -41,7 +41,8 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = { DEFAULT_ANALYTICS: dict[str, Any] = { "min_duration_seconds": 600, "maturity_days": 2, - "min_view_for_efficiency": 0, + "min_view_for_ranking": 2000, + "video_shrink_minutes": 60, "shrink_minutes": None, } @@ -173,7 +174,8 @@ class DiscoverConfig: class AnalyticsConfig: min_duration_seconds: int maturity_days: int - min_view_for_efficiency: int + min_view_for_ranking: int + video_shrink_minutes: int shrink_minutes: int | None @@ -463,9 +465,14 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig: "analytics.maturity_days", minimum=0, ), - min_view_for_efficiency=_as_int( - analytics_raw.get("min_view_for_efficiency", 0), - "analytics.min_view_for_efficiency", + min_view_for_ranking=_as_int( + analytics_raw.get("min_view_for_ranking", 2000), + "analytics.min_view_for_ranking", + minimum=0, + ), + video_shrink_minutes=_as_int( + analytics_raw.get("video_shrink_minutes", 60), + "analytics.video_shrink_minutes", minimum=0, ), shrink_minutes=( diff --git a/reaction_board/web/queries.py b/reaction_board/web/queries.py index 956d467..973b968 100644 --- a/reaction_board/web/queries.py +++ b/reaction_board/web/queries.py @@ -699,6 +699,7 @@ class DashboardQueries: latest = self._latest_date(conn) or self._today() analytics = self.config_store.get().analytics duration_sorts = {"efficiency", "interaction_efficiency"} + min_view_for_ranking = max(0, int(analytics.min_view_for_ranking)) within_field = None if within_days is not None: if within_days < 1: @@ -716,7 +717,7 @@ class DashboardQueries: ) aids = [int(video["aid"]) for video in videos] snapshots = self._snapshot_map(conn, aids) - _, _, heat, percentiles = self._global_heat(conn) + videos_all, snapshots_all, heat, percentiles = self._global_heat(conn) previous: dict[int, dict[str, Any]] = {} if sort in {"growth", "growth_rate", "interaction"}: @@ -740,16 +741,60 @@ class DashboardQueries: if sort in duration_sorts: latest_start, _ = day_bounds(latest) minimum_seconds = max(0, int(analytics.min_duration_seconds)) - min_view = max(0, int(analytics.min_view_for_efficiency)) items = [ item for item in items if item["metrics"] is not None and int(item["duration"] or 0) >= minimum_seconds - and int((item["metrics"] or {}).get("view", 0)) >= min_view + and int((item["metrics"] or {}).get("view", 0)) >= min_view_for_ranking and is_mature( item["pubdate"], latest_start, analytics.maturity_days ) ] + kappa_hours = max(0, int(analytics.video_shrink_minutes)) / 60.0 + total_hours = 0.0 + total_views = 0.0 + total_interactions = 0.0 + for video in videos_all: + hours = content_hours(video.get("duration")) + if hours is None: + continue + metrics = _metric_map(snapshots_all.get(int(video["aid"]))) + if metrics is None: + continue + total_hours += hours + total_views += metrics["view"] + total_interactions += ( + metrics["like"] + metrics["coin"] + metrics["favorite"] + ) + prior_views = total_views / total_hours if total_hours > 0 else 0.0 + prior_interactions = ( + total_interactions / total_hours if total_hours > 0 else 0.0 + ) + for item in items: + hours = content_hours(item.get("duration")) or 0.0 + metrics = item["metrics"] or {} + denominator = hours + kappa_hours + if hours <= 0 or denominator <= 0: + item["efficiency_score"] = None + item["interaction_efficiency_score"] = None + continue + view = int(metrics.get("view", 0)) + interactions = ( + int(metrics.get("like", 0)) + + int(metrics.get("coin", 0)) + + int(metrics.get("favorite", 0)) + ) + item["efficiency_score"] = ( + view + kappa_hours * prior_views + ) / denominator + item["interaction_efficiency_score"] = ( + interactions + kappa_hours * prior_interactions + ) / denominator + if sort == "heat": + items = [ + item for item in items + if int((item["metrics"] or {}).get("view", 0)) >= min_view_for_ranking + ] if sort == "interaction": items = [item for item in items if item["interaction_rate"] is not None] if sort in {"growth", "growth_rate"}: @@ -773,7 +818,11 @@ class DashboardQueries: if sort == "interaction": return item["interaction_rate"] if item["interaction_rate"] is not None else float("-inf") if sort in {"efficiency", "interaction_efficiency"}: - value = item.get(sort) + field = ( + "efficiency_score" if sort == "efficiency" + else "interaction_efficiency_score" + ) + value = item.get(field) return value if value is not None else float("-inf") if sort in {"growth", "growth_rate"}: field = "delta" if sort == "growth" else "rate" @@ -800,13 +849,15 @@ class DashboardQueries: duration_filter = { "min_duration_seconds": max(0, int(analytics.min_duration_seconds)), "maturity_days": max(0, int(analytics.maturity_days)), - "min_view": max(0, int(analytics.min_view_for_efficiency)), + "min_view": min_view_for_ranking, + "shrink_minutes": max(0, int(analytics.video_shrink_minutes)), } if not items: available = False reason = ( f"仅统计时长 ≥ {duration_filter['min_duration_seconds']} 秒、" - f"发布满 {duration_filter['maturity_days']} 天的视频" + f"发布满 {duration_filter['maturity_days']} 天、" + f"播放 ≥ {duration_filter['min_view']} 的视频" ) return { "sort": sort, @@ -1425,7 +1476,8 @@ class DashboardQueries: "analytics": { "min_duration_seconds": cfg.analytics.min_duration_seconds, "maturity_days": cfg.analytics.maturity_days, - "min_view_for_efficiency": cfg.analytics.min_view_for_efficiency, + "min_view_for_ranking": cfg.analytics.min_view_for_ranking, + "video_shrink_minutes": cfg.analytics.video_shrink_minutes, "shrink_minutes": cfg.analytics.shrink_minutes, }, "heat_weights": { diff --git a/reaction_board/web/static/views/about.js b/reaction_board/web/static/views/about.js index 1803835..16b6470 100644 --- a/reaction_board/web/static/views/about.js +++ b/reaction_board/web/static/views/about.js @@ -51,8 +51,8 @@ export async function renderAbout(root) {
  • 互动：点赞 + 投币 + 收藏。综合热度也使用全部七项指标，按配置权重加权。
  • 互动率：(点赞 + 投币 + 收藏) / 播放；播放不足 100 不计算。
  • 综合热度：每项先 log1p，再按中位数与 IQR 做稳健标准化，最后加权求和。分数只用于同一批数据内排序，不能跨日期比较。
    • -
  • 内容效率：播放 / 内容小时。内容小时 = 视频总时长（多 P 为各 P 之和）÷ 3600。
    • -
  • 互动效率：互动 / 内容小时，互动口径同上。
    • +
  • 内容效率：播放 / 内容小时。内容小时 = 视频总时长（多 P 为各 P 之和）÷ 3600。榜单里的单条视频用收缩后的估计：(播放 + κ·μ₀) / (小时 + κ)，κ 默认 ${analytics.video_shrink_minutes} 分钟、μ₀ 为全库播放/小时；详情页保留原始播放/小时。
    • +
  • 互动效率：互动 / 内容小时，互动口径与收缩方式同上。
  • UP 内容效率：把该 UP 的全部视频合并后计算，并用 Gamma–Poisson 经验贝叶斯向全库均值收缩，避免单条视频或极小样本冲榜。收缩强度 κ ${analytics.shrink_minutes ? `固定为 ${analytics.shrink_minutes} 分钟` : "取全库 UP 总内容分钟的中位数"}。
  • 累计触达：总播放。它是规模指标，不代表吸引力；高产 UP 排在前列是正常结果。
  • 分 P：pages 是分 P 数，duration 是该 bvid 各 P 时长之和。B 站不公开单 P 播放，因此没有单 P 榜；分 P 数只用于详情展示，不参与排序。
    • @@ -66,7 +66,7 @@ export async function renderAbout(root) {
  • 只统计发布满 ${analytics.maturity_days} 天的视频；reaction 视频通常在发布前两天完成数据积累，两周内接近平台期，过早比较会系统性低估新视频。
  • 不引入无界的“年龄惩罚”：超过成熟窗口后不再按天加分或扣分。
  • duration 为空或 ≤ 0 的视频（通常只在失效视频上出现）不进入任何时长相关榜，也不做插补。
    • -
  • 当前配置的最小播放门槛为 ${analytics.min_view_for_efficiency}。
    • +
  • 播放低于 ${analytics.min_view_for_ranking} 的视频不进入内容效率、互动效率与综合热度榜；这是为了避免小样本噪声冲榜。
    • diff --git a/reaction_board/web/static/views/rankings.js b/reaction_board/web/static/views/rankings.js index 6e0eef4..f2eaa1c 100644 --- a/reaction_board/web/static/views/rankings.js +++ b/reaction_board/web/static/views/rankings.js @@ -83,10 +83,10 @@ function controls({ scope, sort, within, growthDays, q, folder, minVideos, folde function videoPrimary(item, sort) { if (sort === "efficiency") { - return { primary: "efficiency", label: "播放/小时", text: formatCount(item.efficiency) }; + return { primary: "efficiency_score", label: "播放/小时·收缩", text: formatCount(item.efficiency_score) }; } if (sort === "interaction_efficiency") { - return { primary: "interaction_efficiency", label: "互动/小时", text: formatCount(item.interaction_efficiency) }; + return { primary: "interaction_efficiency_score", label: "互动/小时·收缩", text: formatCount(item.interaction_efficiency_score) }; } if (sort === "interaction") { return { primary: "view", label: "互动率", text: formatPercent(item.interaction_rate) }; @@ -143,8 +143,11 @@ function listHtml(data, scope, sort, startRank) { function durationNote(data) { const filter = data.duration_filter; if (!filter) return ""; + const shrink = filter.shrink_minutes + ? `并按 ${filter.shrink_minutes} 分钟的先验暴露量向全库均值收缩` + : ""; return `
    内容效率与互动效率仅统计时长 ≥ ${filter.min_duration_seconds} 秒、 - 发布满 ${filter.maturity_days} 天的视频；时长缺失的视频不参与排序。
    `; + 发布满 ${filter.maturity_days} 天、播放 ≥ ${filter.min_view} 的视频${shrink}；时长缺失的视频不参与排序。`; } function bindControls(root, scope) { diff --git a/reaction_board/web/static/views/video.js b/reaction_board/web/static/views/video.js index bc6eeb8..36f5cb0 100644 --- a/reaction_board/web/static/views/video.js +++ b/reaction_board/web/static/views/video.js @@ -77,8 +77,8 @@ export async function renderVideo(root, { params, query }) {
    综合热度${formatScore(detail.heat)}
    全库百分位${detail.heat_percentile === null || detail.heat_percentile === undefined ? "—" : `前 ${Math.max(0, 100 - detail.heat_percentile).toFixed(0)}%`}
    播放日增${detail.growth?.available ? formatDelta(detail.growth.delta) : "—"}
    -
    内容效率${detail.efficiency == null ? "—" : `${formatCount(detail.efficiency)}/小时`}
    -
    互动效率${detail.interaction_efficiency == null ? "—" : `${formatCount(detail.interaction_efficiency)}/小时`}
    +
    内容效率（原始）${detail.efficiency == null ? "—" : `${formatCount(detail.efficiency)}/小时`}
    +
    互动效率（原始）${detail.interaction_efficiency == null ? "—" : `${formatCount(detail.interaction_efficiency)}/小时`}
    UP 粉丝（入库时）${formatCount(detail.up_follower)}
    快照数${samples}
    最近快照${escapeHtml(detail.snapshot_date || "—")}
    diff --git a/tests/test_config.py b/tests/test_config.py index 91a4198..810d2e9 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -133,27 +133,31 @@ def test_analytics_defaults_and_validation(): cfg = load_config(path, env={}) assert cfg.analytics.min_duration_seconds == 600 assert cfg.analytics.maturity_days == 2 - assert cfg.analytics.min_view_for_efficiency == 0 + assert cfg.analytics.min_view_for_ranking == 2000 + assert cfg.analytics.video_shrink_minutes == 60 assert cfg.analytics.shrink_minutes is None data = base_config() data["analytics"] = { "min_duration_seconds": 300, "maturity_days": 7, - "min_view_for_efficiency": 100, + "min_view_for_ranking": 100, + "video_shrink_minutes": 30, "shrink_minutes": 90, } write(path, data) cfg = load_config(path, env={}) assert cfg.analytics.min_duration_seconds == 300 assert cfg.analytics.maturity_days == 7 - assert cfg.analytics.min_view_for_efficiency == 100 + assert cfg.analytics.min_view_for_ranking == 100 + assert cfg.analytics.video_shrink_minutes == 30 assert cfg.analytics.shrink_minutes == 90 for bad, hint in ( ({"min_duration_seconds": -1}, "min_duration_seconds"), ({"maturity_days": -1}, "maturity_days"), - ({"min_view_for_efficiency": -1}, "min_view_for_efficiency"), + ({"min_view_for_ranking": -1}, "min_view_for_ranking"), + ({"video_shrink_minutes": -1}, "video_shrink_minutes"), ({"shrink_minutes": 0}, "shrink_minutes"), ({"shrink_minutes": "auto"}, "shrink_minutes"), ): diff --git a/tests/test_web_queries.py b/tests/test_web_queries.py index 6e749c0..36bdbb6 100644 --- a/tests/test_web_queries.py +++ b/tests/test_web_queries.py @@ -65,8 +65,8 @@ def test_video_listing_filters_and_sorting(): heat = q.list_videos(sort="heat") aids = [item["aid"] for item in heat["items"]] assert 104 not in aids, "removed 视频不应进入榜单" - assert aids[0] == 102, aids - assert heat["items"][0]["heat"] > heat["items"][-1]["heat"] + # 综合热度榜应用最小播放门槛 2000：101（1200）与 103（120）被排除 + assert aids == [102], aids assert heat["items"][0]["heat_percentile"] == 100.0 views = q.list_videos(sort="view") @@ -362,18 +362,27 @@ def test_duration_efficiency_queries(): DASHBOARD_CACHE.clear() efficiency = q.list_videos(sort="efficiency") - assert [item["aid"] for item in efficiency["items"]] == [102, 101] + assert [item["aid"] for item in efficiency["items"]] == [102] first = efficiency["items"][0] assert first["duration"] == 7200 assert first["content_hours"] == 2.0 - assert first["efficiency"] == 2600.0 + assert first["efficiency"] == 2600.0 # 原始播放/小时，详情页口径 + assert abs(first["efficiency_score"] - 2207.5) < 0.1 assert efficiency["duration_filter"] == { - "min_duration_seconds": 600, "maturity_days": 2, "min_view": 0, + "min_duration_seconds": 600, "maturity_days": 2, + "min_view": 2000, "shrink_minutes": 60, } + # 播放不足 2000 的视频不进入效率榜（101 只有 1200） + assert all(item["aid"] != 101 for item in efficiency["items"]) + interaction = q.list_videos(sort="interaction_efficiency") - assert [item["aid"] for item in interaction["items"]] == [102, 101] - assert interaction["items"][0]["interaction_efficiency"] == 181.0 + assert [item["aid"] for item in interaction["items"]] == [102] + assert abs(interaction["items"][0]["interaction_efficiency_score"] - 159.36) < 0.1 + + # 综合热度榜也应用最小播放门槛 + heat = q.list_videos(sort="heat") + assert [item["aid"] for item in heat["items"]] == [102] # 时长缺失不计入时长效率榜 ups = q.list_ups(sort="efficiency") @@ -405,6 +414,7 @@ def test_duration_efficiency_queries(): assert fresh["items"] == [] assert fresh["available"] is False assert "发布满 2 天" in fresh["reason"] + assert "播放 ≥ 2000" in fresh["reason"] finally: db.close() @@ -422,7 +432,8 @@ def test_about_payload(): assert about["analytics"] == { "min_duration_seconds": 600, "maturity_days": 2, - "min_view_for_efficiency": 0, + "min_view_for_ranking": 2000, + "video_shrink_minutes": 60, "shrink_minutes": None, } assert about["heat_weights"]["view"] == 0.30