fix(analytics): shrink video efficiency and gate noisy boards by views

This commit is contained in:
kairee committed 2026-11-01 12:25:27 +08:00
1 parent e7034cfb84
commit bc3bef0743
12 files changed
+125 -40

No files matched your search

+5 -3
View File
@@ -602,12 +602,14 @@ interaction_rate = (like + coin + favorite) / max(view, 1)
```text
内容小时 = duration / 3600
内容效率 = view / 内容小时
互动效率 = (like + coin + favorite) / 内容小时
内容效率 = (view + κ_v·μ₀) / (内容小时 + κ_v)
互动效率 = (like + coin + favorite + κ_v·μ₀ᵢ) / (内容小时 + κ_v)
UP 内容效率 = (Σview + κ·μ₀) / (Σ内容分钟 + κ) * 60
```
- 分母只用总时长,不用分 P 数(B 站播放记在 bvid 上,`view/pages` 无统计含义);
- 视频级 EB 收缩 `κ_v = analytics.video_shrink_minutes / 60`(默认 60 分钟),`μ₀` 为全库播放/小时;详情页展示原始播放/小时;
- 效率榜与综合热度榜要求播放 ≥ `analytics.min_view_for_ranking`(默认 2000);
- 效率榜要求 `duration >= analytics.min_duration_seconds`(默认 600 秒)且发布满 `analytics.maturity_days`(默认 2 天);
- 不引入 `log(age)` 等无界年龄项,避免系统性惩罚老视频;
- `duration` 缺失或非正数不参与效率排序;
@@ -735,7 +737,7 @@ uv run python tests/test_moves.py
- 收藏夹同步结束时采集缺失 UP 的粉丝数(`relation/stat`,匿名可用);`first_seen >= 运行开始 - 1h` 记为实时,更早的记为回填;
- `app.py backfill-followers`:一次性回填历史视频粉丝数,支持 `--limit` / `--dry-run`;
- 新增内容效率/互动效率榜与 UP 内容效率(Gamma–Poisson EB 收缩),过滤 < 600 秒与未满 2 天的视频;
- `analytics` 配置段:`min_duration_seconds`、`maturity_days`、`min_view_for_efficiency`、`shrink_minutes`;
- `analytics` 配置段:`min_duration_seconds`、`maturity_days`、`min_view_for_ranking`、`video_shrink_minutes`、`shrink_minutes`;
- 主导航拆分:`/videos`(视频)与 `/ups`(UP主)独立;旧 `/rank` 保留兼容;
- 新增 `/about` 数据说明页与 `/api/about`;
- 卡片精简:列表只保留主指标与互动率,明细移到详情页;
+1 -1
View File
@@ -172,7 +172,7 @@ uv run python app.py serve --preview --config data/preview/config.json
- 首页默认“最新动态”:今日增量、最新入库、最近发布、增长动态、热度速览、收藏夹容量与采集状态;
- 「视频」与「UP主」是两个独立入口:视频榜支持内容效率(默认)、综合热度、播放/投币/收藏/评论、互动率、播放增量/播放增速;UP 榜展示视频数、总时长、总播放、播放·小时、互动·小时等,可分别排序;
- 内容效率 = 播放 / 内容小时,只统计时长 ≥ 600 秒且发布满 2 天的视频;UP 内容效率用经验贝叶斯收缩,避免极小样本冲榜;
- 内容效率 = 播放 / 内容小时;单条视频用 60 分钟先验暴露量向全库均值收缩,只统计时长 ≥ 600 秒、发布满 2 天、播放 ≥ 2000 的视频(综合热度榜同样要求播放 ≥ 2000);UP 内容效率用经验贝叶斯收缩,避免极小样本冲榜;
- 「数据说明」(`/about`)说明数据来源、发布时间/入库时间/快照日期、指标公式、筛选口径与已知局限,数值由 `/api/about` 动态读取;
- 视频详情提供分 P 数、七项指标、内容/互动效率、入库时粉丝数、单指标趋势、快照表和同 UP 对比;UP 页提供聚合趋势、单视频对比与入库时/当前粉丝数;
- 粉丝页只绘制粉丝曲线,不叠加视频数据;支持 7/30/90/全部与自选日期,范围超过一年时按每月峰值日期聚合;最近记录默认 14 天并显示较上一成功日的增量;
+2 -1
View File
@@ -92,7 +92,8 @@
"analytics": {
"min_duration_seconds": 600,
"maturity_days": 2,
"min_view_for_efficiency": 0,
"min_view_for_ranking": 2000,
"video_shrink_minutes": 60,
"shrink_minutes": null
}
}
+2 -1
View File
@@ -102,7 +102,8 @@
"analytics": {
"min_duration_seconds": 600,
"maturity_days": 2,
"min_view_for_efficiency": 0,
"min_view_for_ranking": 2000,
"video_shrink_minutes": 60,
"shrink_minutes": null
}
}
+6 -2
View File
@@ -676,12 +676,15 @@ Reaction 视频时长差异极大(100~336 分钟),而播放是累计计数
```
内容小时 = duration / 3600 # 多 P 的 duration 已是各 P 之和
内容效率 = view / 内容小时
互动效率 = (like + coin + favorite) / 内容小时
内容效率 = (view + κ_v·μ₀) / (内容小时 + κ_v) # 榜单口径(收缩后)
原始效率 = view / 内容小时 # 详情页口径
互动效率 = (互动 + κ_v·μ₀ᵢ) / (内容小时 + κ_v)
UP 内容效率 = EB 收缩后的 Σview / Σ内容小时
```
- 分母只用总时长,**不用分 P 数**:B 站播放记在 bvid 上,`view / pages` 没有统计含义;`pages` 仅用于详情展示与包装诊断;
- **视频级也做 EB 收缩**:`κ_v` 默认 60 分钟(配置 `analytics.video_shrink_minutes`),`μ₀` 为全库播放/小时。短时长视频的暴露量小、速率估计噪声大,收缩后不再虚高;2–3 小时的长视频几乎不受影响;
- 效率榜与综合热度榜只统计**播放 ≥ `analytics.min_view_for_ranking`(默认 2000)** 的视频;播放过低的视频多为小样本噪声,不上榜;
- 效率榜只统计 `duration >= analytics.min_duration_seconds`(默认 600 秒)且发布满 `analytics.maturity_days`(默认 2 天)的视频;不足者标注"样本不足";
- **不用无界年龄项**:reaction 视频通常在发布前两天完成数据积累、两周内接近平台期,`log(age)` 会系统性惩罚老视频,故采用"成熟窗口筛选"而非年龄回归;
- `duration` 为空或 ≤ 0(通常只在失效视频出现)不进入效率榜,不插补;
@@ -839,6 +842,7 @@ storage:SQLite WAL,单写线程
| 39 | **粉丝快照回填(真实性)** | 预览库 488 个 UP、796/796 视频补齐 `up_follower`,0 失败;历史视频标记 `backfilled=1` |
| 40 | **导航拆分与数据说明页** | `/videos`、`/ups`、`/about` 均返回 200;`/api/about` 返回 schema 4、当前阈值、权重与粉丝覆盖度 |
| 41 | **全量离线测试** | 32 个 `tests/test_*.py` 全部通过(含新增 `test_up_profile.py` 与迁移/数据库断言更新) |
| 42 | **内容效率噪声修正** | 预览库实测:视频级 EB 收缩(κ=60 分钟)+ 播放 ≥ 2000 后,效率榜 Top10 全为 ≥110 分钟视频;「什么哦吼吼」4 条不再入榜;综合热度榜同样应用 2000 门槛(637 条) |
**未验证(需授权或外部配合)**:
+12 -5
View File
@@ -41,7 +41,8 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
DEFAULT_ANALYTICS: dict[str, Any] = {
"min_duration_seconds": 600,
"maturity_days": 2,
"min_view_for_efficiency": 0,
"min_view_for_ranking": 2000,
"video_shrink_minutes": 60,
"shrink_minutes": None,
}
@@ -173,7 +174,8 @@ class DiscoverConfig:
class AnalyticsConfig:
min_duration_seconds: int
maturity_days: int
min_view_for_efficiency: int
min_view_for_ranking: int
video_shrink_minutes: int
shrink_minutes: int | None
@@ -463,9 +465,14 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
"analytics.maturity_days",
minimum=0,
),
min_view_for_efficiency=_as_int(
analytics_raw.get("min_view_for_efficiency", 0),
"analytics.min_view_for_efficiency",
min_view_for_ranking=_as_int(
analytics_raw.get("min_view_for_ranking", 2000),
"analytics.min_view_for_ranking",
minimum=0,
),
video_shrink_minutes=_as_int(
analytics_raw.get("video_shrink_minutes", 60),
"analytics.video_shrink_minutes",
minimum=0,
),
shrink_minutes=(
+59 -7
View File
@@ -699,6 +699,7 @@ class DashboardQueries:
latest = self._latest_date(conn) or self._today()
analytics = self.config_store.get().analytics
duration_sorts = {"efficiency", "interaction_efficiency"}
min_view_for_ranking = max(0, int(analytics.min_view_for_ranking))
within_field = None
if within_days is not None:
if within_days < 1:
@@ -716,7 +717,7 @@ class DashboardQueries:
)
aids = [int(video["aid"]) for video in videos]
snapshots = self._snapshot_map(conn, aids)
_, _, heat, percentiles = self._global_heat(conn)
videos_all, snapshots_all, heat, percentiles = self._global_heat(conn)
previous: dict[int, dict[str, Any]] = {}
if sort in {"growth", "growth_rate", "interaction"}:
@@ -740,16 +741,60 @@ class DashboardQueries:
if sort in duration_sorts:
latest_start, _ = day_bounds(latest)
minimum_seconds = max(0, int(analytics.min_duration_seconds))
min_view = max(0, int(analytics.min_view_for_efficiency))
items = [
item for item in items
if item["metrics"] is not None
and int(item["duration"] or 0) >= minimum_seconds
and int((item["metrics"] or {}).get("view", 0)) >= min_view
and int((item["metrics"] or {}).get("view", 0)) >= min_view_for_ranking
and is_mature(
item["pubdate"], latest_start, analytics.maturity_days
)
]
kappa_hours = max(0, int(analytics.video_shrink_minutes)) / 60.0
total_hours = 0.0
total_views = 0.0
total_interactions = 0.0
for video in videos_all:
hours = content_hours(video.get("duration"))
if hours is None:
continue
metrics = _metric_map(snapshots_all.get(int(video["aid"])))
if metrics is None:
continue
total_hours += hours
total_views += metrics["view"]
total_interactions += (
metrics["like"] + metrics["coin"] + metrics["favorite"]
)
prior_views = total_views / total_hours if total_hours > 0 else 0.0
prior_interactions = (
total_interactions / total_hours if total_hours > 0 else 0.0
)
for item in items:
hours = content_hours(item.get("duration")) or 0.0
metrics = item["metrics"] or {}
denominator = hours + kappa_hours
if hours <= 0 or denominator <= 0:
item["efficiency_score"] = None
item["interaction_efficiency_score"] = None
continue
view = int(metrics.get("view", 0))
interactions = (
int(metrics.get("like", 0))
+ int(metrics.get("coin", 0))
+ int(metrics.get("favorite", 0))
)
item["efficiency_score"] = (
view + kappa_hours * prior_views
) / denominator
item["interaction_efficiency_score"] = (
interactions + kappa_hours * prior_interactions
) / denominator
if sort == "heat":
items = [
item for item in items
if int((item["metrics"] or {}).get("view", 0)) >= min_view_for_ranking
]
if sort == "interaction":
items = [item for item in items if item["interaction_rate"] is not None]
if sort in {"growth", "growth_rate"}:
@@ -773,7 +818,11 @@ class DashboardQueries:
if sort == "interaction":
return item["interaction_rate"] if item["interaction_rate"] is not None else float("-inf")
if sort in {"efficiency", "interaction_efficiency"}:
value = item.get(sort)
field = (
"efficiency_score" if sort == "efficiency"
else "interaction_efficiency_score"
)
value = item.get(field)
return value if value is not None else float("-inf")
if sort in {"growth", "growth_rate"}:
field = "delta" if sort == "growth" else "rate"
@@ -800,13 +849,15 @@ class DashboardQueries:
duration_filter = {
"min_duration_seconds": max(0, int(analytics.min_duration_seconds)),
"maturity_days": max(0, int(analytics.maturity_days)),
"min_view": max(0, int(analytics.min_view_for_efficiency)),
"min_view": min_view_for_ranking,
"shrink_minutes": max(0, int(analytics.video_shrink_minutes)),
}
if not items:
available = False
reason = (
f"仅统计时长 ≥ {duration_filter['min_duration_seconds']} 秒、"
f"发布满 {duration_filter['maturity_days']} 天的视频"
f"发布满 {duration_filter['maturity_days']} 天、"
f"播放 ≥ {duration_filter['min_view']} 的视频"
)
return {
"sort": sort,
@@ -1425,7 +1476,8 @@ class DashboardQueries:
"analytics": {
"min_duration_seconds": cfg.analytics.min_duration_seconds,
"maturity_days": cfg.analytics.maturity_days,
"min_view_for_efficiency": cfg.analytics.min_view_for_efficiency,
"min_view_for_ranking": cfg.analytics.min_view_for_ranking,
"video_shrink_minutes": cfg.analytics.video_shrink_minutes,
"shrink_minutes": cfg.analytics.shrink_minutes,
},
"heat_weights": {
+3 -3
View File
@@ -51,8 +51,8 @@ export async function renderAbout(root) {
<li><strong>互动</strong>:点赞 + 投币 + 收藏。综合热度也使用全部七项指标,按配置权重加权。</li>
<li><strong>互动率</strong>:(点赞 + 投币 + 收藏) / 播放;播放不足 100 不计算。</li>
<li><strong>综合热度</strong>:每项先 <code>log1p</code>,再按中位数与 IQR 做稳健标准化,最后加权求和。分数只用于同一批数据内排序,不能跨日期比较。</li>
<li><strong>内容效率</strong>:播放 / 内容小时。内容小时 = 视频总时长(多 P 为各 P 之和)÷ 3600。</li>
<li><strong>互动效率</strong>:互动 / 内容小时,互动口径同上。</li>
<li><strong>内容效率</strong>:播放 / 内容小时。内容小时 = 视频总时长(多 P 为各 P 之和)÷ 3600。榜单里的单条视频用收缩后的估计:<code>(播放 + κ·μ₀) / (小时 + κ)</code>,κ 默认 ${analytics.video_shrink_minutes} 分钟、μ₀ 为全库播放/小时;详情页保留原始播放/小时。</li>
<li><strong>互动效率</strong>:互动 / 内容小时,互动口径与收缩方式同上。</li>
<li><strong>UP 内容效率</strong>:把该 UP 的全部视频合并后计算,并用 Gamma–Poisson 经验贝叶斯向全库均值收缩,避免单条视频或极小样本冲榜。收缩强度 κ ${analytics.shrink_minutes ? `固定为 ${analytics.shrink_minutes} 分钟` : "取全库 UP 总内容分钟的中位数"}。</li>
<li><strong>累计触达</strong>:总播放。它是规模指标,不代表吸引力;高产 UP 排在前列是正常结果。</li>
<li><strong>分 P</strong>:<code>pages</code> 是分 P 数,<code>duration</code> 是该 bvid 各 P 时长之和。B 站不公开单 P 播放,因此没有单 P 榜;分 P 数只用于详情展示,不参与排序。</li>
@@ -66,7 +66,7 @@ export async function renderAbout(root) {
<li>只统计发布满 <strong>${analytics.maturity_days}</strong> 天的视频;reaction 视频通常在发布前两天完成数据积累,两周内接近平台期,过早比较会系统性低估新视频。</li>
<li>不引入无界的“年龄惩罚”:超过成熟窗口后不再按天加分或扣分。</li>
<li><code>duration</code> 为空或 ≤ 0 的视频(通常只在失效视频上出现)不进入任何时长相关榜,也不做插补。</li>
<li>当前配置的最小播放门槛为 <strong>${analytics.min_view_for_efficiency}</strong>。</li>
<li>播放低于 <strong>${analytics.min_view_for_ranking}</strong> 的视频不进入内容效率、互动效率与综合热度榜;这是为了避免小样本噪声冲榜。</li>
</ul>
</section>
+6 -3
View File
@@ -83,10 +83,10 @@ function controls({ scope, sort, within, growthDays, q, folder, minVideos, folde
function videoPrimary(item, sort) {
if (sort === "efficiency") {
return { primary: "efficiency", label: "播放/小时", text: formatCount(item.efficiency) };
return { primary: "efficiency_score", label: "播放/小时·收缩", text: formatCount(item.efficiency_score) };
}
if (sort === "interaction_efficiency") {
return { primary: "interaction_efficiency", label: "互动/小时", text: formatCount(item.interaction_efficiency) };
return { primary: "interaction_efficiency_score", label: "互动/小时·收缩", text: formatCount(item.interaction_efficiency_score) };
}
if (sort === "interaction") {
return { primary: "view", label: "互动率", text: formatPercent(item.interaction_rate) };
@@ -143,8 +143,11 @@ function listHtml(data, scope, sort, startRank) {
function durationNote(data) {
const filter = data.duration_filter;
if (!filter) return "";
const shrink = filter.shrink_minutes
? `并按 ${filter.shrink_minutes} 分钟的先验暴露量向全库均值收缩`
: "";
return `<div class="section-note">内容效率与互动效率仅统计时长 ≥ ${filter.min_duration_seconds} 秒、
发布满 ${filter.maturity_days} 天的视频;时长缺失的视频不参与排序。</div>`;
发布满 ${filter.maturity_days} 天、播放 ≥ ${filter.min_view} 的视频${shrink};时长缺失的视频不参与排序。</div>`;
}
function bindControls(root, scope) {
+2 -2
View File
@@ -77,8 +77,8 @@ export async function renderVideo(root, { params, query }) {
<div><span>综合热度</span><strong>${formatScore(detail.heat)}</strong></div>
<div><span>全库百分位</span><strong>${detail.heat_percentile === null || detail.heat_percentile === undefined ? "—" : `前 ${Math.max(0, 100 - detail.heat_percentile).toFixed(0)}%`}</strong></div>
<div><span>播放日增</span><strong>${detail.growth?.available ? formatDelta(detail.growth.delta) : "—"}</strong></div>
<div><span>内容效率</span><strong>${detail.efficiency == null ? "—" : `${formatCount(detail.efficiency)}/小时`}</strong></div>
<div><span>互动效率</span><strong>${detail.interaction_efficiency == null ? "—" : `${formatCount(detail.interaction_efficiency)}/小时`}</strong></div>
<div><span>内容效率(原始)</span><strong>${detail.efficiency == null ? "—" : `${formatCount(detail.efficiency)}/小时`}</strong></div>
<div><span>互动效率(原始)</span><strong>${detail.interaction_efficiency == null ? "—" : `${formatCount(detail.interaction_efficiency)}/小时`}</strong></div>
<div><span>UP 粉丝(入库时)</span><strong>${formatCount(detail.up_follower)}</strong></div>
<div><span>快照数</span><strong>${samples}</strong></div>
<div><span>最近快照</span><strong>${escapeHtml(detail.snapshot_date || "—")}</strong></div>
+8 -4
View File
@@ -133,27 +133,31 @@ def test_analytics_defaults_and_validation():
cfg = load_config(path, env={})
assert cfg.analytics.min_duration_seconds == 600
assert cfg.analytics.maturity_days == 2
assert cfg.analytics.min_view_for_efficiency == 0
assert cfg.analytics.min_view_for_ranking == 2000
assert cfg.analytics.video_shrink_minutes == 60
assert cfg.analytics.shrink_minutes is None
data = base_config()
data["analytics"] = {
"min_duration_seconds": 300,
"maturity_days": 7,
"min_view_for_efficiency": 100,
"min_view_for_ranking": 100,
"video_shrink_minutes": 30,
"shrink_minutes": 90,
}
write(path, data)
cfg = load_config(path, env={})
assert cfg.analytics.min_duration_seconds == 300
assert cfg.analytics.maturity_days == 7
assert cfg.analytics.min_view_for_efficiency == 100
assert cfg.analytics.min_view_for_ranking == 100
assert cfg.analytics.video_shrink_minutes == 30
assert cfg.analytics.shrink_minutes == 90
for bad, hint in (
({"min_duration_seconds": -1}, "min_duration_seconds"),
({"maturity_days": -1}, "maturity_days"),
({"min_view_for_efficiency": -1}, "min_view_for_efficiency"),
({"min_view_for_ranking": -1}, "min_view_for_ranking"),
({"video_shrink_minutes": -1}, "video_shrink_minutes"),
({"shrink_minutes": 0}, "shrink_minutes"),
({"shrink_minutes": "auto"}, "shrink_minutes"),
):
+19 -8
View File
@@ -65,8 +65,8 @@ def test_video_listing_filters_and_sorting():
heat = q.list_videos(sort="heat")
aids = [item["aid"] for item in heat["items"]]
assert 104 not in aids, "removed 视频不应进入榜单"
assert aids[0] == 102, aids
assert heat["items"][0]["heat"] > heat["items"][-1]["heat"]
# 综合热度榜应用最小播放门槛 2000:101(1200)与 103(120)被排除
assert aids == [102], aids
assert heat["items"][0]["heat_percentile"] == 100.0
views = q.list_videos(sort="view")
@@ -362,18 +362,27 @@ def test_duration_efficiency_queries():
DASHBOARD_CACHE.clear()
efficiency = q.list_videos(sort="efficiency")
assert [item["aid"] for item in efficiency["items"]] == [102, 101]
assert [item["aid"] for item in efficiency["items"]] == [102]
first = efficiency["items"][0]
assert first["duration"] == 7200
assert first["content_hours"] == 2.0
assert first["efficiency"] == 2600.0
assert first["efficiency"] == 2600.0 # 原始播放/小时,详情页口径
assert abs(first["efficiency_score"] - 2207.5) < 0.1
assert efficiency["duration_filter"] == {
"min_duration_seconds": 600, "maturity_days": 2, "min_view": 0,
"min_duration_seconds": 600, "maturity_days": 2,
"min_view": 2000, "shrink_minutes": 60,
}
# 播放不足 2000 的视频不进入效率榜(101 只有 1200)
assert all(item["aid"] != 101 for item in efficiency["items"])
interaction = q.list_videos(sort="interaction_efficiency")
assert [item["aid"] for item in interaction["items"]] == [102, 101]
assert interaction["items"][0]["interaction_efficiency"] == 181.0
assert [item["aid"] for item in interaction["items"]] == [102]
assert abs(interaction["items"][0]["interaction_efficiency_score"] - 159.36) < 0.1
# 综合热度榜也应用最小播放门槛
heat = q.list_videos(sort="heat")
assert [item["aid"] for item in heat["items"]] == [102]
# 时长缺失不计入时长效率榜
ups = q.list_ups(sort="efficiency")
@@ -405,6 +414,7 @@ def test_duration_efficiency_queries():
assert fresh["items"] == []
assert fresh["available"] is False
assert "发布满 2 天" in fresh["reason"]
assert "播放 ≥ 2000" in fresh["reason"]
finally:
db.close()
@@ -422,7 +432,8 @@ def test_about_payload():
assert about["analytics"] == {
"min_duration_seconds": 600,
"maturity_days": 2,
"min_view_for_efficiency": 0,
"min_view_for_ranking": 2000,
"video_shrink_minutes": 60,
"shrink_minutes": None,
}
assert about["heat_weights"]["view"] == 0.30