feat(analytics): lower maturity window to 2 days and reword about page
This commit is contained in:
1 parent
b405fc5fd7
commit
e7034cfb84
11 files changed
+66
-28
No files matched your search
+2
-2
@@ -608,7 +608,7 @@ UP 内容效率 = (Σview + κ·μ₀) / (Σ内容分钟 + κ) * 60
|
||||
```
|
||||
|
||||
- 分母只用总时长,不用分 P 数(B 站播放记在 bvid 上,`view/pages` 无统计含义);
|
||||
- 效率榜要求 `duration >= analytics.min_duration_seconds`(默认 600 秒)且发布满 `analytics.maturity_days`(默认 14 天);
|
||||
- 效率榜要求 `duration >= analytics.min_duration_seconds`(默认 600 秒)且发布满 `analytics.maturity_days`(默认 2 天);
|
||||
- 不引入 `log(age)` 等无界年龄项,避免系统性惩罚老视频;
|
||||
- `duration` 缺失或非正数不参与效率排序;
|
||||
- UP 收缩强度 `κ` 取全库 UP 总内容分钟中位数,`analytics.shrink_minutes` 可覆盖;
|
||||
@@ -734,7 +734,7 @@ uv run python tests/test_moves.py
|
||||
- 迁移 v4:`video.pages`、`video.up_follower` / `up_follower_at` / `up_follower_backfilled`、`up_profile`;
|
||||
- 收藏夹同步结束时采集缺失 UP 的粉丝数(`relation/stat`,匿名可用);`first_seen >= 运行开始 - 1h` 记为实时,更早的记为回填;
|
||||
- `app.py backfill-followers`:一次性回填历史视频粉丝数,支持 `--limit` / `--dry-run`;
|
||||
- 新增内容效率/互动效率榜与 UP 内容效率(Gamma–Poisson EB 收缩),过滤 < 600 秒与未满 14 天的视频;
|
||||
- 新增内容效率/互动效率榜与 UP 内容效率(Gamma–Poisson EB 收缩),过滤 < 600 秒与未满 2 天的视频;
|
||||
- `analytics` 配置段:`min_duration_seconds`、`maturity_days`、`min_view_for_efficiency`、`shrink_minutes`;
|
||||
- 主导航拆分:`/videos`(视频)与 `/ups`(UP主)独立;旧 `/rank` 保留兼容;
|
||||
- 新增 `/about` 数据说明页与 `/api/about`;
|
||||
|
||||
@@ -172,7 +172,7 @@ uv run python app.py serve --preview --config data/preview/config.json
|
||||
|
||||
- 首页默认“最新动态”:今日增量、最新入库、最近发布、增长动态、热度速览、收藏夹容量与采集状态;
|
||||
- 「视频」与「UP主」是两个独立入口:视频榜支持内容效率(默认)、综合热度、播放/投币/收藏/评论、互动率、播放增量/播放增速;UP 榜展示视频数、总时长、总播放、播放·小时、互动·小时等,可分别排序;
|
||||
- 内容效率 = 播放 / 内容小时,只统计时长 ≥ 600 秒且发布满 14 天的视频;UP 内容效率用经验贝叶斯收缩,避免极小样本冲榜;
|
||||
- 内容效率 = 播放 / 内容小时,只统计时长 ≥ 600 秒且发布满 2 天的视频;UP 内容效率用经验贝叶斯收缩,避免极小样本冲榜;
|
||||
- 「数据说明」(`/about`)说明数据来源、发布时间/入库时间/快照日期、指标公式、筛选口径与已知局限,数值由 `/api/about` 动态读取;
|
||||
- 视频详情提供分 P 数、七项指标、内容/互动效率、入库时粉丝数、单指标趋势、快照表和同 UP 对比;UP 页提供聚合趋势、单视频对比与入库时/当前粉丝数;
|
||||
- 粉丝页只绘制粉丝曲线,不叠加视频数据;支持 7/30/90/全部与自选日期,范围超过一年时按每月峰值日期聚合;最近记录默认 14 天并显示较上一成功日的增量;
|
||||
|
||||
@@ -91,7 +91,7 @@
|
||||
},
|
||||
"analytics": {
|
||||
"min_duration_seconds": 600,
|
||||
"maturity_days": 14,
|
||||
"maturity_days": 2,
|
||||
"min_view_for_efficiency": 0,
|
||||
"shrink_minutes": null
|
||||
}
|
||||
|
||||
+1
-1
@@ -101,7 +101,7 @@
|
||||
},
|
||||
"analytics": {
|
||||
"min_duration_seconds": 600,
|
||||
"maturity_days": 14,
|
||||
"maturity_days": 2,
|
||||
"min_view_for_efficiency": 0,
|
||||
"shrink_minutes": null
|
||||
}
|
||||
|
||||
+4
-4
@@ -682,8 +682,8 @@ UP 内容效率 = EB 收缩后的 Σview / Σ内容小时
|
||||
```
|
||||
|
||||
- 分母只用总时长,**不用分 P 数**:B 站播放记在 bvid 上,`view / pages` 没有统计含义;`pages` 仅用于详情展示与包装诊断;
|
||||
- 效率榜只统计 `duration >= analytics.min_duration_seconds`(默认 600 秒)且发布满 `analytics.maturity_days`(默认 14 天)的视频;不足者标注"样本不足";
|
||||
- **不用无界年龄项**:reaction 视频约两周进入平台期,`log(age)` 会系统性惩罚老视频,故采用"成熟窗口筛选"而非年龄回归;
|
||||
- 效率榜只统计 `duration >= analytics.min_duration_seconds`(默认 600 秒)且发布满 `analytics.maturity_days`(默认 2 天)的视频;不足者标注"样本不足";
|
||||
- **不用无界年龄项**:reaction 视频通常在发布前两天完成数据积累、两周内接近平台期,`log(age)` 会系统性惩罚老视频,故采用"成熟窗口筛选"而非年龄回归;
|
||||
- `duration` 为空或 ≤ 0(通常只在失效视频出现)不进入效率榜,不插补;
|
||||
- UP 效率用 Gamma–Poisson 经验贝叶斯:`rate = (Σview + κ·μ₀) / (Σ内容分钟 + κ)`,`μ₀` 为全库播放/分钟,`κ` 自适应取全库 UP 总内容分钟中位数(配置 `analytics.shrink_minutes` 可覆盖,缺省回退 120 分钟);
|
||||
- 视频榜默认 Tab 为内容效率,原始播放榜保留;互动单项只保留播放/投币/收藏/评论,点赞、弹幕、转发并入"互动";
|
||||
@@ -835,7 +835,7 @@ storage:SQLite WAL,单写线程
|
||||
| 35 | **只读页面** | 首页/视频/UP主/视频详情/UP 详情/粉丝/数据说明/候选/系统页面可访问;390px 与 1280px 下无脚本错误 |
|
||||
| 36 | **P2 只读边界** | 业务 POST/PUT/PATCH/DELETE 均返回 405,请求前后数据库行数不变 |
|
||||
| 37 | **粉丝历史回填(CSV)** | 从 B 站粉丝历史 CSV 导入 2020-06-03~2026-10-31 共 1788 条,替换旧 91 条;155 个缺失日保持断开 |
|
||||
| 38 | **迁移 v4 与时长效率榜** | 预览库升级到 schema 4;`/api/videos?sort=efficiency`、`/api/ups?sort=efficiency` 正确排序,`min_duration_seconds=600` 与 `maturity_days=14` 过滤生效,`duration` 缺失不参与 |
|
||||
| 38 | **迁移 v4 与时长效率榜** | 预览库升级到 schema 4;`/api/videos?sort=efficiency`、`/api/ups?sort=efficiency` 正确排序,`min_duration_seconds=600` 与 `maturity_days=2` 过滤生效,`duration` 缺失不参与 |
|
||||
| 39 | **粉丝快照回填(真实性)** | 预览库 488 个 UP、796/796 视频补齐 `up_follower`,0 失败;历史视频标记 `backfilled=1` |
|
||||
| 40 | **导航拆分与数据说明页** | `/videos`、`/ups`、`/about` 均返回 200;`/api/about` 返回 schema 4、当前阈值、权重与粉丝覆盖度 |
|
||||
| 41 | **全量离线测试** | 32 个 `tests/test_*.py` 全部通过(含新增 `test_up_profile.py` 与迁移/数据库断言更新) |
|
||||
@@ -862,7 +862,7 @@ P0、P1、P2、P3 与第 9 轮已实施;P2 交付了只读本地看板与可
|
||||
|
||||
1. schema v4:`video.pages`、`video.up_follower*`、`up_profile`;
|
||||
2. 入库时采集 UP 粉丝快照 + `python app.py backfill-followers` 历史回填(基于粉丝数的榜单暂不做);
|
||||
3. 时长效率榜(内容效率/互动效率)与 UP 内容效率(EB 收缩),过滤 < 600 秒与未满 14 天的视频;
|
||||
3. 时长效率榜(内容效率/互动效率)与 UP 内容效率(EB 收缩),过滤 < 600 秒与未满 2 天的视频;
|
||||
4. 主导航拆分:「榜单」→「视频」+「UP主」,两者独立页面;新增「数据说明」页(`/about` + `/api/about`);
|
||||
5. 列表精简:卡片只保留主指标 + 互动率,七项明细与分 P、粉丝快照移到详情页。
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
|
||||
MIN_VIEW_FOR_INTERACTION = 100
|
||||
MIN_VIEW_FOR_GROWTH_RATE = 200
|
||||
MIN_EFFICIENCY_DURATION = 600
|
||||
DEFAULT_MATURITY_DAYS = 14
|
||||
DEFAULT_MATURITY_DAYS = 2
|
||||
DEFAULT_SHRINK_MINUTES = 120.0
|
||||
_IQR_TO_SIGMA = 1.349
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
|
||||
|
||||
DEFAULT_ANALYTICS: dict[str, Any] = {
|
||||
"min_duration_seconds": 600,
|
||||
"maturity_days": 14,
|
||||
"maturity_days": 2,
|
||||
"min_view_for_efficiency": 0,
|
||||
"shrink_minutes": None,
|
||||
}
|
||||
@@ -459,7 +459,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
||||
minimum=0,
|
||||
),
|
||||
maturity_days=_as_int(
|
||||
analytics_raw.get("maturity_days", 14),
|
||||
analytics_raw.get("maturity_days", 2),
|
||||
"analytics.maturity_days",
|
||||
minimum=0,
|
||||
),
|
||||
|
||||
@@ -738,7 +738,7 @@ class DashboardQueries:
|
||||
]
|
||||
|
||||
if sort in duration_sorts:
|
||||
_, latest_end = day_bounds(latest)
|
||||
latest_start, _ = day_bounds(latest)
|
||||
minimum_seconds = max(0, int(analytics.min_duration_seconds))
|
||||
min_view = max(0, int(analytics.min_view_for_efficiency))
|
||||
items = [
|
||||
@@ -747,7 +747,7 @@ class DashboardQueries:
|
||||
and int(item["duration"] or 0) >= minimum_seconds
|
||||
and int((item["metrics"] or {}).get("view", 0)) >= min_view
|
||||
and is_mature(
|
||||
item["pubdate"], latest_end, analytics.maturity_days
|
||||
item["pubdate"], latest_start, analytics.maturity_days
|
||||
)
|
||||
]
|
||||
if sort == "interaction":
|
||||
@@ -1396,10 +1396,24 @@ class DashboardQueries:
|
||||
preview_row = conn.execute(
|
||||
"SELECT v FROM kv WHERE k = 'preview:seeded'"
|
||||
).fetchone()
|
||||
fans_start_date = None
|
||||
track_fans = [
|
||||
int(mid) for mid in cfg.track_fans if not isinstance(mid, bool)
|
||||
]
|
||||
if track_fans:
|
||||
placeholders = ",".join("?" for _ in track_fans)
|
||||
row = conn.execute(
|
||||
f"SELECT MIN(stat_date) AS d FROM up_stat "
|
||||
f"WHERE ok = 1 AND mid IN ({placeholders})",
|
||||
track_fans,
|
||||
).fetchone()
|
||||
fans_start_date = None if row is None else row["d"]
|
||||
return {
|
||||
"data_date": latest,
|
||||
"snapshot_days": self._snapshot_days(conn),
|
||||
"snapshot_first_date": first_date,
|
||||
"fans_start_date": fans_start_date,
|
||||
"track_fans": track_fans,
|
||||
"schema_version": int(self.db.schema_version or 0),
|
||||
"preview": bool(preview_row and str(preview_row["v"]) == "1"),
|
||||
"counts": counts,
|
||||
|
||||
@@ -14,6 +14,9 @@ export async function renderAbout(root) {
|
||||
const coverage = followers.videos_total
|
||||
? Math.round((followers.videos_covered / followers.videos_total) * 100)
|
||||
: 0;
|
||||
const fansStart = data.fans_start_date
|
||||
? escapeHtml(data.fans_start_date)
|
||||
: "暂无";
|
||||
|
||||
root.innerHTML = `<section class="page about-page">
|
||||
<header class="page-head">
|
||||
@@ -23,19 +26,20 @@ export async function renderAbout(root) {
|
||||
<section class="panel">
|
||||
<h2>数据来源与更新</h2>
|
||||
<ul class="about-list">
|
||||
<li>视频元数据、播放与互动来自 B 站公开接口;粉丝数来自 <code>relation/stat</code>,匿名可用。</li>
|
||||
<li>发现与收藏夹同步每小时运行一次,每日快照在 00:01 采集;最新数据日:<strong>${escapeHtml(data.data_date || "暂无")}</strong>。</li>
|
||||
<li>当前快照覆盖 <strong>${data.snapshot_days}</strong> 天${data.snapshot_first_date ? `(起始 ${escapeHtml(data.snapshot_first_date)})` : ""};历史快照无法从 B 站回填,只能从部署当天开始积累。</li>
|
||||
<li>数据库 schema 版本 <code>${data.schema_version}</code>;${data.preview ? "当前为预览数据,不影响真实数据库。" : "当前为真实数据库。"}</li>
|
||||
<li>所有数据来自 B 站以及部分 MCN 统计网站。</li>
|
||||
<li>新视频人工入库;库内视频数据每天更新一次;最新数据日:<strong>${escapeHtml(data.data_date || "暂无")}</strong>。</li>
|
||||
<li>当前视频数据覆盖 <strong>${data.snapshot_days}</strong> 天${data.snapshot_first_date ? `(起始 ${escapeHtml(data.snapshot_first_date)})` : ""};历史快照无法从 B 站回填,只能从部署当天开始积累。</li>
|
||||
<li>咻咻满粉丝数据的起点日期为 <strong>${fansStart}</strong>。</li>
|
||||
</ul>
|
||||
${data.preview ? `<p class="section-note">当前为预览数据,不影响真实数据库。</p>` : ""}
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>三个时间概念</h2>
|
||||
<ul class="about-list">
|
||||
<li><strong>发布时间</strong>(<code>pubdate</code>):视频在 B 站上线的时刻。</li>
|
||||
<li><strong>入库时间</strong>(<code>first_seen</code>):本站第一次看到并写入该视频的时刻。人工加收藏或搜索确认都可能导致入库晚于发布。</li>
|
||||
<li><strong>快照日期</strong>(<code>stat_date</code>):每天采集一次播放与互动,写入 <code>snapshot</code>。缺失日不插值,曲线直接断开。</li>
|
||||
<li><strong>发布时间</strong>(<code>pubdate</code>):视频在 B 站发布的时刻。</li>
|
||||
<li><strong>入库时间</strong>(<code>first_seen</code>):本站第一次录入该视频的时刻。</li>
|
||||
<li><strong>快照日期</strong>(<code>stat_date</code>):每天更新一次视频的播放与互动数据。缺失日不插值,曲线直接断开。</li>
|
||||
</ul>
|
||||
<p class="section-note">因此,同一个视频的“发布后 7 天数据”和“入库后 7 天数据”可能不是同一段区间;详情页会同时展示发布时间与入库时间。</p>
|
||||
</section>
|
||||
@@ -59,7 +63,7 @@ export async function renderAbout(root) {
|
||||
<h2>榜单筛选</h2>
|
||||
<ul class="about-list">
|
||||
<li>内容效率与互动效率只统计时长 ≥ <strong>${analytics.min_duration_seconds}</strong> 秒的视频,排除结构不同的短视频与切片。</li>
|
||||
<li>只统计发布满 <strong>${analytics.maturity_days}</strong> 天的视频;reaction 视频通常在两周内接近平台期,过早比较会系统性低估新视频。</li>
|
||||
<li>只统计发布满 <strong>${analytics.maturity_days}</strong> 天的视频;reaction 视频通常在发布前两天完成数据积累,两周内接近平台期,过早比较会系统性低估新视频。</li>
|
||||
<li>不引入无界的“年龄惩罚”:超过成熟窗口后不再按天加分或扣分。</li>
|
||||
<li><code>duration</code> 为空或 ≤ 0 的视频(通常只在失效视频上出现)不进入任何时长相关榜,也不做插补。</li>
|
||||
<li>当前配置的最小播放门槛为 <strong>${analytics.min_view_for_efficiency}</strong>。</li>
|
||||
@@ -75,8 +79,7 @@ export async function renderAbout(root) {
|
||||
<section class="panel">
|
||||
<h2>UP 粉丝快照</h2>
|
||||
<ul class="about-list">
|
||||
<li>视频入库时记录一次 UP 当前粉丝数,写入 <code>video.up_follower</code>;同一 UP 的旧视频不会被新快照覆盖。</li>
|
||||
<li><code>up_profile.follower_first</code> 保留首次采集值,<code>follower_latest</code> 随刷新更新。</li>
|
||||
<li>视频入库时记录一次 UP 当前粉丝数,后续更新视频数据不刷新 UP 的入库时粉丝数。</li>
|
||||
<li>历史视频的粉丝数是回填值,会标记 <code>backfilled</code>;它与真实的“入库时”存在时间差,暂不用于榜单排序。</li>
|
||||
<li>当前覆盖:${followers.videos_covered} / ${followers.videos_total} 条视频(${coverage}%),涉及 ${followers.ups_tracked} 个 UP;缺失 ${followers.videos_missing} 条,可运行 <code>app.py backfill-followers</code> 补齐。</li>
|
||||
</ul>
|
||||
@@ -92,5 +95,13 @@ export async function renderAbout(root) {
|
||||
<li>样本不足(少于 2 天快照、单视频 UP)会显示提示,不伪造 0。</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>技术信息</h2>
|
||||
<ul class="about-list">
|
||||
<li>数据库 schema 版本 <code>${data.schema_version}</code>。</li>
|
||||
<li>${data.preview ? "当前为预览数据,不影响真实数据库。" : "当前为真实数据库。"}</li>
|
||||
</ul>
|
||||
</section>
|
||||
</section>`;
|
||||
}
|
||||
}
|
||||
@@ -132,7 +132,7 @@ def test_analytics_defaults_and_validation():
|
||||
write(path, base_config())
|
||||
cfg = load_config(path, env={})
|
||||
assert cfg.analytics.min_duration_seconds == 600
|
||||
assert cfg.analytics.maturity_days == 14
|
||||
assert cfg.analytics.maturity_days == 2
|
||||
assert cfg.analytics.min_view_for_efficiency == 0
|
||||
assert cfg.analytics.shrink_minutes is None
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ sys.stdout.reconfigure(encoding="utf-8")
|
||||
from reaction_board.config import ConfigStore # noqa: E402
|
||||
from reaction_board.database import Database # noqa: E402
|
||||
from reaction_board.repository import Repository # noqa: E402
|
||||
from reaction_board.timeutil import day_bounds # noqa: E402
|
||||
from reaction_board.web.cache import DASHBOARD_CACHE # noqa: E402
|
||||
from reaction_board.web.queries import DashboardQueries # noqa: E402
|
||||
from tests.web_fixtures import DATES, build_db, write_config # noqa: E402
|
||||
@@ -367,7 +368,7 @@ def test_duration_efficiency_queries():
|
||||
assert first["content_hours"] == 2.0
|
||||
assert first["efficiency"] == 2600.0
|
||||
assert efficiency["duration_filter"] == {
|
||||
"min_duration_seconds": 600, "maturity_days": 14, "min_view": 0,
|
||||
"min_duration_seconds": 600, "maturity_days": 2, "min_view": 0,
|
||||
}
|
||||
|
||||
interaction = q.list_videos(sort="interaction_efficiency")
|
||||
@@ -393,6 +394,17 @@ def test_duration_efficiency_queries():
|
||||
DASHBOARD_CACHE.clear()
|
||||
missing = q.list_videos(sort="efficiency")
|
||||
assert [item["aid"] for item in missing["items"]] == [102]
|
||||
|
||||
# 发布未满 2 天的视频不进入效率榜
|
||||
db.write(lambda conn: conn.execute(
|
||||
"UPDATE video SET duration = 7200, pubdate = ? WHERE aid = 102",
|
||||
(int(day_bounds("2026-10-30")[0]),),
|
||||
))
|
||||
DASHBOARD_CACHE.clear()
|
||||
fresh = q.list_videos(sort="efficiency")
|
||||
assert fresh["items"] == []
|
||||
assert fresh["available"] is False
|
||||
assert "发布满 2 天" in fresh["reason"]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -405,10 +417,11 @@ def test_about_payload():
|
||||
assert about["data_date"] == DATES[-1]
|
||||
assert about["snapshot_days"] == 3
|
||||
assert about["snapshot_first_date"] == DATES[0]
|
||||
assert about["fans_start_date"] == DATES[0]
|
||||
assert about["schema_version"] == 4
|
||||
assert about["analytics"] == {
|
||||
"min_duration_seconds": 600,
|
||||
"maturity_days": 14,
|
||||
"maturity_days": 2,
|
||||
"min_view_for_efficiency": 0,
|
||||
"shrink_minutes": None,
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user