From c5b71856750fb4dfbe1da9854c529393494ecb34 Mon Sep 17 00:00:00 2001 From: kaireewu Date: Mon, 2 Nov 2026 20:44:17 +0800 Subject: [PATCH] feat(growth): same-day 1-day baselines for new videos Let 1-day view growth use a zero baseline for videos published today, and the first successful sample for older videos ingested today, without mixing those into 7/30-day rankings. --- project.md | 10 +- reaction_board/analytics.py | 11 +- reaction_board/migrations.py | 12 +++ reaction_board/repository.py | 38 +++++-- reaction_board/services/snapshot.py | 3 +- reaction_board/web/queries.py | 114 +++++++++++++++++--- reaction_board/web/static/components.js | 10 +- reaction_board/web/static/views/about.js | 2 +- reaction_board/web/static/views/home.js | 2 +- reaction_board/web/static/views/rankings.js | 7 ++ reaction_board/web/static/views/video.js | 2 +- tests/test_analytics.py | 12 +++ tests/test_cli.py | 2 +- tests/test_database.py | 8 +- tests/test_migrations.py | 54 ++++++++-- tests/test_repository.py | 54 ++++++++++ tests/test_snapshot.py | 14 ++- tests/test_web_queries.py | 105 ++++++++++++++++-- tests/test_web_server.py | 2 +- tests/web_fixtures.py | 2 +- tools/seed_preview_db.py | 6 +- 21 files changed, 411 insertions(+), 59 deletions(-) diff --git a/project.md b/project.md index 4261107..b4935ce 100644 --- a/project.md +++ b/project.md @@ -380,7 +380,7 @@ for folder in 配置的年度夹（2025、2026、…）: - 两个夹共 831 条，一次性导入 `video` 表； - **历史快照无法回填**：B 站不提供任意日期历史数据，只能从部署当天开始积累； - **开发联调阶段产生的数据一律保留**（不清库、不重建），正式上线直接续用同一个 `app.db`； -- 新发现/新入库的视频从"入库当天"开始积累快照，第一天没有增量是正常的，看板对"仅 1 个数据点"的视频标注"样本不足"。 +- 新发现/新入库的视频从"入库当天"开始积累快照。近 1 日：当日发布按 0 起算（一次成功采样即可）；更早发布的新入库按入库时播放量起算，第二次采样走晚间全量，不进小时快照。近 7/30 日仍要跨日快照。 --- @@ -473,6 +473,8 @@ CREATE TABLE snapshot ( ok INTEGER NOT NULL DEFAULT 1, err TEXT, src TEXT DEFAULT 'view', -- view | infos（降级时标记） + view_open INTEGER, -- 同日第一次成功采样的播放量 + open_captured_at INTEGER, -- 开盘采样时刻 PRIMARY KEY (stat_date, aid) ); CREATE INDEX idx_snap_aid ON snapshot(aid, stat_date); @@ -672,11 +674,13 @@ w = {view:0.30, like:0.20, coin:0.20, favorite:0.15, ### 8.2 增长榜 -> 看板 UI 中“增长”标注为**播放增量**（区间内绝对增量），“增速”标注为**播放增速**（增量 / 基线播放）。两者基线都取不早于 `最新快照 - N 天` 的最近一条成功快照。 +> 看板 UI 中“增长”标注为**播放增量**（区间内绝对增量），“增速”标注为**播放增速**（增量 / 基线播放）。近 7/30 日基线取不早于 `最新快照 - N 天` 的最近一条成功快照。近 1 日：有更早日期快照则用前一日；否则当日发布基准为 0，更早发布的新入库用当天第一次成功采样（`view_open`）。 ``` d7 = (view_today - view_{today-7}) / 实际间隔天数 r7 = (view_today - view_{today-7}) / max(view_{today-7}, 1) +d1_published_today = view_today - 0 +d1_imported_today = view_today - view_open # 需同日第二次采样 ``` | 榜 | 口径 | 基数门槛 | @@ -688,7 +692,7 @@ r7 = (view_today - view_{today-7}) / max(view_{today-7}, 1) | 单项榜 | 7 项直接排序 | — | | UP 榜 | 按 `up_mid` 聚合 Σ/AVG | — | -**冷启动**：不足 2 天数据的视频用"发布后 24h 增量"，并标注"样本不足"；缺失日不插值。 +**冷启动**：近 1 日不要求跨日快照。当日发布相对 0；更早发布且当天入库的片子相对 `view_open`（第二次采样走晚间全量，不进小时快照）。近 7/30 日仍要跨日快照。缺失日不插值。 ### 8.3 趋势 diff --git a/reaction_board/analytics.py b/reaction_board/analytics.py index d7b4cf2..70043ab 100644 --- a/reaction_board/analytics.py +++ b/reaction_board/analytics.py @@ -313,15 +313,20 @@ def compute_growth( current: int | float | None, previous: int | float | None, gap_days: int | float | None, + *, + allow_zero_gap: bool = False, ) -> Growth: """计算累计差值、日均差值和相对增速。 - 缺失基线或间隔不足 1 天时返回不可用，不把 0 当作真实增长。 + 缺失基线或间隔为负时返回不可用。默认间隔不足 1 天也不可用，不把 0 当成真实增长。 + ``allow_zero_gap=True`` 时允许同一天的两次采样（或相对 0）计算增量，``per_day`` 为 None。 """ if current is None or previous is None or gap_days is None: return Growth(available=False) gap = int(gap_days) - if gap <= 0: + if gap < 0: + return Growth(available=False) + if gap == 0 and not allow_zero_gap: return Growth(available=False) current_value = max(0, int(current)) previous_value = max(0, int(previous)) @@ -329,7 +334,7 @@ def compute_growth( return Growth( available=True, delta=delta, - per_day=delta / gap, + per_day=None if gap == 0 else delta / gap, rate=delta / max(previous_value, 1), gap_days=gap, ) diff --git a/reaction_board/migrations.py b/reaction_board/migrations.py index f3b3202..6948d8a 100644 --- a/reaction_board/migrations.py +++ b/reaction_board/migrations.py @@ -244,6 +244,18 @@ MIGRATIONS: tuple[Migration, ...] = ( "CREATE INDEX idx_visit_date ON visit(stat_date)", ), ), + Migration( + version=8, + name="snapshot open view", + statements=( + # 同日第一次成功采样的播放量，供近 1 日「入库后增量」做基准。 + # 已有成功行无法找回被覆盖的入库值，只能把当前 view 当作开盘。 + 'ALTER TABLE snapshot ADD COLUMN view_open INTEGER', + "ALTER TABLE snapshot ADD COLUMN open_captured_at INTEGER", + 'UPDATE snapshot SET view_open = "view", open_captured_at = captured_at ' + "WHERE ok = 1 AND view_open IS NULL", + ), + ), ) diff --git a/reaction_board/repository.py b/reaction_board/repository.py index f0df923..23be4d2 100644 --- a/reaction_board/repository.py +++ b/reaction_board/repository.py @@ -690,12 +690,17 @@ class Repository: src: str = "view", ) -> None: values = metrics or {} + ok_flag = 1 if ok else 0 + view_value = values.get("view") + open_view = view_value if ok else None + open_at = int(captured_at) if ok else None def run(conn: sqlite3.Connection) -> None: conn.execute( """ INSERT INTO snapshot(stat_date, aid, "view", "like", favorite, coin, - share, reply, danmaku, captured_at, ok, err, src) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + share, reply, danmaku, captured_at, ok, err, src, + view_open, open_captured_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(stat_date, aid) DO UPDATE SET "view" = excluded."view", "like" = excluded."like", @@ -707,12 +712,22 @@ class Repository: captured_at = excluded.captured_at, ok = excluded.ok, err = excluded.err, - src = excluded.src + src = excluded.src, + view_open = CASE + WHEN excluded.ok = 1 AND snapshot.view_open IS NULL + THEN excluded."view" + ELSE snapshot.view_open + END, + open_captured_at = CASE + WHEN excluded.ok = 1 AND snapshot.open_captured_at IS NULL + THEN excluded.captured_at + ELSE snapshot.open_captured_at + END """, ( stat_date, int(aid), - values.get("view"), + view_value, values.get("like"), values.get("favorite"), values.get("coin"), @@ -720,9 +735,11 @@ class Repository: values.get("reply"), values.get("danmaku"), int(captured_at), - 1 if ok else 0, + ok_flag, err, src, + open_view, + open_at, ), ) self.db.write(run) @@ -741,18 +758,23 @@ class Repository: """ INSERT INTO snapshot( stat_date, aid, "view", "like", favorite, coin, - share, reply, danmaku, captured_at, ok, err, src + share, reply, danmaku, captured_at, ok, err, src, + view_open, open_captured_at ) SELECT ?, aid, "view", "like", favorite, coin, - share, reply, danmaku, ?, 1, NULL, ? + share, reply, danmaku, ?, 1, NULL, ?, + "view", ? FROM snapshot WHERE stat_date = ? AND ok = 1 AND aid NOT IN ( SELECT aid FROM snapshot WHERE stat_date = ? ) """, - (target_date, int(captured_at), src, source_date, target_date), + ( + target_date, int(captured_at), src, int(captured_at), + source_date, target_date, + ), ) return int(cur.rowcount or 0) return self.db.write(run) diff --git a/reaction_board/services/snapshot.py b/reaction_board/services/snapshot.py index 59e40c9..da8bc8e 100644 --- a/reaction_board/services/snapshot.py +++ b/reaction_board/services/snapshot.py @@ -310,7 +310,8 @@ class SnapshotService: """刷新近 N 天新视频的当日快照；不碰完成日标记与日备份。 「近 N 天」按 pubdate 墙钟窗口：``now - recent_video_days * 86400``， - 不是上海日历日差。允许覆盖同日已有 ``ok=1`` 行以便小时级刷新。 + 不是上海日历日差。当天入库但发布更早的旧片不进小时任务。 + 允许覆盖同日已有 ``ok=1`` 行以便小时级刷新。 """ cfg = self.config_store.get() client = self._client() diff --git a/reaction_board/web/queries.py b/reaction_board/web/queries.py index 5db1331..a74b58c 100644 --- a/reaction_board/web/queries.py +++ b/reaction_board/web/queries.py @@ -385,6 +385,18 @@ class DashboardQueries: percentiles = score_percentiles(scores) return videos, snapshots, scores, percentiles + @staticmethod + def _publish_date(pubdate: object) -> str | None: + if pubdate is None or isinstance(pubdate, bool): + return None + try: + ts = int(pubdate) + except (TypeError, ValueError): + return None + if ts <= 0: + return None + return shanghai_date(ts) + @staticmethod def _growth_for( snapshot: Mapping[str, Any] | None, @@ -406,6 +418,68 @@ class DashboardQueries: gap = None return compute_growth(current, before, gap) + @classmethod + def _same_day_growth( + cls, + snapshot: Mapping[str, Any] | None, + video: Mapping[str, Any], + metric: str = "view", + ) -> dict[str, Any] | None: + """近 1 日、没有更早快照时：当日发布相对 0；否则用 view_open。""" + if snapshot is None: + return None + current = snapshot.get(metric) + if current is None: + return None + stat_date = str(snapshot.get("stat_date") or "") + published_on = cls._publish_date(video.get("pubdate")) + if published_on is not None and published_on == stat_date: + growth = compute_growth(current, 0, 0, allow_zero_gap=True) + payload = growth.as_dict() + payload["baseline"] = "zero" + return payload + try: + open_at = snapshot.get("open_captured_at") + captured_at = snapshot.get("captured_at") + view_open = snapshot.get("view_open") + open_ready = ( + open_at is not None + and captured_at is not None + and int(open_at) < int(captured_at) + and view_open is not None + ) + except (TypeError, ValueError): + open_ready = False + if not open_ready: + return None + growth = compute_growth(current, view_open, 0, allow_zero_gap=True) + payload = growth.as_dict() + payload["baseline"] = "open" + return payload + + @classmethod + def _growth_result( + cls, + video: Mapping[str, Any], + snapshot: Mapping[str, Any] | None, + previous: Mapping[str, Any] | None, + *, + days_back: int, + metric: str = "view", + ) -> dict[str, Any]: + if previous is not None: + growth = cls._growth_for(snapshot, previous, metric=metric) + payload = growth.as_dict() + payload["baseline"] = "snapshot" if growth.available else None + return payload + if days_back == 1: + fallback = cls._same_day_growth(snapshot, video, metric=metric) + if fallback is not None: + return fallback + payload = compute_growth(None, None, None).as_dict() + payload["baseline"] = None + return payload + def _item( self, video: Mapping[str, Any], @@ -415,6 +489,7 @@ class DashboardQueries: percentile: float | None = None, previous: Mapping[str, Any] | None = None, include_growth: bool = False, + growth_days: int | None = None, ) -> dict[str, Any]: metrics = _metric_map(snapshot) duration = video.get("duration") @@ -455,7 +530,9 @@ class DashboardQueries: ), } if include_growth: - item["growth"] = self._growth_for(snapshot, previous).as_dict() + item["growth"] = self._growth_result( + video, snapshot, previous, days_back=int(growth_days or 1), + ) return item # ------------------------------------------------------------ overview @@ -474,7 +551,7 @@ class DashboardQueries: videos = self._videos(conn) aids = [int(video["aid"]) for video in videos] snapshots = self._snapshot_map(conn, aids) - # 播放增量：按片相对当前点的相邻成功日，避免 seed/recent 拉到 T-2。 + # 播放增量：近 1 日。有更早日用相邻成功日；当日发布相对 0；否则用 view_open。 previous = self._snapshot_previous_relative(conn, snapshots, days_back=1) totals = {metric: 0 for metric in METRICS} for video in videos: @@ -489,12 +566,13 @@ class DashboardQueries: for video in videos: aid = int(video["aid"]) snapshot = snapshots.get(aid) - prior = previous.get(aid) - if snapshot is None or prior is None: + if snapshot is None: continue - growth = self._growth_for(snapshot, prior) - if growth.available and growth.delta is not None: - view_delta_sum += growth.delta + growth = self._growth_result( + video, snapshot, previous.get(aid), days_back=1, + ) + if growth.get("available") and growth.get("delta") is not None: + view_delta_sum += int(growth["delta"]) available_deltas += 1 if available_deltas: view_delta = view_delta_sum @@ -868,6 +946,7 @@ class DashboardQueries: percentile=percentiles.get(int(video["aid"])), previous=previous.get(int(video["aid"])), include_growth=sort in {"growth", "growth_rate"}, + growth_days=growth_days, ) for video in videos ] @@ -977,9 +1056,15 @@ class DashboardQueries: reason = None if sort in {"growth", "growth_rate"} and not growth_available: available = False - reason = ( - f"近 {growth_days} 日增长需要至少 {growth_days + 1} 天快照" - ) + if growth_days == 1: + reason = ( + "近 1 日增长需要前一日快照；当日发布有一次成功采样即可；" + "更早发布的新入库需要当天至少两次成功采样" + ) + else: + reason = ( + f"近 {growth_days} 日增长需要至少 {growth_days + 1} 天快照" + ) duration_filter = None if sort in duration_sorts: duration_filter = { @@ -1031,16 +1116,19 @@ class DashboardQueries: video = dict(row) snapshots = self._snapshot_map(conn, [aid]) snapshot = snapshots.get(aid) - previous = self._snapshot_map(conn, [aid], before=snapshot["stat_date"] if snapshot else None).get(aid) + previous = self._snapshot_map( + conn, [aid], before=snapshot["stat_date"] if snapshot else None, + ).get(aid) _, _, heat, percentiles = self._global_heat(conn) - growth = self._growth_for(snapshot, previous) item = self._item( video, snapshot, heat=heat.get(aid), percentile=percentiles.get(aid), ) - item["growth"] = growth.as_dict() + item["growth"] = self._growth_result( + video, snapshot, previous, days_back=1, + ) item["snapshot_count"] = int(conn.execute( "SELECT COUNT(*) FROM snapshot WHERE aid = ?", (aid,) ).fetchone()[0]) diff --git a/reaction_board/web/static/components.js b/reaction_board/web/static/components.js index 311eac0..3f25c1b 100644 --- a/reaction_board/web/static/components.js +++ b/reaction_board/web/static/components.js @@ -30,6 +30,12 @@ export function avatar(item, className = "avatar") { loading="lazy" referrerpolicy="no-referrer">`; } +function growthBaselineTag(growth) { + if (growth?.baseline === "zero") return "今日"; + if (growth?.baseline === "open") return "入库后"; + return ""; +} + export function videoCard(item, { rank = null, primary = "view", primaryLabel = "", primaryText = null, showStatus = false, @@ -73,7 +79,7 @@ export function videoCard(item, { ${escapeHtml(primaryValue ?? "—")}${escapeHtml(label)} - ${delta !== null ? `${formatDelta(delta)}` : ""} + ${delta !== null ? `${formatDelta(delta)}${growthBaselineTag(growth)}` : ""} 互动 ${formatPercent(item.interaction_rate)} @@ -118,7 +124,7 @@ export function metricGrid(item) { return `
${["view", "like", "coin", "favorite", "share", "reply", "danmaku"].map((metric) => { const extra = metric === "view" && growth?.available - ? `${formatDelta(growth.delta)}` + ? `${formatDelta(growth.delta)}${growthBaselineTag(growth)}` : ""; return `
${escapeHtml(metricLabel(metric))} diff --git a/reaction_board/web/static/views/about.js b/reaction_board/web/static/views/about.js index 16b6470..3416ade 100644 --- a/reaction_board/web/static/views/about.js +++ b/reaction_board/web/static/views/about.js @@ -92,7 +92,7 @@ export async function renderAbout(root) {
  • B 站不公开完播率与单 P 播放，内容小时是“供给时长”，不是“实际观看时长”。
  • 多 P 可能让同一用户产生多次播放计数，本站不单独校正，属于已知偏差。
  • 粉丝数可能包含不活跃账号；粉丝快照暂不参与任何榜单。
    • -
  • 样本不足（少于 2 天快照、单视频 UP）会显示提示，不伪造 0。
    • +
  • 近 7/30 日增长仍要跨日快照。近 1 日：当日发布按 0 起算；更早发布的新入库按入库时播放量起算，需当天至少两次采样。样本不足时显示提示，不伪造 0。
    • diff --git a/reaction_board/web/static/views/home.js b/reaction_board/web/static/views/home.js index fbccdd4..54cb26f 100644 --- a/reaction_board/web/static/views/home.js +++ b/reaction_board/web/static/views/home.js @@ -69,7 +69,7 @@ export async function renderHome(root) { const growthEmptyTitle = growth.reason || `近 ${growthDays} 日增长暂不可用`; const growthEmptyHint = growthDays <= 1 - ? "当前快照不足，完成至少 2 天快照后即可显示播放增量。" + ? "当日发布按 0 起算；更早发布的新入库需要当天至少两次采样。" : `需要至少 ${growthDays + 1} 天快照才能计算近 ${growthDays} 日增长。`; const updatedLabel = formatClock(overview.updated_at); const dataDateLabel = formatDataDate(overview.data_date); diff --git a/reaction_board/web/static/views/rankings.js b/reaction_board/web/static/views/rankings.js index fdf2c98..701c60c 100644 --- a/reaction_board/web/static/views/rankings.js +++ b/reaction_board/web/static/views/rankings.js @@ -179,6 +179,12 @@ function durationNote(data) { return `
    内容效率与互动效率：时长 ≥ ${filter.min_duration_seconds} 秒、发布满 ${filter.maturity_days} 天、播放 ≥ ${filter.min_view}${shrink}；时长缺失不参与。
    `; } +function growthNote(sort, growthDays) { + if (sort !== "growth" && sort !== "growth_rate") return ""; + if (Number(growthDays) !== 1) return ""; + return `
    近 1 日：当日发布按 0 起算；更早发布的新入库按入库时播放量起算，需要当天至少两次采样。
    `; +} + function bindControls(root) { root.querySelector("#rank-sort")?.addEventListener("change", (event) => { replaceQuery({ sort: event.target.value, page: null }); @@ -245,6 +251,7 @@ async function renderScope(root, { query }, scope) { ${controls({ scope, sort, within, year, growthDays, q, years })} ${durationNote(data)} + ${growthNote(sort, growthDays)} ${data.available === false ? `
    ${sort.startsWith("growth") ? "增长数据尚不可用" : "榜单筛选后为空"} ${escapeHtml(data.reason || "")} diff --git a/reaction_board/web/static/views/video.js b/reaction_board/web/static/views/video.js index 43d57f0..2cdc2da 100644 --- a/reaction_board/web/static/views/video.js +++ b/reaction_board/web/static/views/video.js @@ -62,7 +62,7 @@ export async function renderVideo(root, { params, query }) { ${escapeHtml(formatDuration(detail.duration))} ${detail.pages ? `${formatNumber(detail.pages)} P` : ""} ${badge(statusLabel(detail.status), `status-${detail.status}`)} - ${samples < 2 ? badge("样本不足", "warning") : ""} + ${samples < 2 && !detail.growth?.available ? badge("样本不足", "warning") : ""}
    ${detail.blacklisted ? `

    该 UP 在黑名单中；黑名单只影响未来发现，不影响当前视频追踪。

    ` : ""} ${detail.up_follower_backfilled ? `

    UP 粉丝数为回填值，不是真实的入库时点。

    ` : ""} diff --git a/tests/test_analytics.py b/tests/test_analytics.py index 18135e7..f33f350 100644 --- a/tests/test_analytics.py +++ b/tests/test_analytics.py @@ -131,6 +131,18 @@ def test_compute_growth(): no_gap = compute_growth(100, 90, 0) assert no_gap.available is False + same_day = compute_growth(100, 90, 0, allow_zero_gap=True) + assert same_day.available is True + assert same_day.delta == 10 + assert same_day.per_day is None + assert same_day.gap_days == 0 + assert abs(same_day.rate - 10 / 90) < 1e-12 + + from_zero_same_day = compute_growth(50, 0, 0, allow_zero_gap=True) + assert from_zero_same_day.delta == 50 + assert from_zero_same_day.rate == 50.0 + assert from_zero_same_day.per_day is None + growth = compute_growth(1000, 800, 7) assert growth.available is True assert growth.delta == 200 diff --git a/tests/test_cli.py b/tests/test_cli.py index 9f287e5..b5098cb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -51,7 +51,7 @@ def test_init_and_status(): path = write_config(tmp) code, output = run_cli(["init", "--config", str(path)]) assert code == 0 - assert '"schema_version": 7' in output + assert '"schema_version": 8' in output db = Database(Path(tmp) / "app.db") db.initialize() diff --git a/tests/test_database.py b/tests/test_database.py index c979628..94bf6c4 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -24,7 +24,7 @@ def test_write_read_transaction(): with tempfile.TemporaryDirectory() as tmp: db = make_db(tmp) try: - assert db.schema_version == 7 + assert db.schema_version == 8 db.write(lambda c: c.execute("INSERT INTO kv(k, v) VALUES ('a', '1')")) assert db.read(lambda c: c.execute("SELECT v FROM kv WHERE k='a'").fetchone()[0]) == "1" @@ -94,8 +94,8 @@ def test_close_and_reinitialize(): db2 = make_db(tmp) try: - assert db2.schema_version == 7 - assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 7 + assert db2.schema_version == 8 + assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 8 finally: db2.close() @@ -123,7 +123,7 @@ def test_migration_backup_on_existing_db(): db = Database(path, backup_dir=Path(tmp) / "backup") try: applied = db.initialize() - assert applied == [1, 2, 3, 4, 5, 6, 7] + assert applied == [1, 2, 3, 4, 5, 6, 7, 8] assert len(list((Path(tmp) / "backup").glob("app-pre-v0-*.db"))) == 1 finally: db.close() diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 7a6f2a6..70fe41e 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -38,8 +38,8 @@ def test_fresh_and_idempotent(): conn = sqlite3.connect(Path(tmp) / "app.db") try: applied = run_migrations(conn) - assert applied == [1, 2, 3, 4, 5, 6, 7] - assert current_version(conn) == 7 + assert applied == [1, 2, 3, 4, 5, 6, 7, 8] + assert current_version(conn) == 8 assert EXPECTED_TABLES <= table_names(conn) assert "schema_version" in table_names(conn) columns = { @@ -70,10 +70,14 @@ def test_fresh_and_idempotent(): assert { "stat_date", "ip", "path", "hits", "first_at", "last_at", } <= visit_columns + snapshot_columns = { + row[1] for row in conn.execute("PRAGMA table_info(snapshot)") + } + assert {"view_open", "open_captured_at"} <= snapshot_columns # 再跑一次不应重复应用 assert run_migrations(conn) == [] row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone() - assert row[0] == 7 + assert row[0] == 8 finally: conn.close() @@ -88,15 +92,15 @@ def test_backup_before_upgrade(): conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')") conn.commit() - v8 = Migration(8, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",)) + v9 = Migration(9, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",)) applied = run_migrations( conn, backup_dir=backup_dir, now=1_800_000_000, - migrations=MIGRATIONS + (v8,), + migrations=MIGRATIONS + (v9,), ) - assert applied == [8] - backups = list(backup_dir.glob("app-pre-v7-*.db")) + assert applied == [9] + backups = list(backup_dir.glob("app-pre-v8-*.db")) assert len(backups) == 1 check = sqlite3.connect(backups[0]) try: @@ -181,13 +185,46 @@ def test_v5_normalizes_unfaved(): ) conn.commit() applied = run_migrations(conn) - assert applied == [5, 6, 7] + assert applied == [5, 6, 7, 8] rows = dict(conn.execute("SELECT aid, status FROM video").fetchall()) assert rows == {1: "active", 2: "dropped"} finally: conn.close() +def test_v8_backfills_view_open(): + with tempfile.TemporaryDirectory() as tmp: + conn = sqlite3.connect(Path(tmp) / "app.db") + try: + run_migrations(conn, migrations=MIGRATIONS[:7]) + conn.execute( + "INSERT INTO video(aid, title, pubdate, status, first_seen) " + "VALUES (1, 't', 1, 'active', 1)" + ) + conn.execute( + 'INSERT INTO snapshot(stat_date, aid, "view", captured_at, ok) ' + "VALUES ('2026-10-31', 1, 500, 100, 1)" + ) + conn.execute( + 'INSERT INTO snapshot(stat_date, aid, "view", captured_at, ok) ' + "VALUES ('2026-10-30', 1, NULL, 90, 0)" + ) + conn.commit() + assert run_migrations(conn) == [8] + ok_row = conn.execute( + "SELECT view_open, open_captured_at FROM snapshot " + "WHERE stat_date='2026-10-31'" + ).fetchone() + assert ok_row[0] == 500 and ok_row[1] == 100 + fail_row = conn.execute( + "SELECT view_open, open_captured_at FROM snapshot " + "WHERE stat_date='2026-10-30'" + ).fetchone() + assert fail_row[0] is None and fail_row[1] is None + finally: + conn.close() + + def main(): test_fresh_and_idempotent() test_backup_before_upgrade() @@ -195,6 +232,7 @@ def main(): test_prune_backups() test_validate_migration_order() test_v5_normalizes_unfaved() + test_v8_backfills_view_open() print("OK: SQLite 迁移、备份与回滚测试全部通过") diff --git a/tests/test_repository.py b/tests/test_repository.py index 6e39dd1..8c37ca2 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -296,6 +296,59 @@ def test_pending_snapshot_aids(): db.close() +def test_upsert_snapshot_preserves_view_open(): + with tempfile.TemporaryDirectory() as tmp: + db, repo = setup(tmp) + try: + metrics = { + "view": 10, "like": 1, "favorite": 0, "coin": 0, + "share": 0, "reply": 0, "danmaku": 0, + } + repo.upsert_snapshot( + "2026-10-31", 1, metrics=metrics, ok=True, err=None, captured_at=10, + ) + first = repo.get_snapshot("2026-10-31", 1) + assert first["view"] == 10 + assert first["view_open"] == 10 + assert first["open_captured_at"] == 10 + + metrics["view"] = 40 + repo.upsert_snapshot( + "2026-10-31", 1, metrics=metrics, ok=True, err=None, captured_at=20, + ) + second = repo.get_snapshot("2026-10-31", 1) + assert second["view"] == 40 + assert second["view_open"] == 10 + assert second["open_captured_at"] == 10 + assert second["captured_at"] == 20 + + repo.upsert_snapshot( + "2026-10-31", 2, metrics=None, ok=False, err="x", captured_at=1, + ) + failed = repo.get_snapshot("2026-10-31", 2) + assert failed["view_open"] is None + metrics["view"] = 7 + repo.upsert_snapshot( + "2026-10-31", 2, metrics=metrics, ok=True, err=None, captured_at=5, + ) + recovered = repo.get_snapshot("2026-10-31", 2) + assert recovered["view"] == 7 + assert recovered["view_open"] == 7 + assert recovered["open_captured_at"] == 5 + + copied = repo.copy_ok_snapshots( + "2026-10-31", "2026-11-01", captured_at=30, src="seed", + ) + assert copied == 2 + seeded = repo.get_snapshot("2026-11-01", 1) + assert seeded["view"] == 40 + assert seeded["view_open"] == 40 + assert seeded["open_captured_at"] == 30 + assert seeded["src"] == "seed" + finally: + db.close() + + def test_up_follower_snapshot_and_backfill(): with tempfile.TemporaryDirectory() as tmp: db, repo = setup(tmp) @@ -367,6 +420,7 @@ def main(): test_primary_folder_is_order_independent() test_dropped_invalid_preserves_original_title() test_pending_snapshot_aids() + test_upsert_snapshot_preserves_view_open() test_up_follower_snapshot_and_backfill() print("OK: 收藏夹/视频状态机与黑名单测试全部通过") diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index 02856ee..7769bfe 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -444,19 +444,22 @@ def test_run_recent_videos_refreshes_without_completion_markers(): 1: view_for(1, view=111), 2: view_for(2, view=222), 3: view_for(3, view=333), + 4: view_for(4, view=444), }) db, repo, service = make_service(tmp, transport, track_fans=()) try: - # 1/2 在墙钟近 2 天内；3 为旧片 + # 1/2 在墙钟近 2 天内；3 为旧片；4 发布已久但当天入库，不进小时任务 add_video(repo, 1) add_video(repo, 2) add_video(repo, 3) + add_video(repo, 4) repo.db.write(lambda conn: conn.executemany( "UPDATE video SET pubdate = ? WHERE aid = ?", [ (clock - 3600, 1), (clock - 86400, 2), (clock - 10 * 86400, 3), + (clock - 10 * 86400, 4), ], )) # 预置同日 ok=1，验证小时任务可覆盖刷新 @@ -477,23 +480,27 @@ def test_run_recent_videos_refreshes_without_completion_markers(): )) == {1, 2} row = repo.db.read(lambda conn: dict(conn.execute( - "SELECT view, ok, captured_at FROM snapshot WHERE stat_date=? AND aid=1", + "SELECT view, ok, captured_at, view_open FROM snapshot " + "WHERE stat_date=? AND aid=1", (DATE,), ).fetchone())) assert row["view"] == 111 assert row["ok"] == 1 assert row["captured_at"] == clock + assert row["view_open"] == 1 # 完成日 / 备份标记保持原样 assert repo.kv_get(service.completed_key(DATE)) is None assert repo.kv_get("snapshot:last_completed") is None assert repo.kv_get(service.backup_key(DATE)) is None assert transport.count("/web-interface/view") == 2 + assert repo.get_snapshot(DATE, 4) is None # 全量任务随后才写完成日；且已 ok=1 的跳过 full = service.run_videos(date=DATE) assert full.skipped >= 2 # 1、2 已被小时任务写过 - assert full.success >= 1 # 至少旧片 3 + assert full.success >= 1 # 至少旧片 3、4 + assert repo.get_snapshot(DATE, 4) is not None assert repo.kv_get(service.completed_key(DATE)) == "1" assert repo.kv_get("snapshot:last_completed") == DATE finally: @@ -562,6 +569,7 @@ def test_seed_from_last_completed_does_not_mark_completed(): row = repo.get_snapshot(today, 1) assert row is not None assert int(row["view"]) == 10 + assert int(row["view_open"]) == 10 assert row["src"] == "seed" fan = repo.get_up_stat(today, 37754047) assert fan is not None and int(fan["follower"]) == 100 diff --git a/tests/test_web_queries.py b/tests/test_web_queries.py index 5ab401b..af90ca7 100644 --- a/tests/test_web_queries.py +++ b/tests/test_web_queries.py @@ -15,7 +15,7 @@ from reaction_board.repository import Repository # noqa: E402 from reaction_board.timeutil import day_bounds # noqa: E402 from reaction_board.web.cache import DASHBOARD_CACHE # noqa: E402 from reaction_board.web.queries import DashboardQueries # noqa: E402 -from tests.web_fixtures import DATES, build_db, write_config # noqa: E402 +from tests.web_fixtures import DATES, FOLDER, build_db, resource, write_config # noqa: E402 FIXED_NOW = 1_793_459_000 # 2026-10-31 前后 @@ -46,7 +46,7 @@ def test_overview(): assert overview["counts"]["candidates_pending"] == 1 assert overview["totals"]["view"] == 1200 + 5200 + 120 assert overview["interactions"] > 0 - assert overview["view_delta"] == (100 + 100 + 0) # 101/102 的日增量 + assert overview["view_delta"] == (100 + 100 + 120) # 101/102 跨日 + 103 当日发布 assert overview["snapshot_health"]["total"] == 3 assert overview["folders"][0]["media_count"] == 3 assert overview["folders"][0]["ratio"] < 1 @@ -484,7 +484,7 @@ def test_about_payload(): assert about["snapshot_days"] == 3 assert about["snapshot_first_date"] == DATES[0] assert about["fans_start_date"] == DATES[0] - assert about["schema_version"] == 7 + assert about["schema_version"] == 8 assert about["analytics"] == { "min_duration_seconds": 600, "maturity_days": 2, @@ -577,9 +577,13 @@ def test_growth_auto_falls_back_to_one_day(): assert auto["available"] is True assert auto["growth_days"] == 1 assert auto["days"] == 1 - assert {item["aid"] for item in auto["items"]} == {101, 102} + assert {item["aid"] for item in auto["items"]} == {101, 102, 103} first = next(item for item in auto["items"] if item["aid"] == 101) assert first["growth"]["delta"] == 100 # 相邻日 + assert first["growth"]["baseline"] == "snapshot" + today = next(item for item in auto["items"] if item["aid"] == 103) + assert today["growth"]["baseline"] == "zero" + assert today["growth"]["delta"] == 120 forced7 = q.list_videos(sort="growth", growth_days=7) assert forced7["available"] is False @@ -667,9 +671,10 @@ def test_overview_adjacent_previous_with_seed_and_dual_fields(): + overview["counts"]["unfaved"] + overview["counts"]["removed"] ) - # 当前=seed(=完成日数值)，相邻 previous=完成日本身 → 增量 0 + # 101/102：当前=seed(=完成日数值)，相邻 previous=完成日本身 → 增量 0 + # 103：当日发布，近 1 日相对 0，计入当前播放 assert overview["view_delta_available"] is True - assert overview["view_delta"] == 0 + assert overview["view_delta"] == 120 growth = q.list_videos(sort="growth", growth_days=1) assert growth["available"] is True @@ -678,6 +683,93 @@ def test_overview_adjacent_previous_with_seed_and_dual_fields(): assert item["growth"]["available"] is True assert item["growth"]["delta"] == 0 assert item["growth"]["gap_days"] == 1 + assert item["growth"]["baseline"] == "snapshot" + finally: + db.close() + + +def _view_metrics(view: int) -> dict[str, int]: + return { + "view": view, "like": 0, "coin": 0, "favorite": 0, + "share": 0, "reply": 0, "danmaku": 0, + } + + +def test_same_day_growth_baselines(): + """近 1 日：当日发布相对 0；更早发布用 view_open；7 日不用这两条退路。""" + with tempfile.TemporaryDirectory() as tmp: + db, q = setup(tmp) + try: + repo = Repository(db) + today = DATES[-1] + start = int(day_bounds(today)[0]) + old_pub = int(day_bounds("2026-09-01")[0]) + + repo.upsert_snapshot( + today, 103, metrics=_view_metrics(200), + ok=True, err=None, captured_at=start + 3600, + ) + repo.upsert_from_fav( + resource(105, up_mid=1, up_name="UP A", pubdate=old_pub), + FOLDER.media_id, was_added=True, now=start + 10, + ) + repo.upsert_snapshot( + today, 105, metrics=_view_metrics(500), + ok=True, err=None, captured_at=start + 10, + ) + repo.upsert_snapshot( + today, 105, metrics=_view_metrics(800), + ok=True, err=None, captured_at=start + 20, + ) + repo.upsert_from_fav( + resource(106, up_mid=1, up_name="UP A", pubdate=old_pub), + FOLDER.media_id, was_added=True, now=start + 30, + ) + repo.upsert_snapshot( + today, 106, metrics=_view_metrics(900), + ok=True, err=None, captured_at=start + 30, + ) + repo.upsert_from_fav( + resource(107, up_mid=1, up_name="UP A", pubdate=old_pub), + FOLDER.media_id, was_added=True, now=start + 40, + ) + repo.upsert_snapshot( + today, 107, metrics=_view_metrics(50), + ok=True, err=None, captured_at=start + 40, + ) + db.write(lambda conn: conn.execute( + "UPDATE video SET pubdate = 0 WHERE aid = 107" + )) + DASHBOARD_CACHE.clear() + + one = q.list_videos(sort="growth", growth_days=1, page_size=20) + by_aid = {item["aid"]: item for item in one["items"]} + assert 103 in by_aid and 105 in by_aid + assert 106 not in by_aid and 107 not in by_aid + assert by_aid[103]["growth"]["baseline"] == "zero" + assert by_aid[103]["growth"]["delta"] == 200 + assert by_aid[103]["growth"]["gap_days"] == 0 + assert by_aid[105]["growth"]["baseline"] == "open" + assert by_aid[105]["growth"]["delta"] == 300 + assert by_aid[101]["growth"]["baseline"] == "snapshot" + assert by_aid[101]["growth"]["delta"] == 100 + + seven = q.list_videos(sort="growth", growth_days=7) + seven_aids = {item["aid"] for item in seven["items"]} + assert 103 not in seven_aids + assert 105 not in seven_aids + + detail103 = q.video_detail(103) + assert detail103["growth"]["baseline"] == "zero" + assert detail103["growth"]["delta"] == 200 + detail105 = q.video_detail(105) + assert detail105["growth"]["baseline"] == "open" + assert detail105["growth"]["delta"] == 300 + assert q.video_detail(106)["growth"]["available"] is False + assert q.video_detail(107)["growth"]["available"] is False + + overview = q.overview() + assert overview["view_delta"] == 100 + 100 + 200 + 300 finally: db.close() @@ -699,6 +791,7 @@ def main_test(): test_growth_auto_falls_back_to_one_day() test_snapshot_recent_next_run_follows_completion() test_overview_adjacent_previous_with_seed_and_dual_fields() + test_same_day_growth_baselines() print("OK: Web 查询层总览/榜单/详情/UP/时长效率/粉丝/候选/系统/访问统计测试全部通过") diff --git a/tests/test_web_server.py b/tests/test_web_server.py index cfb8872..447cad7 100644 --- a/tests/test_web_server.py +++ b/tests/test_web_server.py @@ -75,7 +75,7 @@ def test_readonly_routes_and_static(): assert status == 200 and payload["ok"] is True status, _, payload = running.json("/api/about") assert status == 200 - assert payload["data"]["schema_version"] == 7 + assert payload["data"]["schema_version"] == 8 assert payload["data"]["analytics"]["min_duration_seconds"] == 600 status, _, overview = running.json("/api/overview") diff --git a/tests/web_fixtures.py b/tests/web_fixtures.py index f67aaff..342e42a 100644 --- a/tests/web_fixtures.py +++ b/tests/web_fixtures.py @@ -138,7 +138,7 @@ def build_db(tmp: str | Path) -> Path: day, 37754047, follower=1000 + index * 10, following=5, ok=True, captured_at=ts(day) + 60, ) - # 103 只有最后一天快照，用于验证样本不足；104 另加一条失败快照 + # 103 只有最后一天快照（发布日=快照日，近 1 日相对 0）；104 另加一条失败快照 repo.upsert_snapshot( DATES[-1], 103, metrics={"view": 120, "like": 5, "coin": 1, "favorite": 2, "share": 0, "reply": 1, "danmaku": 1}, diff --git a/tools/seed_preview_db.py b/tools/seed_preview_db.py index 0ed8973..20302e2 100644 --- a/tools/seed_preview_db.py +++ b/tools/seed_preview_db.py @@ -237,8 +237,8 @@ def _insert_snapshot( """ INSERT OR REPLACE INTO snapshot( stat_date, aid, "view", "like", favorite, coin, share, reply, - danmaku, captured_at, ok, err, src - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'preview') + danmaku, captured_at, ok, err, src, view_open, open_captured_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'preview', ?, ?) """, ( stat_date, @@ -253,6 +253,8 @@ def _insert_snapshot( int(captured_at), 1 if ok else 0, err, + values.get("view") if ok else None, + int(captured_at) if ok else None, ), )