feat(growth): same-day 1-day baselines for new videos
Let 1-day view growth use a zero baseline for videos published today, and the first successful sample for older videos ingested today, without mixing those into 7/30-day rankings.
This commit is contained in:
1 parent
8e7ead1669
commit
c5b7185675
21 files changed
+411
-59
No files matched your search
@@ -131,6 +131,18 @@ def test_compute_growth():
|
||||
no_gap = compute_growth(100, 90, 0)
|
||||
assert no_gap.available is False
|
||||
|
||||
same_day = compute_growth(100, 90, 0, allow_zero_gap=True)
|
||||
assert same_day.available is True
|
||||
assert same_day.delta == 10
|
||||
assert same_day.per_day is None
|
||||
assert same_day.gap_days == 0
|
||||
assert abs(same_day.rate - 10 / 90) < 1e-12
|
||||
|
||||
from_zero_same_day = compute_growth(50, 0, 0, allow_zero_gap=True)
|
||||
assert from_zero_same_day.delta == 50
|
||||
assert from_zero_same_day.rate == 50.0
|
||||
assert from_zero_same_day.per_day is None
|
||||
|
||||
growth = compute_growth(1000, 800, 7)
|
||||
assert growth.available is True
|
||||
assert growth.delta == 200
|
||||
|
||||
+1
-1
@@ -51,7 +51,7 @@ def test_init_and_status():
|
||||
path = write_config(tmp)
|
||||
code, output = run_cli(["init", "--config", str(path)])
|
||||
assert code == 0
|
||||
assert '"schema_version": 7' in output
|
||||
assert '"schema_version": 8' in output
|
||||
|
||||
db = Database(Path(tmp) / "app.db")
|
||||
db.initialize()
|
||||
|
||||
@@ -24,7 +24,7 @@ def test_write_read_transaction():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db = make_db(tmp)
|
||||
try:
|
||||
assert db.schema_version == 7
|
||||
assert db.schema_version == 8
|
||||
db.write(lambda c: c.execute("INSERT INTO kv(k, v) VALUES ('a', '1')"))
|
||||
assert db.read(lambda c: c.execute("SELECT v FROM kv WHERE k='a'").fetchone()[0]) == "1"
|
||||
|
||||
@@ -94,8 +94,8 @@ def test_close_and_reinitialize():
|
||||
|
||||
db2 = make_db(tmp)
|
||||
try:
|
||||
assert db2.schema_version == 7
|
||||
assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 7
|
||||
assert db2.schema_version == 8
|
||||
assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 8
|
||||
finally:
|
||||
db2.close()
|
||||
|
||||
@@ -123,7 +123,7 @@ def test_migration_backup_on_existing_db():
|
||||
db = Database(path, backup_dir=Path(tmp) / "backup")
|
||||
try:
|
||||
applied = db.initialize()
|
||||
assert applied == [1, 2, 3, 4, 5, 6, 7]
|
||||
assert applied == [1, 2, 3, 4, 5, 6, 7, 8]
|
||||
assert len(list((Path(tmp) / "backup").glob("app-pre-v0-*.db"))) == 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -38,8 +38,8 @@ def test_fresh_and_idempotent():
|
||||
conn = sqlite3.connect(Path(tmp) / "app.db")
|
||||
try:
|
||||
applied = run_migrations(conn)
|
||||
assert applied == [1, 2, 3, 4, 5, 6, 7]
|
||||
assert current_version(conn) == 7
|
||||
assert applied == [1, 2, 3, 4, 5, 6, 7, 8]
|
||||
assert current_version(conn) == 8
|
||||
assert EXPECTED_TABLES <= table_names(conn)
|
||||
assert "schema_version" in table_names(conn)
|
||||
columns = {
|
||||
@@ -70,10 +70,14 @@ def test_fresh_and_idempotent():
|
||||
assert {
|
||||
"stat_date", "ip", "path", "hits", "first_at", "last_at",
|
||||
} <= visit_columns
|
||||
snapshot_columns = {
|
||||
row[1] for row in conn.execute("PRAGMA table_info(snapshot)")
|
||||
}
|
||||
assert {"view_open", "open_captured_at"} <= snapshot_columns
|
||||
# 再跑一次不应重复应用
|
||||
assert run_migrations(conn) == []
|
||||
row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone()
|
||||
assert row[0] == 7
|
||||
assert row[0] == 8
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@@ -88,15 +92,15 @@ def test_backup_before_upgrade():
|
||||
conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')")
|
||||
conn.commit()
|
||||
|
||||
v8 = Migration(8, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
|
||||
v9 = Migration(9, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
|
||||
applied = run_migrations(
|
||||
conn,
|
||||
backup_dir=backup_dir,
|
||||
now=1_800_000_000,
|
||||
migrations=MIGRATIONS + (v8,),
|
||||
migrations=MIGRATIONS + (v9,),
|
||||
)
|
||||
assert applied == [8]
|
||||
backups = list(backup_dir.glob("app-pre-v7-*.db"))
|
||||
assert applied == [9]
|
||||
backups = list(backup_dir.glob("app-pre-v8-*.db"))
|
||||
assert len(backups) == 1
|
||||
check = sqlite3.connect(backups[0])
|
||||
try:
|
||||
@@ -181,13 +185,46 @@ def test_v5_normalizes_unfaved():
|
||||
)
|
||||
conn.commit()
|
||||
applied = run_migrations(conn)
|
||||
assert applied == [5, 6, 7]
|
||||
assert applied == [5, 6, 7, 8]
|
||||
rows = dict(conn.execute("SELECT aid, status FROM video").fetchall())
|
||||
assert rows == {1: "active", 2: "dropped"}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_v8_backfills_view_open():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
conn = sqlite3.connect(Path(tmp) / "app.db")
|
||||
try:
|
||||
run_migrations(conn, migrations=MIGRATIONS[:7])
|
||||
conn.execute(
|
||||
"INSERT INTO video(aid, title, pubdate, status, first_seen) "
|
||||
"VALUES (1, 't', 1, 'active', 1)"
|
||||
)
|
||||
conn.execute(
|
||||
'INSERT INTO snapshot(stat_date, aid, "view", captured_at, ok) '
|
||||
"VALUES ('2026-10-31', 1, 500, 100, 1)"
|
||||
)
|
||||
conn.execute(
|
||||
'INSERT INTO snapshot(stat_date, aid, "view", captured_at, ok) '
|
||||
"VALUES ('2026-10-30', 1, NULL, 90, 0)"
|
||||
)
|
||||
conn.commit()
|
||||
assert run_migrations(conn) == [8]
|
||||
ok_row = conn.execute(
|
||||
"SELECT view_open, open_captured_at FROM snapshot "
|
||||
"WHERE stat_date='2026-10-31'"
|
||||
).fetchone()
|
||||
assert ok_row[0] == 500 and ok_row[1] == 100
|
||||
fail_row = conn.execute(
|
||||
"SELECT view_open, open_captured_at FROM snapshot "
|
||||
"WHERE stat_date='2026-10-30'"
|
||||
).fetchone()
|
||||
assert fail_row[0] is None and fail_row[1] is None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def main():
|
||||
test_fresh_and_idempotent()
|
||||
test_backup_before_upgrade()
|
||||
@@ -195,6 +232,7 @@ def main():
|
||||
test_prune_backups()
|
||||
test_validate_migration_order()
|
||||
test_v5_normalizes_unfaved()
|
||||
test_v8_backfills_view_open()
|
||||
print("OK: SQLite 迁移、备份与回滚测试全部通过")
|
||||
|
||||
|
||||
|
||||
@@ -296,6 +296,59 @@ def test_pending_snapshot_aids():
|
||||
db.close()
|
||||
|
||||
|
||||
def test_upsert_snapshot_preserves_view_open():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, repo = setup(tmp)
|
||||
try:
|
||||
metrics = {
|
||||
"view": 10, "like": 1, "favorite": 0, "coin": 0,
|
||||
"share": 0, "reply": 0, "danmaku": 0,
|
||||
}
|
||||
repo.upsert_snapshot(
|
||||
"2026-10-31", 1, metrics=metrics, ok=True, err=None, captured_at=10,
|
||||
)
|
||||
first = repo.get_snapshot("2026-10-31", 1)
|
||||
assert first["view"] == 10
|
||||
assert first["view_open"] == 10
|
||||
assert first["open_captured_at"] == 10
|
||||
|
||||
metrics["view"] = 40
|
||||
repo.upsert_snapshot(
|
||||
"2026-10-31", 1, metrics=metrics, ok=True, err=None, captured_at=20,
|
||||
)
|
||||
second = repo.get_snapshot("2026-10-31", 1)
|
||||
assert second["view"] == 40
|
||||
assert second["view_open"] == 10
|
||||
assert second["open_captured_at"] == 10
|
||||
assert second["captured_at"] == 20
|
||||
|
||||
repo.upsert_snapshot(
|
||||
"2026-10-31", 2, metrics=None, ok=False, err="x", captured_at=1,
|
||||
)
|
||||
failed = repo.get_snapshot("2026-10-31", 2)
|
||||
assert failed["view_open"] is None
|
||||
metrics["view"] = 7
|
||||
repo.upsert_snapshot(
|
||||
"2026-10-31", 2, metrics=metrics, ok=True, err=None, captured_at=5,
|
||||
)
|
||||
recovered = repo.get_snapshot("2026-10-31", 2)
|
||||
assert recovered["view"] == 7
|
||||
assert recovered["view_open"] == 7
|
||||
assert recovered["open_captured_at"] == 5
|
||||
|
||||
copied = repo.copy_ok_snapshots(
|
||||
"2026-10-31", "2026-11-01", captured_at=30, src="seed",
|
||||
)
|
||||
assert copied == 2
|
||||
seeded = repo.get_snapshot("2026-11-01", 1)
|
||||
assert seeded["view"] == 40
|
||||
assert seeded["view_open"] == 40
|
||||
assert seeded["open_captured_at"] == 30
|
||||
assert seeded["src"] == "seed"
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_up_follower_snapshot_and_backfill():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, repo = setup(tmp)
|
||||
@@ -367,6 +420,7 @@ def main():
|
||||
test_primary_folder_is_order_independent()
|
||||
test_dropped_invalid_preserves_original_title()
|
||||
test_pending_snapshot_aids()
|
||||
test_upsert_snapshot_preserves_view_open()
|
||||
test_up_follower_snapshot_and_backfill()
|
||||
print("OK: 收藏夹/视频状态机与黑名单测试全部通过")
|
||||
|
||||
|
||||
+11
-3
@@ -444,19 +444,22 @@ def test_run_recent_videos_refreshes_without_completion_markers():
|
||||
1: view_for(1, view=111),
|
||||
2: view_for(2, view=222),
|
||||
3: view_for(3, view=333),
|
||||
4: view_for(4, view=444),
|
||||
})
|
||||
db, repo, service = make_service(tmp, transport, track_fans=())
|
||||
try:
|
||||
# 1/2 在墙钟近 2 天内;3 为旧片
|
||||
# 1/2 在墙钟近 2 天内;3 为旧片;4 发布已久但当天入库,不进小时任务
|
||||
add_video(repo, 1)
|
||||
add_video(repo, 2)
|
||||
add_video(repo, 3)
|
||||
add_video(repo, 4)
|
||||
repo.db.write(lambda conn: conn.executemany(
|
||||
"UPDATE video SET pubdate = ? WHERE aid = ?",
|
||||
[
|
||||
(clock - 3600, 1),
|
||||
(clock - 86400, 2),
|
||||
(clock - 10 * 86400, 3),
|
||||
(clock - 10 * 86400, 4),
|
||||
],
|
||||
))
|
||||
# 预置同日 ok=1,验证小时任务可覆盖刷新
|
||||
@@ -477,23 +480,27 @@ def test_run_recent_videos_refreshes_without_completion_markers():
|
||||
)) == {1, 2}
|
||||
|
||||
row = repo.db.read(lambda conn: dict(conn.execute(
|
||||
"SELECT view, ok, captured_at FROM snapshot WHERE stat_date=? AND aid=1",
|
||||
"SELECT view, ok, captured_at, view_open FROM snapshot "
|
||||
"WHERE stat_date=? AND aid=1",
|
||||
(DATE,),
|
||||
).fetchone()))
|
||||
assert row["view"] == 111
|
||||
assert row["ok"] == 1
|
||||
assert row["captured_at"] == clock
|
||||
assert row["view_open"] == 1
|
||||
|
||||
# 完成日 / 备份标记保持原样
|
||||
assert repo.kv_get(service.completed_key(DATE)) is None
|
||||
assert repo.kv_get("snapshot:last_completed") is None
|
||||
assert repo.kv_get(service.backup_key(DATE)) is None
|
||||
assert transport.count("/web-interface/view") == 2
|
||||
assert repo.get_snapshot(DATE, 4) is None
|
||||
|
||||
# 全量任务随后才写完成日;且已 ok=1 的跳过
|
||||
full = service.run_videos(date=DATE)
|
||||
assert full.skipped >= 2 # 1、2 已被小时任务写过
|
||||
assert full.success >= 1 # 至少旧片 3
|
||||
assert full.success >= 1 # 至少旧片 3、4
|
||||
assert repo.get_snapshot(DATE, 4) is not None
|
||||
assert repo.kv_get(service.completed_key(DATE)) == "1"
|
||||
assert repo.kv_get("snapshot:last_completed") == DATE
|
||||
finally:
|
||||
@@ -562,6 +569,7 @@ def test_seed_from_last_completed_does_not_mark_completed():
|
||||
row = repo.get_snapshot(today, 1)
|
||||
assert row is not None
|
||||
assert int(row["view"]) == 10
|
||||
assert int(row["view_open"]) == 10
|
||||
assert row["src"] == "seed"
|
||||
fan = repo.get_up_stat(today, 37754047)
|
||||
assert fan is not None and int(fan["follower"]) == 100
|
||||
|
||||
@@ -15,7 +15,7 @@ from reaction_board.repository import Repository # noqa: E402
|
||||
from reaction_board.timeutil import day_bounds # noqa: E402
|
||||
from reaction_board.web.cache import DASHBOARD_CACHE # noqa: E402
|
||||
from reaction_board.web.queries import DashboardQueries # noqa: E402
|
||||
from tests.web_fixtures import DATES, build_db, write_config # noqa: E402
|
||||
from tests.web_fixtures import DATES, FOLDER, build_db, resource, write_config # noqa: E402
|
||||
|
||||
FIXED_NOW = 1_793_459_000 # 2026-10-31 前后
|
||||
|
||||
@@ -46,7 +46,7 @@ def test_overview():
|
||||
assert overview["counts"]["candidates_pending"] == 1
|
||||
assert overview["totals"]["view"] == 1200 + 5200 + 120
|
||||
assert overview["interactions"] > 0
|
||||
assert overview["view_delta"] == (100 + 100 + 0) # 101/102 的日增量
|
||||
assert overview["view_delta"] == (100 + 100 + 120) # 101/102 跨日 + 103 当日发布
|
||||
assert overview["snapshot_health"]["total"] == 3
|
||||
assert overview["folders"][0]["media_count"] == 3
|
||||
assert overview["folders"][0]["ratio"] < 1
|
||||
@@ -484,7 +484,7 @@ def test_about_payload():
|
||||
assert about["snapshot_days"] == 3
|
||||
assert about["snapshot_first_date"] == DATES[0]
|
||||
assert about["fans_start_date"] == DATES[0]
|
||||
assert about["schema_version"] == 7
|
||||
assert about["schema_version"] == 8
|
||||
assert about["analytics"] == {
|
||||
"min_duration_seconds": 600,
|
||||
"maturity_days": 2,
|
||||
@@ -577,9 +577,13 @@ def test_growth_auto_falls_back_to_one_day():
|
||||
assert auto["available"] is True
|
||||
assert auto["growth_days"] == 1
|
||||
assert auto["days"] == 1
|
||||
assert {item["aid"] for item in auto["items"]} == {101, 102}
|
||||
assert {item["aid"] for item in auto["items"]} == {101, 102, 103}
|
||||
first = next(item for item in auto["items"] if item["aid"] == 101)
|
||||
assert first["growth"]["delta"] == 100 # 相邻日
|
||||
assert first["growth"]["baseline"] == "snapshot"
|
||||
today = next(item for item in auto["items"] if item["aid"] == 103)
|
||||
assert today["growth"]["baseline"] == "zero"
|
||||
assert today["growth"]["delta"] == 120
|
||||
|
||||
forced7 = q.list_videos(sort="growth", growth_days=7)
|
||||
assert forced7["available"] is False
|
||||
@@ -667,9 +671,10 @@ def test_overview_adjacent_previous_with_seed_and_dual_fields():
|
||||
+ overview["counts"]["unfaved"]
|
||||
+ overview["counts"]["removed"]
|
||||
)
|
||||
# 当前=seed(=完成日数值),相邻 previous=完成日本身 → 增量 0
|
||||
# 101/102:当前=seed(=完成日数值),相邻 previous=完成日本身 → 增量 0
|
||||
# 103:当日发布,近 1 日相对 0,计入当前播放
|
||||
assert overview["view_delta_available"] is True
|
||||
assert overview["view_delta"] == 0
|
||||
assert overview["view_delta"] == 120
|
||||
|
||||
growth = q.list_videos(sort="growth", growth_days=1)
|
||||
assert growth["available"] is True
|
||||
@@ -678,6 +683,93 @@ def test_overview_adjacent_previous_with_seed_and_dual_fields():
|
||||
assert item["growth"]["available"] is True
|
||||
assert item["growth"]["delta"] == 0
|
||||
assert item["growth"]["gap_days"] == 1
|
||||
assert item["growth"]["baseline"] == "snapshot"
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _view_metrics(view: int) -> dict[str, int]:
|
||||
return {
|
||||
"view": view, "like": 0, "coin": 0, "favorite": 0,
|
||||
"share": 0, "reply": 0, "danmaku": 0,
|
||||
}
|
||||
|
||||
|
||||
def test_same_day_growth_baselines():
|
||||
"""近 1 日:当日发布相对 0;更早发布用 view_open;7 日不用这两条退路。"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, q = setup(tmp)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
today = DATES[-1]
|
||||
start = int(day_bounds(today)[0])
|
||||
old_pub = int(day_bounds("2026-09-01")[0])
|
||||
|
||||
repo.upsert_snapshot(
|
||||
today, 103, metrics=_view_metrics(200),
|
||||
ok=True, err=None, captured_at=start + 3600,
|
||||
)
|
||||
repo.upsert_from_fav(
|
||||
resource(105, up_mid=1, up_name="UP A", pubdate=old_pub),
|
||||
FOLDER.media_id, was_added=True, now=start + 10,
|
||||
)
|
||||
repo.upsert_snapshot(
|
||||
today, 105, metrics=_view_metrics(500),
|
||||
ok=True, err=None, captured_at=start + 10,
|
||||
)
|
||||
repo.upsert_snapshot(
|
||||
today, 105, metrics=_view_metrics(800),
|
||||
ok=True, err=None, captured_at=start + 20,
|
||||
)
|
||||
repo.upsert_from_fav(
|
||||
resource(106, up_mid=1, up_name="UP A", pubdate=old_pub),
|
||||
FOLDER.media_id, was_added=True, now=start + 30,
|
||||
)
|
||||
repo.upsert_snapshot(
|
||||
today, 106, metrics=_view_metrics(900),
|
||||
ok=True, err=None, captured_at=start + 30,
|
||||
)
|
||||
repo.upsert_from_fav(
|
||||
resource(107, up_mid=1, up_name="UP A", pubdate=old_pub),
|
||||
FOLDER.media_id, was_added=True, now=start + 40,
|
||||
)
|
||||
repo.upsert_snapshot(
|
||||
today, 107, metrics=_view_metrics(50),
|
||||
ok=True, err=None, captured_at=start + 40,
|
||||
)
|
||||
db.write(lambda conn: conn.execute(
|
||||
"UPDATE video SET pubdate = 0 WHERE aid = 107"
|
||||
))
|
||||
DASHBOARD_CACHE.clear()
|
||||
|
||||
one = q.list_videos(sort="growth", growth_days=1, page_size=20)
|
||||
by_aid = {item["aid"]: item for item in one["items"]}
|
||||
assert 103 in by_aid and 105 in by_aid
|
||||
assert 106 not in by_aid and 107 not in by_aid
|
||||
assert by_aid[103]["growth"]["baseline"] == "zero"
|
||||
assert by_aid[103]["growth"]["delta"] == 200
|
||||
assert by_aid[103]["growth"]["gap_days"] == 0
|
||||
assert by_aid[105]["growth"]["baseline"] == "open"
|
||||
assert by_aid[105]["growth"]["delta"] == 300
|
||||
assert by_aid[101]["growth"]["baseline"] == "snapshot"
|
||||
assert by_aid[101]["growth"]["delta"] == 100
|
||||
|
||||
seven = q.list_videos(sort="growth", growth_days=7)
|
||||
seven_aids = {item["aid"] for item in seven["items"]}
|
||||
assert 103 not in seven_aids
|
||||
assert 105 not in seven_aids
|
||||
|
||||
detail103 = q.video_detail(103)
|
||||
assert detail103["growth"]["baseline"] == "zero"
|
||||
assert detail103["growth"]["delta"] == 200
|
||||
detail105 = q.video_detail(105)
|
||||
assert detail105["growth"]["baseline"] == "open"
|
||||
assert detail105["growth"]["delta"] == 300
|
||||
assert q.video_detail(106)["growth"]["available"] is False
|
||||
assert q.video_detail(107)["growth"]["available"] is False
|
||||
|
||||
overview = q.overview()
|
||||
assert overview["view_delta"] == 100 + 100 + 200 + 300
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -699,6 +791,7 @@ def main_test():
|
||||
test_growth_auto_falls_back_to_one_day()
|
||||
test_snapshot_recent_next_run_follows_completion()
|
||||
test_overview_adjacent_previous_with_seed_and_dual_fields()
|
||||
test_same_day_growth_baselines()
|
||||
print("OK: Web 查询层总览/榜单/详情/UP/时长效率/粉丝/候选/系统/访问统计测试全部通过")
|
||||
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ def test_readonly_routes_and_static():
|
||||
assert status == 200 and payload["ok"] is True
|
||||
status, _, payload = running.json("/api/about")
|
||||
assert status == 200
|
||||
assert payload["data"]["schema_version"] == 7
|
||||
assert payload["data"]["schema_version"] == 8
|
||||
assert payload["data"]["analytics"]["min_duration_seconds"] == 600
|
||||
|
||||
status, _, overview = running.json("/api/overview")
|
||||
|
||||
@@ -138,7 +138,7 @@ def build_db(tmp: str | Path) -> Path:
|
||||
day, 37754047, follower=1000 + index * 10, following=5,
|
||||
ok=True, captured_at=ts(day) + 60,
|
||||
)
|
||||
# 103 只有最后一天快照,用于验证样本不足;104 另加一条失败快照
|
||||
# 103 只有最后一天快照(发布日=快照日,近 1 日相对 0);104 另加一条失败快照
|
||||
repo.upsert_snapshot(
|
||||
DATES[-1], 103,
|
||||
metrics={"view": 120, "like": 5, "coin": 1, "favorite": 2, "share": 0, "reply": 1, "danmaku": 1},
|
||||
|
||||
Reference in new issue
Block a user