Compare commits

..
3 Commits
Author SHA1 Message Date
bot-ci d0bc724eee fix(system): drop frontend taskRows sort; rely on backend started_at order 2026-11-02 19:43:43 +08:00
bot-ci e6883f99a4 fix(system): sort recent tasks by started_at desc
进行中任务可能尚无 finished_at;改为按 started_at 倒序,
缺开始时间的排末尾。测例用 finished 错开,避免误用完成时间。
2026-11-02 19:38:34 +08:00
bot-ci e7b48ba5d9 fix(system): sort recent tasks by finished_at desc
「最近任务」原先按 kv 键名排序;改为按 finished_at 倒序,
缺完成时间的排末尾。前端渲染侧同步按 finished_at 排序兜底。
2026-11-02 19:30:08 +08:00
36 changed files with 202 additions and 3502 deletions

No files matched your search

+2 -14
View File
@@ -456,9 +456,6 @@ P1 实现说明:
- 时区使用固定 UTC+8(中国无夏令时),避免 Windows 缺少 `tzdata`; - 时区使用固定 UTC+8(中国无夏令时),避免 Windows 缺少 `tzdata`;
- 午夜 `seed_at`(默认 `00:00`)把 `snapshot:last_completed` 的 ok=1 行克隆为当日壳,写入 `snapshot:seeded:<date>=1` / `fans:seeded:<date>=1`,**不**推进完成日; - 午夜 `seed_at`(默认 `00:00`)把 `snapshot:last_completed` 的 ok=1 行克隆为当日壳,写入 `snapshot:seeded:<date>=1` / `fans:seeded:<date>=1`,**不**推进完成日;
- 也可手动:`python app.py snapshot-seed [--date YYYY-MM-DD]`(Debian / `docker compose exec re-xxm python app.py snapshot-seed`);
- 目标日已有 `snapshot:completed:<date>` 时 seed 记 `already_completed` 成功(即使 `last_completed` 已等于当日),**不得**误判 `no_source`;
- 若全站 `snapshot` 正在跑,或目标日未完成且 `last_completed` 暂不可用(`no_source`),seed **不记成功**,约 10 分钟后重试直到真正克隆成功;
- 晚间 `snapshot_at`(默认 `23:15`)对 seeded 日 force 重采;成功后写 `snapshot:completed:<date>=1`、推进 `last_completed` 并清除 seeded; - 晚间 `snapshot_at`(默认 `23:15`)对 seeded 日 force 重采;成功后写 `snapshot:completed:<date>=1`、推进 `last_completed` 并清除 seeded;
- 当日全部视频与粉丝采集成功后在 `kv` 写入 `snapshot:completed:<date>=1`;启动时未完成会补跑(纯 seed 日不误标完成); - 当日全部视频与粉丝采集成功后在 `kv` 写入 `snapshot:completed:<date>=1`;启动时未完成会补跑(纯 seed 日不误标完成);
- 完成标记不是唯一门槛:每次检查 `pending_snapshot_aids(date)`,新入库或从 `dropped` 恢复的视频会立即补采当天快照; - 完成标记不是唯一门槛:每次检查 `pending_snapshot_aids(date)`,新入库或从 `dropped` 恢复的视频会立即补采当天快照;
@@ -469,15 +466,6 @@ P1 实现说明:
- Cookie 登录态通过 `x/web-interface/nav` 检查,失效只告警,不阻塞匿名 `view`; - Cookie 登录态通过 `x/web-interface/nav` 检查,失效只告警,不阻塞匿名 `view`;
- Telegram 摘要推迟到 P3,P1 先写结构化 JSON 日志和 `kv` 状态。 - Telegram 摘要推迟到 P3,P1 先写结构化 JSON 日志和 `kv` 状态。
调度避让与补跑:
- 常驻启动时日更任务(`snapshot_seed` / `snapshot` / `fans`)按墙钟下一档 `seed_at` / `snapshot_at` 排期(可参考上次成功/失败记录),**不得** `next_run=0` 重启即跑;
- 仅到达当日 `snapshot_at` 后(或 CLI `--force`)才对 seed 壳 `force` 打穿;白天重启不因 `snapshot:seeded` 全量强采;
- 全站 `snapshot` **不因**其它任务 running 而 skip;到期/强制必须执行;
- 上海时区 **23:00–01:00** 为全站快照窗:`discovery` / `snapshot_recent` / `candidate_cleanup` 不执行,延后到 01:00 后;`fans` 短耗时,不避开任何任务;
- 任务 skip(含静默窗延后)写入 `ok=false` + `skipped=true`,**不得**记成成功或推进「当日已完成」;
- skip / fail 后短间隔补跑(约 5 分钟;`snapshot_seed` 避让/无源为约 10 分钟);日更成功才推到次日 `snapshot_at` / `seed_at`。
--- ---
## 8. Telegram 与入库操作 ## 8. Telegram 与入库操作
@@ -620,7 +608,7 @@ interaction_rate = (like + coin + favorite) / max(view, 1)
- 查询前一有效快照而不是填补缺失日期; - 查询前一有效快照而不是填补缺失日期;
- 缺失日期在图表中断开; - 缺失日期在图表中断开;
- 增速榜要求播放基数至少 1000; - 增速榜要求播放基数至少 200;
- 互动率榜要求播放至少 100; - 互动率榜要求播放至少 100;
- 样本不足两天的视频明确标注,不伪造增长数据。 - 样本不足两天的视频明确标注,不伪造增长数据。
@@ -768,7 +756,7 @@ uv run python tests/test_moves.py
- 粉丝采集对每条 `relation/stat` 加 45s 硬超时看门狗(防 DNS/传输层挂起),每 25 条输出 `up_profile.progress` 进度;单条失败不阻塞整批; - 粉丝采集对每条 `relation/stat` 加 45s 硬超时看门狗(防 DNS/传输层挂起),每 25 条输出 `up_profile.progress` 进度;单条失败不阻塞整批;
- `app.py backfill-followers`:一次性回填历史视频粉丝数,支持 `--limit` / `--dry-run`; - `app.py backfill-followers`:一次性回填历史视频粉丝数,支持 `--limit` / `--dry-run`;
- 新增内容效率/互动效率榜与 UP 内容效率(Gamma–Poisson EB 收缩),过滤 < 600 秒与未满 2 天的视频; - 新增内容效率/互动效率榜与 UP 内容效率(Gamma–Poisson EB 收缩),过滤 < 600 秒与未满 2 天的视频;
- `analytics` 配置段:`min_duration_seconds`、`maturity_days`、`min_view_for_ranking`、`video_shrink_minutes`、`shrink_minutes`、`growth_rate_prior`; - `analytics` 配置段:`min_duration_seconds`、`maturity_days`、`min_view_for_ranking`、`video_shrink_minutes`、`shrink_minutes`;
- 主导航拆分:`/videos`(视频)与 `/ups`(UP主)独立;旧 `/rank` 保留兼容; - 主导航拆分:`/videos`(视频)与 `/ups`(UP主)独立;旧 `/rank` 保留兼容;
- 新增 `/about` 数据说明页与 `/api/about`; - 新增 `/about` 数据说明页与 `/api/about`;
- 卡片精简:列表只保留主指标与互动率,明细移到详情页; - 卡片精简:列表只保留主指标与互动率,明细移到详情页;
-1
View File
@@ -108,6 +108,5 @@
"video_shrink_minutes": 60, "video_shrink_minutes": 60,
"shrink_minutes": null, "shrink_minutes": null,
"home_growth_days": "auto", "home_growth_days": "auto",
"growth_rate_prior": 1000,
}, },
} }
-2
View File
@@ -123,7 +123,5 @@
"shrink_minutes": null, "shrink_minutes": null,
// "auto" = 7 日有数据用 7,否则 1 // "auto" = 7 日有数据用 7,否则 1
"home_growth_days": "auto", "home_growth_days": "auto",
// 增速榜排序:Δ / (基线 + prior)。卡片上的百分比仍是 Δ / 基线。
"growth_rate_prior": 1000,
}, },
} }
+5 -10
View File
@@ -380,7 +380,7 @@ for folder in 配置的年度夹(2025、2026、…):
- 两个夹共 831 条,一次性导入 `video` 表; - 两个夹共 831 条,一次性导入 `video` 表;
- **历史快照无法回填**:B 站不提供任意日期历史数据,只能从部署当天开始积累; - **历史快照无法回填**:B 站不提供任意日期历史数据,只能从部署当天开始积累;
- **开发联调阶段产生的数据一律保留**(不清库、不重建),正式上线直接续用同一个 `app.db`; - **开发联调阶段产生的数据一律保留**(不清库、不重建),正式上线直接续用同一个 `app.db`;
- 新发现/新入库的视频从"入库当天"开始积累快照。近 1 日:当日发布按 0 起算(一次成功采样即可);更早发布的新入库按入库时播放量起算,第二次采样走晚间全量,不进小时快照。近 7/30 日仍要跨日快照。 - 新发现/新入库的视频从"入库当天"开始积累快照,第一天没有增量是正常的,看板对"仅 1 个数据点"的视频标注"样本不足"。
--- ---
@@ -473,8 +473,6 @@ CREATE TABLE snapshot (
ok INTEGER NOT NULL DEFAULT 1, ok INTEGER NOT NULL DEFAULT 1,
err TEXT, err TEXT,
src TEXT DEFAULT 'view', -- view | infos(降级时标记) src TEXT DEFAULT 'view', -- view | infos(降级时标记)
view_open INTEGER, -- 同日第一次成功采样的播放量
open_captured_at INTEGER, -- 开盘采样时刻
PRIMARY KEY (stat_date, aid) PRIMARY KEY (stat_date, aid)
); );
CREATE INDEX idx_snap_aid ON snapshot(aid, stat_date); CREATE INDEX idx_snap_aid ON snapshot(aid, stat_date);
@@ -674,26 +672,23 @@ w = {view:0.30, like:0.20, coin:0.20, favorite:0.15,
### 8.2 增长榜 ### 8.2 增长榜
> 看板 UI 中“增长”标注为**播放增量**(区间内绝对增量),“增速”标注为**播放增速**(增量 / 基线播放)。近 7/30 日基线取不早于 `最新快照 - N 天` 的最近一条成功快照。近 1 日:有更早日期快照则用前一日;否则当日发布基准为 0,更早发布的新入库用当天第一次成功采样(`view_open`)。 > 看板 UI 中“增长”标注为**播放增量**(区间内绝对增量),“增速”标注为**播放增速**(增量 / 基线播放)。两者基线都取不早于 `最新快照 - N 天` 的最近一条成功快照。
``` ```
d7 = (view_today - view_{today-7}) / 实际间隔天数 d7 = (view_today - view_{today-7}) / 实际间隔天数
r7 = (view_today - view_{today-7}) / max(view_{today-7}, 1) r7 = (view_today - view_{today-7}) / max(view_{today-7}, 1)
d1_published_today = view_today - 0 # 只进增量榜,不计算增速
d1_imported_today = view_today - view_open # 需同日第二次采样
sort_rate = Δ / (基线 + analytics.growth_rate_prior) # 默认 prior=1000,只用于增速榜排序
``` ```
| 榜 | 口径 | 基数门槛 | | 榜 | 口径 | 基数门槛 |
|---|---|---| |---|---|---|
| 累计热度 | §8.1 | — | | 累计热度 | §8.1 | — |
| 近 7 天播放增量 | `d7(view)` | — | | 近 7 天播放增量 | `d7(view)` | — |
| 近 7 天增速 | `r7(view)` | 基线播放 `>= 1000` | | 近 7 天增速 | `r7(view)` | `view >= 200` |
| 互动率 | `(like+coin+fav)/view` | `view >= 100` | | 互动率 | `(like+coin+fav)/view` | `view >= 100` |
| 单项榜 | 7 项直接排序 | — | | 单项榜 | 7 项直接排序 | — |
| UP 榜 | 按 `up_mid` 聚合 Σ/AVG | — | | UP 榜 | 按 `up_mid` 聚合 Σ/AVG | — |
**冷启动**:近 1 日不要求跨日快照。当日发布相对 0,只进播放增量、不进播放增速;更早发布且当天入库的片子相对 `view_open`(第二次采样走晚间全量,不进小时快照)。近 7/30 日仍要跨日快照。缺失日不插值。 **冷启动**:不足 2 天数据的视频用"发布后 24h 增量",并标注"样本不足";缺失日不插值。
### 8.3 趋势 ### 8.3 趋势
@@ -800,7 +795,7 @@ secrets.example.json 凭据样例(auth 回写目标)
**进程结构** **进程结构**
``` ```
scheduler 线程:discovery(每 discover_minutes,避开 23:00–01:00) / snapshot_seed(每日 00:00;snapshot 占用或 no_source 则 10 分钟重试,CLI: snapshot-seed) / snapshot(每日 23:15,不因其它任务 skip) / fans(每日 23:15,不避开) / snapshot_recent(小时,避开 23:00–01:00) / candidate_cleanup(每日,避开 23:00–01:00) scheduler 线程:discovery(每 60min) / snapshot_seed(每日 00:00) / snapshot+fans(每日 23:15) / snapshot_recent(小时) / candidate_cleanup(每日)
web 线程:ThreadingHTTPServer :8080 web 线程:ThreadingHTTPServer :8080
telegram:getUpdates long polling(回调 + 私聊文本入库) telegram:getUpdates long polling(回调 + 私聊文本入库)
storage:SQLite WAL,单写线程 storage:SQLite WAL,单写线程
+4 -32
View File
@@ -34,7 +34,7 @@ DEFAULT_HEAT_WEIGHTS: dict[str, float] = {
} }
MIN_VIEW_FOR_INTERACTION = 100 MIN_VIEW_FOR_INTERACTION = 100
MIN_VIEW_FOR_GROWTH_RATE = 1000 MIN_VIEW_FOR_GROWTH_RATE = 200
MIN_EFFICIENCY_DURATION = 600 MIN_EFFICIENCY_DURATION = 600
DEFAULT_MATURITY_DAYS = 2 DEFAULT_MATURITY_DAYS = 2
DEFAULT_SHRINK_MINUTES = 120.0 DEFAULT_SHRINK_MINUTES = 120.0
@@ -313,20 +313,15 @@ def compute_growth(
current: int | float | None, current: int | float | None,
previous: int | float | None, previous: int | float | None,
gap_days: int | float | None, gap_days: int | float | None,
*,
allow_zero_gap: bool = False,
) -> Growth: ) -> Growth:
"""计算累计差值、日均差值和相对增速。 """计算累计差值、日均差值和相对增速。
缺失基线或间隔为负时返回不可用。默认间隔不足 1 天也不可用,不把 0 当成真实增长。 缺失基线或间隔不足 1 天时返回不可用,不把 0 当作真实增长。
``allow_zero_gap=True`` 时允许同一天的两次采样(或相对 0)计算增量,``per_day`` 为 None。
""" """
if current is None or previous is None or gap_days is None: if current is None or previous is None or gap_days is None:
return Growth(available=False) return Growth(available=False)
gap = int(gap_days) gap = int(gap_days)
if gap < 0: if gap <= 0:
return Growth(available=False)
if gap == 0 and not allow_zero_gap:
return Growth(available=False) return Growth(available=False)
current_value = max(0, int(current)) current_value = max(0, int(current))
previous_value = max(0, int(previous)) previous_value = max(0, int(previous))
@@ -334,35 +329,12 @@ def compute_growth(
return Growth( return Growth(
available=True, available=True,
delta=delta, delta=delta,
per_day=None if gap == 0 else delta / gap, per_day=delta / gap,
rate=delta / max(previous_value, 1), rate=delta / max(previous_value, 1),
gap_days=gap, gap_days=gap,
) )
def smoothed_growth_rate(
delta: int | float | None,
previous: int | float | None,
prior: int | float,
) -> float | None:
"""增速榜排序分:``Δ / (基线 + prior)``。
``rate`` 仍是真实相对增速。``prior`` 只削弱小基数在排序里的优势。
"""
if delta is None or previous is None or isinstance(delta, bool) or isinstance(previous, bool):
return None
if isinstance(prior, bool):
return None
try:
strength = float(prior)
base = max(0, int(previous)) + strength
except (TypeError, ValueError):
return None
if strength < 0 or base <= 0:
return None
return int(delta) / base
def normalize_cover(url: object) -> str | None: def normalize_cover(url: object) -> str | None:
"""把 B 站封面统一为 https;空值返回 None。""" """把 B 站封面统一为 https;空值返回 None。"""
if not isinstance(url, str): if not isinstance(url, str):
+94 -816
View File
File diff suppressed because it is too large. Load diff
-7
View File
@@ -52,7 +52,6 @@ DEFAULT_ANALYTICS: dict[str, Any] = {
"video_shrink_minutes": 60, "video_shrink_minutes": 60,
"shrink_minutes": None, "shrink_minutes": None,
"home_growth_days": "auto", "home_growth_days": "auto",
"growth_rate_prior": 1000,
} }
DEFAULT_CONFIG: dict[str, Any] = { DEFAULT_CONFIG: dict[str, Any] = {
@@ -211,7 +210,6 @@ class AnalyticsConfig:
video_shrink_minutes: int video_shrink_minutes: int
shrink_minutes: int | None shrink_minutes: int | None
home_growth_days: str | int home_growth_days: str | int
growth_rate_prior: int
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -594,11 +592,6 @@ def _build_config(data: Mapping[str, Any], path: Path, *, secrets_path: Path) ->
) )
), ),
home_growth_days=home_growth_days, home_growth_days=home_growth_days,
growth_rate_prior=_as_int(
analytics_raw.get("growth_rate_prior", 1000),
"analytics.growth_rate_prior",
minimum=0,
),
) )
return AppConfig( return AppConfig(
-12
View File
@@ -244,18 +244,6 @@ MIGRATIONS: tuple[Migration, ...] = (
"CREATE INDEX idx_visit_date ON visit(stat_date)", "CREATE INDEX idx_visit_date ON visit(stat_date)",
), ),
), ),
Migration(
version=8,
name="snapshot open view",
statements=(
# 同日第一次成功采样的播放量,供近 1 日「入库后增量」做基准。
# 已有成功行无法找回被覆盖的入库值,只能把当前 view 当作开盘。
'ALTER TABLE snapshot ADD COLUMN view_open INTEGER',
"ALTER TABLE snapshot ADD COLUMN open_captured_at INTEGER",
'UPDATE snapshot SET view_open = "view", open_captured_at = captured_at '
"WHERE ok = 1 AND view_open IS NULL",
),
),
) )
+8 -30
View File
@@ -690,17 +690,12 @@ class Repository:
src: str = "view", src: str = "view",
) -> None: ) -> None:
values = metrics or {} values = metrics or {}
ok_flag = 1 if ok else 0
view_value = values.get("view")
open_view = view_value if ok else None
open_at = int(captured_at) if ok else None
def run(conn: sqlite3.Connection) -> None: def run(conn: sqlite3.Connection) -> None:
conn.execute( conn.execute(
""" """
INSERT INTO snapshot(stat_date, aid, "view", "like", favorite, coin, INSERT INTO snapshot(stat_date, aid, "view", "like", favorite, coin,
share, reply, danmaku, captured_at, ok, err, src, share, reply, danmaku, captured_at, ok, err, src)
view_open, open_captured_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(stat_date, aid) DO UPDATE SET ON CONFLICT(stat_date, aid) DO UPDATE SET
"view" = excluded."view", "view" = excluded."view",
"like" = excluded."like", "like" = excluded."like",
@@ -712,22 +707,12 @@ class Repository:
captured_at = excluded.captured_at, captured_at = excluded.captured_at,
ok = excluded.ok, ok = excluded.ok,
err = excluded.err, err = excluded.err,
src = excluded.src, src = excluded.src
view_open = CASE
WHEN excluded.ok = 1 AND snapshot.view_open IS NULL
THEN excluded."view"
ELSE snapshot.view_open
END,
open_captured_at = CASE
WHEN excluded.ok = 1 AND snapshot.open_captured_at IS NULL
THEN excluded.captured_at
ELSE snapshot.open_captured_at
END
""", """,
( (
stat_date, stat_date,
int(aid), int(aid),
view_value, values.get("view"),
values.get("like"), values.get("like"),
values.get("favorite"), values.get("favorite"),
values.get("coin"), values.get("coin"),
@@ -735,11 +720,9 @@ class Repository:
values.get("reply"), values.get("reply"),
values.get("danmaku"), values.get("danmaku"),
int(captured_at), int(captured_at),
ok_flag, 1 if ok else 0,
err, err,
src, src,
open_view,
open_at,
), ),
) )
self.db.write(run) self.db.write(run)
@@ -758,23 +741,18 @@ class Repository:
""" """
INSERT INTO snapshot( INSERT INTO snapshot(
stat_date, aid, "view", "like", favorite, coin, stat_date, aid, "view", "like", favorite, coin,
share, reply, danmaku, captured_at, ok, err, src, share, reply, danmaku, captured_at, ok, err, src
view_open, open_captured_at
) )
SELECT SELECT
?, aid, "view", "like", favorite, coin, ?, aid, "view", "like", favorite, coin,
share, reply, danmaku, ?, 1, NULL, ?, share, reply, danmaku, ?, 1, NULL, ?
"view", ?
FROM snapshot FROM snapshot
WHERE stat_date = ? AND ok = 1 WHERE stat_date = ? AND ok = 1
AND aid NOT IN ( AND aid NOT IN (
SELECT aid FROM snapshot WHERE stat_date = ? SELECT aid FROM snapshot WHERE stat_date = ?
) )
""", """,
( (target_date, int(captured_at), src, source_date, target_date),
target_date, int(captured_at), src, int(captured_at),
source_date, target_date,
),
) )
return int(cur.rowcount or 0) return int(cur.rowcount or 0)
return self.db.write(run) return self.db.write(run)
+5 -49
View File
@@ -3,9 +3,6 @@
P0 只注册收藏夹同步任务;P1/P3 会在同一调度器上追加快照、粉丝和发现任务。 P0 只注册收藏夹同步任务;P1/P3 会在同一调度器上追加快照、粉丝和发现任务。
调度只负责“什么时候运行”,具体任务保持幂等,并在自己的服务层处理重试。 调度只负责“什么时候运行”,具体任务保持幂等,并在自己的服务层处理重试。
任务 ``fn`` 可返回 :class:`TaskControl` 以覆盖 ``ok`` / ``next_run``
(用于静默窗延后、失败补跑等);未返回时仍按 ``next_run_fn`` / ``interval`` 排期。
""" """
from __future__ import annotations from __future__ import annotations
@@ -15,15 +12,6 @@ from dataclasses import dataclass
from typing import Any, Callable, Sequence from typing import Any, Callable, Sequence
@dataclass
class TaskControl:
"""任务可选返回值:覆盖成功标记与下次运行时间。"""
ok: bool = True
next_run: float | None = None
error: str | None = None
@dataclass @dataclass
class ScheduledTask: class ScheduledTask:
name: str name: str
@@ -59,7 +47,6 @@ class Scheduler:
clock=time.time, clock=time.time,
sleeper=time.sleep, sleeper=time.sleep,
logger=None, logger=None,
due_reader: Callable[[str], float | None] | None = None,
) -> None: ) -> None:
if not tasks: if not tasks:
raise ValueError("至少需要一个调度任务") raise ValueError("至少需要一个调度任务")
@@ -70,8 +57,6 @@ class Scheduler:
self._clock = clock self._clock = clock
self._sleeper = sleeper self._sleeper = sleeper
self.logger = logger self.logger = logger
# 每轮从落盘 due_at 刷新 next_run(seed 挂当晚 seal 等同进程可见)
self._due_reader = due_reader
def _log(self, level: str, event: str, **fields) -> None: def _log(self, level: str, event: str, **fields) -> None:
if self.logger is not None: if self.logger is not None:
@@ -81,56 +66,27 @@ class Scheduler:
current = self._clock() if now is None else now current = self._clock() if now is None else now
results: list[TaskResult] = [] results: list[TaskResult] = []
for task in self.tasks: for task in self.tasks:
if self._due_reader is not None:
disk_due = self._due_reader(task.name)
if disk_due is not None:
task.next_run = float(disk_due)
if task.next_run > current: if task.next_run > current:
continue continue
started = current started = current
error: str | None = None error: str | None = None
control: TaskControl | None = None
try: try:
raw = task.fn() task.fn()
except Exception as exc: # noqa: BLE001 单任务失败不终止调度 except Exception as exc: # noqa: BLE001 单任务失败不终止调度
error = str(exc) error = str(exc)
self._log("error", "scheduler.task_error", task=task.name, error=error)
else: else:
if isinstance(raw, TaskControl): self._log("info", "scheduler.task_ok", task=task.name)
control = raw
finished = self._clock() finished = self._clock()
if control is not None and control.next_run is not None: if task.next_run_fn is not None:
task.next_run = float(control.next_run)
elif task.next_run_fn is not None:
task.next_run = task.next_run_fn(started) task.next_run = task.next_run_fn(started)
else: else:
task.next_run = started + task.interval task.next_run = started + task.interval
ok = error is None
if control is not None:
ok = bool(control.ok) and error is None
if control.error and error is None:
error = control.error
# Skip/fail ≠ success:先定 ok/error,再记 scheduler 日志。
if ok:
self._log("info", "scheduler.task_ok", task=task.name)
elif control is not None:
self._log(
"warning",
"scheduler.task_fail",
task=task.name,
error=error,
)
else:
self._log(
"error",
"scheduler.task_error",
task=task.name,
error=error,
)
result = TaskResult( result = TaskResult(
name=task.name, name=task.name,
started_at=started, started_at=started,
finished_at=finished, finished_at=finished,
ok=ok, ok=error is None,
error=error, error=error,
) )
results.append(result) results.append(result)
+17 -58
View File
@@ -51,7 +51,6 @@ class SnapshotSummary:
completed: bool = False completed: bool = False
already_completed: bool = False already_completed: bool = False
finished_at: int = 0 finished_at: int = 0
reason: str | None = None
errors: list[str] = field(default_factory=list) errors: list[str] = field(default_factory=list)
def as_dict(self) -> dict[str, Any]: def as_dict(self) -> dict[str, Any]:
@@ -74,7 +73,6 @@ class SnapshotSummary:
"backup_error": self.backup_error, "backup_error": self.backup_error,
"completed": self.completed, "completed": self.completed,
"already_completed": self.already_completed, "already_completed": self.already_completed,
"reason": self.reason,
"errors": self.errors, "errors": self.errors,
} }
@@ -142,20 +140,12 @@ class SnapshotService:
return f"fans:seeded:{stat_date}" return f"fans:seeded:{stat_date}"
def run_if_needed( def run_if_needed(
self, self, *, date: str | None = None, force: bool = False
*,
date: str | None = None,
force: bool = False,
keep_scraped: bool | None = None,
) -> SnapshotSummary: ) -> SnapshotSummary:
"""视频快照(含 UP 头像补齐);粉丝采集请用 ``run_fans_if_needed``。 """视频快照(含 UP 头像补齐);粉丝采集请用 ``run_fans_if_needed``。
当日若仅有午夜 seed 壳且未 ``force``,直接返回等待晚间落盘, 当日若仅有午夜 seed 壳且未 ``force``,直接返回等待晚间落盘,
避免把「全员 skip」误标成 completed。 避免把「全员 skip」误标成 completed。
``force`` + ``keep_scraped``:
- 晚间 seal / 断点续跑:``keep_scraped=True``(默认)保留已采 ok=1;
- CLI ``--force``:显式 ``keep_scraped=False`` 全量重采。
""" """
now = self._now() now = self._now()
stat_date = date or shanghai_date(now) stat_date = date or shanghai_date(now)
@@ -163,10 +153,7 @@ class SnapshotService:
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}") raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
seeded = self.repo.kv_get(self.seeded_key(stat_date)) == "1" seeded = self.repo.kv_get(self.seeded_key(stat_date)) == "1"
if force: if force:
ks = True if keep_scraped is None else bool(keep_scraped) return self.run_videos(date=stat_date, force=True)
return self.run_videos(
date=stat_date, force=True, keep_scraped=ks,
)
completed = self.repo.kv_get(self.completed_key(stat_date)) == "1" completed = self.repo.kv_get(self.completed_key(stat_date)) == "1"
if seeded and not completed: if seeded and not completed:
@@ -180,7 +167,6 @@ class SnapshotService:
finished_at=now, finished_at=now,
completed=False, completed=False,
already_completed=False, already_completed=False,
reason="seeded_pending_seal",
) )
pending = self.repo.pending_snapshot_aids(stat_date) pending = self.repo.pending_snapshot_aids(stat_date)
if completed and not pending: if completed and not pending:
@@ -217,19 +203,8 @@ class SnapshotService:
summary = self.run_videos(date=date, force=force) summary = self.run_videos(date=date, force=force)
return self.run_fans(date=summary.stat_date, force=force, summary=summary) return self.run_fans(date=summary.stat_date, force=force, summary=summary)
def run_videos( def run_videos(self, *, date: str | None = None, force: bool = False) -> SnapshotSummary:
self, """采集视频七项指标并在全部成功后标记完成日 / 备份。"""
*,
date: str | None = None,
force: bool = False,
keep_scraped: bool = False,
) -> SnapshotSummary:
"""采集视频七项指标并在全部成功后标记完成日 / 备份。
``force``:忽略完成标记,重跑。
``keep_scraped``:force 时仍跳过非 seed 的 ok=1(seal 断点续跑);
CLI ``--force`` 不传此标志,会重采全部成功行。
"""
cfg = self.config_store.get() cfg = self.config_store.get()
client = self._client() client = self._client()
now = self._now() now = self._now()
@@ -250,19 +225,11 @@ class SnapshotService:
summary.total = len(videos) summary.total = len(videos)
for index, aid in enumerate(videos, 1): for index, aid in enumerate(videos, 1):
existing = self.repo.get_snapshot(stat_date, aid) existing = self.repo.get_snapshot(stat_date, aid)
if existing is not None and int(existing.get("ok") or 0) == 1: if existing is not None and int(existing.get("ok") or 0) == 1 and not force:
src = str(existing.get("src") or "") # 断点续跑:同日已成功则跳过。小时任务刷新过的行也是 ok=1,
if keep_scraped and src == "seed": # 自然覆盖「1 小时内 captured_at 已更新则不再次更新」的要求。
# seal 续跑:必须打穿克隆壳
pass
elif keep_scraped and src != "seed":
summary.skipped += 1 summary.skipped += 1
continue continue
elif not force:
# 普通续跑:同日已成功则跳过(含小时任务刚刷新的)
summary.skipped += 1
continue
# force 且非 keep_scraped:CLI 全量重采,不 skip
stat, error, unavailable = self._fetch_view(client, aid) stat, error, unavailable = self._fetch_view(client, aid)
if unavailable: if unavailable:
@@ -343,8 +310,7 @@ class SnapshotService:
"""刷新近 N 天新视频的当日快照;不碰完成日标记与日备份。 """刷新近 N 天新视频的当日快照;不碰完成日标记与日备份。
「近 N 天」按 pubdate 墙钟窗口:``now - recent_video_days * 86400``, 「近 N 天」按 pubdate 墙钟窗口:``now - recent_video_days * 86400``,
不是上海日历日差。当天入库但发布更早的旧片不进小时任务。 不是上海日历日差。允许覆盖同日已有 ``ok=1`` 行以便小时级刷新。
允许覆盖同日已有 ``ok=1`` 行以便小时级刷新。
""" """
cfg = self.config_store.get() cfg = self.config_store.get()
client = self._client() client = self._client()
@@ -456,7 +422,6 @@ class SnapshotService:
completed=False, completed=False,
already_completed=False, already_completed=False,
fans_skipped=len(cfg.track_fans), fans_skipped=len(cfg.track_fans),
reason="seeded_pending_seal",
) )
return self.run_fans(date=stat_date, force=True) return self.run_fans(date=stat_date, force=True)
if not self._pending_fan_mids(stat_date, cfg.track_fans): if not self._pending_fan_mids(stat_date, cfg.track_fans):
@@ -570,8 +535,15 @@ class SnapshotService:
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}") raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
summary = SnapshotSummary(stat_date=stat_date, started_at=now) summary = SnapshotSummary(stat_date=stat_date, started_at=now)
# 目标日已完成:先于 no_source。全站快照当天完成后 source = self.repo.kv_get("snapshot:last_completed")
# last_completed == stat_date,不可误判为无源再 10 分钟重试。 if not source or source >= stat_date:
self._log(
"info", "snapshot.seed_skip",
stat_date=stat_date, reason="no_source", source=source,
)
summary.finished_at = self._now()
summary.completed = True
return summary
if self.repo.kv_get(self.completed_key(stat_date)) == "1": if self.repo.kv_get(self.completed_key(stat_date)) == "1":
self._log( self._log(
"info", "snapshot.seed_skip", "info", "snapshot.seed_skip",
@@ -580,17 +552,6 @@ class SnapshotService:
summary.finished_at = self._now() summary.finished_at = self._now()
summary.completed = True summary.completed = True
summary.already_completed = True summary.already_completed = True
summary.reason = "already_completed"
return summary
source = self.repo.kv_get("snapshot:last_completed")
if not source or source >= stat_date:
self._log(
"info", "snapshot.seed_skip",
stat_date=stat_date, reason="no_source", source=source,
)
summary.finished_at = self._now()
summary.completed = False
summary.reason = "no_source"
return summary return summary
if self.repo.kv_get(self.seeded_key(stat_date)) == "1": if self.repo.kv_get(self.seeded_key(stat_date)) == "1":
self._log( self._log(
@@ -600,7 +561,6 @@ class SnapshotService:
summary.finished_at = self._now() summary.finished_at = self._now()
summary.completed = True summary.completed = True
summary.already_completed = True summary.already_completed = True
summary.reason = "already_seeded"
return summary return summary
copied = self.repo.copy_ok_snapshots( copied = self.repo.copy_ok_snapshots(
@@ -622,7 +582,6 @@ class SnapshotService:
self.repo.bump_cache_version("fans") self.repo.bump_cache_version("fans")
summary.finished_at = self._now() summary.finished_at = self._now()
summary.completed = True # seed 任务本身成功;不等于快照落盘完成 summary.completed = True # seed 任务本身成功;不等于快照落盘完成
summary.reason = "seeded"
self._log( self._log(
"info", "snapshot.seed_done", "info", "snapshot.seed_done",
stat_date=stat_date, source=source, stat_date=stat_date, source=source,
-438
View File
@@ -1,438 +0,0 @@
# -*- coding: utf-8 -*-
"""调度任务状态持久化(跨进程重启 / 容器重建)。
每条任务一条 kv 记录 ``task:state:{name}``,字段见 :class:`TaskState`。
调度器只认落盘的 ``due_at`` / ``status`` / ``stat_date`` / ``progress``,
不再靠启动时把 ``next_run=0`` 猜「要不要立刻跑」。
"""
from __future__ import annotations
import json
import time
from dataclasses import asdict, dataclass
from datetime import timedelta
from typing import Any, Mapping
from .repository import Repository
from .timeutil import parse_hhmm, shanghai_date, shanghai_datetime
STATUS_PENDING = "pending"
STATUS_RUNNING = "running"
STATUS_DONE = "done"
STATUS_FAILED = "failed"
STATUS_DEFERRED = "deferred"
_ACTIVE = frozenset({STATUS_PENDING, STATUS_RUNNING, STATUS_FAILED, STATUS_DEFERRED})
@dataclass
class TaskState:
name: str
status: str
due_at: float
stat_date: str | None = None
progress: dict[str, Any] | None = None
updated_at: float = 0.0
error: str | None = None
def as_dict(self) -> dict[str, Any]:
payload = asdict(self)
# JSON 友好:去掉 None progress 也可保留
return payload
def state_key(name: str) -> str:
return f"task:state:{name}"
def load_state(repo: Repository, name: str) -> TaskState | None:
raw = repo.kv_get(state_key(name))
if not raw:
return None
try:
data = json.loads(raw)
except (TypeError, ValueError, json.JSONDecodeError):
return None
if not isinstance(data, dict) or not data.get("name"):
return None
try:
due_at = float(data.get("due_at") or 0)
updated_at = float(data.get("updated_at") or 0)
except (TypeError, ValueError):
return None
progress = data.get("progress")
if progress is not None and not isinstance(progress, dict):
progress = None
return TaskState(
name=str(data.get("name") or name),
status=str(data.get("status") or STATUS_PENDING),
due_at=due_at,
stat_date=(str(data["stat_date"]) if data.get("stat_date") else None),
progress=progress,
updated_at=updated_at,
error=(str(data["error"]) if data.get("error") else None),
)
def save_state(repo: Repository, state: TaskState) -> None:
repo.kv_set(state_key(state.name), json.dumps(state.as_dict(), ensure_ascii=False))
def _now(now: float | None) -> float:
return time.time() if now is None else float(now)
def recover_stale_running(
repo: Repository,
name: str,
*,
now: float | None = None,
) -> TaskState | None:
"""进程已死但仍为 ``running``:改为 ``pending``,保留 progress / stat_date。
``due_at`` 取 ``min(原 due_at, now)``,使过期/中断任务可立即续跑,
未到期的(极少见)仍尊重原 due_at。
"""
current = _now(now)
state = load_state(repo, name)
if state is None or state.status != STATUS_RUNNING:
return state
state.status = STATUS_PENDING
state.due_at = min(float(state.due_at or current), current)
state.updated_at = current
save_state(repo, state)
return state
def mark_running(
repo: Repository,
name: str,
*,
stat_date: str | None,
now: float | None = None,
progress: Mapping[str, Any] | None = None,
due_at: float | None = None,
) -> TaskState:
current = _now(now)
prev = load_state(repo, name)
state = TaskState(
name=name,
status=STATUS_RUNNING,
due_at=float(due_at if due_at is not None else (prev.due_at if prev else current)),
stat_date=stat_date if stat_date is not None else (prev.stat_date if prev else None),
progress=dict(progress) if progress is not None else (prev.progress if prev else None),
updated_at=current,
error=None,
)
save_state(repo, state)
return state
def mark_done(
repo: Repository,
name: str,
*,
due_at: float,
now: float | None = None,
stat_date: str | None = None,
progress: Mapping[str, Any] | None = None,
) -> TaskState:
"""成功:记 done,并写入下一次 ``due_at``(下一条墙钟/间隔)。"""
current = _now(now)
state = TaskState(
name=name,
status=STATUS_DONE,
due_at=float(due_at),
stat_date=stat_date,
progress=dict(progress) if progress is not None else None,
updated_at=current,
error=None,
)
save_state(repo, state)
return state
def mark_failed(
repo: Repository,
name: str,
*,
due_at: float,
now: float | None = None,
stat_date: str | None = None,
progress: Mapping[str, Any] | None = None,
error: str | None = None,
) -> TaskState:
current = _now(now)
prev = load_state(repo, name)
state = TaskState(
name=name,
status=STATUS_FAILED,
due_at=float(due_at),
stat_date=stat_date if stat_date is not None else (prev.stat_date if prev else None),
progress=(
dict(progress) if progress is not None
else (prev.progress if prev else None)
),
updated_at=current,
error=error,
)
save_state(repo, state)
return state
def mark_deferred(
repo: Repository,
name: str,
*,
due_at: float,
now: float | None = None,
stat_date: str | None = None,
progress: Mapping[str, Any] | None = None,
error: str | None = None,
) -> TaskState:
"""主动延后(静默窗 / seed 避让等):保留进度,写未来 due_at。"""
current = _now(now)
prev = load_state(repo, name)
state = TaskState(
name=name,
status=STATUS_DEFERRED,
due_at=float(due_at),
stat_date=stat_date if stat_date is not None else (prev.stat_date if prev else None),
progress=(
dict(progress) if progress is not None
else (prev.progress if prev else None)
),
updated_at=current,
error=error,
)
save_state(repo, state)
return state
def ensure_pending(
repo: Repository,
name: str,
*,
due_at: float,
stat_date: str | None,
now: float | None = None,
overwrite_done: bool = True,
) -> TaskState:
"""保证存在一条 pending(如 seed 成功后挂上当晚 seal)。
同日未完成:取更早的 due_at。
跨日:不得把新一天的 due 粘到昨天的 ``stat_date`` 上——
若已有更早日期的未完成记录,保留它(跨天续跑优先);
若已有更新日期的未完成,则重置为调用方的 due/stat_date。
``overwrite_done`` 时覆盖已完成记录。
"""
current = _now(now)
prev = load_state(repo, name)
if prev is not None and prev.status in _ACTIVE and prev.status != STATUS_RUNNING:
if (
stat_date is not None
and prev.stat_date
and prev.stat_date != stat_date
):
# 未完成的更早归属日优先保留,避免 seed 挂今晚时冲掉昨天续跑。
if prev.stat_date < stat_date:
return prev
# prev 归属日更新:按调用方重置,禁止跨日字段拼接。
state = TaskState(
name=name,
status=STATUS_PENDING,
due_at=float(due_at),
stat_date=stat_date,
progress=None,
updated_at=current,
error=None,
)
save_state(repo, state)
return state
# 同日(或缺一边 stat_date):取更早 due_at,补上缺失的 stat_date
due = min(float(prev.due_at or due_at), float(due_at))
sd = prev.stat_date or stat_date
if due == prev.due_at and sd == prev.stat_date and prev.status == STATUS_PENDING:
return prev
prev.due_at = due
prev.stat_date = sd
prev.status = STATUS_PENDING
prev.updated_at = current
save_state(repo, prev)
return prev
if prev is not None and prev.status == STATUS_RUNNING:
return prev
if prev is not None and prev.status == STATUS_DONE and not overwrite_done:
return prev
state = TaskState(
name=name,
status=STATUS_PENDING,
due_at=float(due_at),
stat_date=stat_date,
progress=None,
updated_at=current,
error=None,
)
save_state(repo, state)
return state
def bootstrap_from_last(
repo: Repository,
name: str,
*,
wall_due_at: float,
now: float | None = None,
retry_seconds: float = 300.0,
) -> TaskState | None:
"""升级/冷启动:无 ``task:state`` 时从 ``task:last`` 回填。
- 成功 → ``done`` + ``due_at=wall_due_at``
- 失败/跳过 → ``failed``/``deferred``,``due_at`` 为补跑点(已过则立刻)
"""
current = _now(now)
existing = load_state(repo, name)
if existing is not None:
return existing
raw = repo.kv_get(f"task:last:{name}")
if not raw:
return None
try:
data = json.loads(raw)
except (TypeError, ValueError, json.JSONDecodeError):
return None
if not isinstance(data, dict):
return None
ok = bool(data.get("ok"))
skipped = bool(data.get("skipped"))
try:
finished = float(data.get("finished_at") or data.get("started_at") or current)
except (TypeError, ValueError):
finished = current
summary = data.get("summary")
stat_date = None
if isinstance(summary, dict) and summary.get("stat_date"):
stat_date = str(summary["stat_date"])
error = data.get("error")
error_s = str(error) if error else None
if ok and not skipped:
state = TaskState(
name=name,
status=STATUS_DONE,
due_at=float(wall_due_at),
stat_date=stat_date,
updated_at=current,
error=None,
)
save_state(repo, state)
return state
due = finished + float(retry_seconds)
if due > current:
pass
else:
due = current
state = TaskState(
name=name,
status=STATUS_DEFERRED if skipped else STATUS_FAILED,
due_at=float(due),
stat_date=stat_date,
updated_at=current,
error=error_s,
)
save_state(repo, state)
return state
def bootstrap_seed_markers(
repo: Repository,
name: str,
*,
slot_hhmm: str,
wall_due_at: float,
now: float | None = None,
) -> TaskState | None:
"""无 state 时:若存在未 seal 的 seed 壳,挂 pending(跨天优先昨天)。
已过当日 ``slot_hhmm``(或归属日早于今天)→ ``due_at=now`` 尽快补跑。
"""
del wall_due_at # 未完成 seed 不走纯墙钟
current = _now(now)
existing = load_state(repo, name)
if existing is not None:
return existing
if name not in {"snapshot", "fans"}:
return None
today = shanghai_date(current)
yday = (shanghai_datetime(current) - timedelta(days=1)).strftime("%Y-%m-%d")
hour, minute = parse_hhmm(slot_hhmm)
today_slot = shanghai_datetime(current).replace(
hour=hour, minute=minute, second=0, microsecond=0,
).timestamp()
def _seeded_unfinished(day: str) -> bool:
if name == "snapshot":
if repo.kv_get(f"snapshot:seeded:{day}") != "1":
return False
return repo.kv_get(f"snapshot:completed:{day}") != "1"
# fans:seed 键在成功 seal 后删除,有键即未完成
return repo.kv_get(f"fans:seeded:{day}") == "1"
for day in (yday, today):
if not _seeded_unfinished(day):
continue
if day < today or current >= today_slot:
due = current
else:
due = float(today_slot)
return ensure_pending(
repo, name, due_at=due, stat_date=day, now=current,
)
return None
def startup_next_run(
repo: Repository,
name: str,
*,
wall_due_at: float,
now: float | None = None,
slot_hhmm: str | None = None,
retry_seconds: float = 300.0,
) -> float:
"""常驻启动时的 ``ScheduledTask.next_run``:只认落盘状态。
- 无状态时先 ``bootstrap_from_last`` / seed 标记回填
- 恢复残留 ``running`` → ``pending``
- ``due_at`` 是唯一时间门闩(失败延时不会被提前)
"""
current = _now(now)
if load_state(repo, name) is None:
bootstrap_from_last(
repo, name, wall_due_at=wall_due_at, now=current,
retry_seconds=retry_seconds,
)
if slot_hhmm:
bootstrap_seed_markers(
repo, name, slot_hhmm=slot_hhmm,
wall_due_at=wall_due_at, now=current,
)
state = recover_stale_running(repo, name, now=current)
if state is None:
return float(wall_due_at)
due = float(state.due_at or 0)
if due <= 0:
return float(wall_due_at)
# 无论 status:due_at 是唯一时间门闩
if due > current:
return due
# 已到期:未完成要续跑;done 也表示下一轮已到期
return due
def resume_stat_date(repo: Repository, name: str, *, fallback: str) -> str:
"""续跑时优先用状态里的归属日,避免跨日重启跑错天。"""
state = load_state(repo, name)
if state is not None and state.stat_date and state.status in _ACTIVE:
return state.stat_date
return fallback
-36
View File
@@ -57,39 +57,3 @@ def day_bounds(date: str) -> tuple[float, float]:
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {date!r}") raise ValueError(f"日期格式必须是 YYYY-MM-DD: {date!r}")
start = datetime.strptime(date, "%Y-%m-%d").replace(tzinfo=SHANGHAI_TZ) start = datetime.strptime(date, "%Y-%m-%d").replace(tzinfo=SHANGHAI_TZ)
return start.timestamp(), (start + timedelta(days=1)).timestamp() return start.timestamp(), (start + timedelta(days=1)).timestamp()
# 全站快照静默窗:上海时区 [23:00, 01:00)。此间 discovery / recent / cleanup 不执行。
SNAPSHOT_QUIET_START_HOUR = 23
SNAPSHOT_QUIET_END_HOUR = 1
def in_snapshot_quiet_window(ts: float | None = None) -> bool:
"""是否处于全站快照静默窗 [23:00, 01:00)(上海时区,跨日)。"""
dt = shanghai_datetime(ts)
return dt.hour >= SNAPSHOT_QUIET_START_HOUR or dt.hour < SNAPSHOT_QUIET_END_HOUR
def next_after_snapshot_quiet_window(ts: float) -> float:
"""若 ``ts`` 落在静默窗内,推到窗结束后的 01:00;否则原样返回。"""
if not in_snapshot_quiet_window(ts):
return ts
dt = shanghai_datetime(ts)
if dt.hour >= SNAPSHOT_QUIET_START_HOUR:
target = (dt + timedelta(days=1)).replace(
hour=SNAPSHOT_QUIET_END_HOUR, minute=0, second=0, microsecond=0,
)
else:
target = dt.replace(
hour=SNAPSHOT_QUIET_END_HOUR, minute=0, second=0, microsecond=0,
)
if target.timestamp() <= ts:
target += timedelta(days=1)
return target.timestamp()
def interval_next_run(now: float, interval: float) -> float:
"""``now + interval``,若落入静默窗则推到 01:00 之后。"""
if interval <= 0:
raise ValueError("interval 必须大于 0")
return next_after_snapshot_quiet_window(now + interval)
+20 -204
View File
@@ -20,7 +20,6 @@ from ..analytics import (
METRICS, METRICS,
MIN_VIEW_FOR_GROWTH_RATE, MIN_VIEW_FOR_GROWTH_RATE,
compute_growth, compute_growth,
smoothed_growth_rate,
content_hours, content_hours,
eb_rate, eb_rate,
effective_shrink_minutes, effective_shrink_minutes,
@@ -34,14 +33,7 @@ from ..analytics import (
) )
from ..config import ConfigStore from ..config import ConfigStore
from ..database import Database from ..database import Database
from ..timeutil import ( from ..timeutil import day_bounds, next_daily_run, shanghai_date, shanghai_datetime
day_bounds,
interval_next_run,
next_after_snapshot_quiet_window,
next_daily_run,
shanghai_date,
shanghai_datetime,
)
from .cache import DASHBOARD_CACHE, TTL_FANS, TTL_VIDEOS, build_cache_key from .cache import DASHBOARD_CACHE, TTL_FANS, TTL_VIDEOS, build_cache_key
TRACKED_STATUSES = ("active", "unfaved") TRACKED_STATUSES = ("active", "unfaved")
@@ -66,19 +58,6 @@ def _chunks(values: Sequence[int], size: int = 400) -> Iterable[Sequence[int]]:
yield values[index:index + size] yield values[index:index + size]
def _growth_baseline_view(item: Mapping[str, Any]) -> int | None:
"""增速分母:当前播放减去区间增量。zero 基线的增量等于当前播放,结果为 0。"""
growth = item.get("growth") or {}
current = (item.get("metrics") or {}).get("view")
delta = growth.get("delta")
if current is None or delta is None:
return None
try:
return int(current) - int(delta)
except (TypeError, ValueError):
return None
def _date_minus(day: str, days: int) -> str: def _date_minus(day: str, days: int) -> str:
return (date.fromisoformat(day) - timedelta(days=int(days))).isoformat() return (date.fromisoformat(day) - timedelta(days=int(days))).isoformat()
@@ -406,18 +385,6 @@ class DashboardQueries:
percentiles = score_percentiles(scores) percentiles = score_percentiles(scores)
return videos, snapshots, scores, percentiles return videos, snapshots, scores, percentiles
@staticmethod
def _publish_date(pubdate: object) -> str | None:
if pubdate is None or isinstance(pubdate, bool):
return None
try:
ts = int(pubdate)
except (TypeError, ValueError):
return None
if ts <= 0:
return None
return shanghai_date(ts)
@staticmethod @staticmethod
def _growth_for( def _growth_for(
snapshot: Mapping[str, Any] | None, snapshot: Mapping[str, Any] | None,
@@ -439,70 +406,6 @@ class DashboardQueries:
gap = None gap = None
return compute_growth(current, before, gap) return compute_growth(current, before, gap)
@classmethod
def _same_day_growth(
cls,
snapshot: Mapping[str, Any] | None,
video: Mapping[str, Any],
metric: str = "view",
) -> dict[str, Any] | None:
"""近 1 日、没有更早快照时:当日发布相对 0;否则用 view_open。"""
if snapshot is None:
return None
current = snapshot.get(metric)
if current is None:
return None
stat_date = str(snapshot.get("stat_date") or "")
published_on = cls._publish_date(video.get("pubdate"))
if published_on is not None and published_on == stat_date:
growth = compute_growth(current, 0, 0, allow_zero_gap=True)
payload = growth.as_dict()
payload["baseline"] = "zero"
# previous=0 时 rate = view/1,会按原始播放碾压真日环比。
payload["rate"] = None
return payload
try:
open_at = snapshot.get("open_captured_at")
captured_at = snapshot.get("captured_at")
view_open = snapshot.get("view_open")
open_ready = (
open_at is not None
and captured_at is not None
and int(open_at) < int(captured_at)
and view_open is not None
)
except (TypeError, ValueError):
open_ready = False
if not open_ready:
return None
growth = compute_growth(current, view_open, 0, allow_zero_gap=True)
payload = growth.as_dict()
payload["baseline"] = "open"
return payload
@classmethod
def _growth_result(
cls,
video: Mapping[str, Any],
snapshot: Mapping[str, Any] | None,
previous: Mapping[str, Any] | None,
*,
days_back: int,
metric: str = "view",
) -> dict[str, Any]:
if previous is not None:
growth = cls._growth_for(snapshot, previous, metric=metric)
payload = growth.as_dict()
payload["baseline"] = "snapshot" if growth.available else None
return payload
if days_back == 1:
fallback = cls._same_day_growth(snapshot, video, metric=metric)
if fallback is not None:
return fallback
payload = compute_growth(None, None, None).as_dict()
payload["baseline"] = None
return payload
def _item( def _item(
self, self,
video: Mapping[str, Any], video: Mapping[str, Any],
@@ -512,7 +415,6 @@ class DashboardQueries:
percentile: float | None = None, percentile: float | None = None,
previous: Mapping[str, Any] | None = None, previous: Mapping[str, Any] | None = None,
include_growth: bool = False, include_growth: bool = False,
growth_days: int | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
metrics = _metric_map(snapshot) metrics = _metric_map(snapshot)
duration = video.get("duration") duration = video.get("duration")
@@ -553,9 +455,7 @@ class DashboardQueries:
), ),
} }
if include_growth: if include_growth:
item["growth"] = self._growth_result( item["growth"] = self._growth_for(snapshot, previous).as_dict()
video, snapshot, previous, days_back=int(growth_days or 1),
)
return item return item
# ------------------------------------------------------------ overview # ------------------------------------------------------------ overview
@@ -574,7 +474,7 @@ class DashboardQueries:
videos = self._videos(conn) videos = self._videos(conn)
aids = [int(video["aid"]) for video in videos] aids = [int(video["aid"]) for video in videos]
snapshots = self._snapshot_map(conn, aids) snapshots = self._snapshot_map(conn, aids)
# 播放增量:近 1 日。有更早日用相邻成功日;当日发布相对 0;否则用 view_open。 # 播放增量:按片相对当前点的相邻成功日,避免 seed/recent 拉到 T-2。
previous = self._snapshot_previous_relative(conn, snapshots, days_back=1) previous = self._snapshot_previous_relative(conn, snapshots, days_back=1)
totals = {metric: 0 for metric in METRICS} totals = {metric: 0 for metric in METRICS}
for video in videos: for video in videos:
@@ -589,13 +489,12 @@ class DashboardQueries:
for video in videos: for video in videos:
aid = int(video["aid"]) aid = int(video["aid"])
snapshot = snapshots.get(aid) snapshot = snapshots.get(aid)
if snapshot is None: prior = previous.get(aid)
if snapshot is None or prior is None:
continue continue
growth = self._growth_result( growth = self._growth_for(snapshot, prior)
video, snapshot, previous.get(aid), days_back=1, if growth.available and growth.delta is not None:
) view_delta_sum += growth.delta
if growth.get("available") and growth.get("delta") is not None:
view_delta_sum += int(growth["delta"])
available_deltas += 1 available_deltas += 1
if available_deltas: if available_deltas:
view_delta = view_delta_sum view_delta = view_delta_sum
@@ -982,7 +881,6 @@ class DashboardQueries:
percentile=percentiles.get(int(video["aid"])), percentile=percentiles.get(int(video["aid"])),
previous=previous.get(int(video["aid"])), previous=previous.get(int(video["aid"])),
include_growth=sort in {"growth", "growth_rate"}, include_growth=sort in {"growth", "growth_rate"},
growth_days=growth_days,
) )
for video in videos for video in videos
] ]
@@ -1054,13 +952,9 @@ class DashboardQueries:
if sort == "growth_rate": if sort == "growth_rate":
items = [ items = [
item for item in items item for item in items
if item.get("growth", {}).get("baseline") != "zero" if (item["metrics"] or {}).get("view", 0) >= MIN_VIEW_FOR_GROWTH_RATE
and item.get("growth", {}).get("rate") is not None
and (_growth_baseline_view(item) or 0) >= MIN_VIEW_FOR_GROWTH_RATE
] ]
rate_prior = max(0, int(analytics.growth_rate_prior))
def sort_key(item: Mapping[str, Any]): def sort_key(item: Mapping[str, Any]):
if sort == "added": if sort == "added":
return (int(item["first_seen"]), int(item["pubdate"])) return (int(item["first_seen"]), int(item["pubdate"]))
@@ -1077,18 +971,9 @@ class DashboardQueries:
) )
value = item.get(field) value = item.get(field)
return value if value is not None else float("-inf") return value if value is not None else float("-inf")
if sort == "growth": if sort in {"growth", "growth_rate"}:
value = item.get("growth", {}).get("delta") field = "delta" if sort == "growth" else "rate"
return value if value is not None else float("-inf") value = item.get("growth", {}).get(field)
if sort == "growth_rate":
growth = item.get("growth") or {}
delta = growth.get("delta")
current = (item.get("metrics") or {}).get("view")
if delta is None or current is None:
return float("-inf")
value = smoothed_growth_rate(
delta, int(current) - int(delta), rate_prior,
)
return value if value is not None else float("-inf") return value if value is not None else float("-inf")
value = (item["metrics"] or {}).get(sort) value = (item["metrics"] or {}).get(sort)
return value if value is not None else -1 return value if value is not None else -1
@@ -1105,17 +990,6 @@ class DashboardQueries:
reason = None reason = None
if sort in {"growth", "growth_rate"} and not growth_available: if sort in {"growth", "growth_rate"} and not growth_available:
available = False available = False
if growth_days == 1 and sort == "growth_rate":
reason = (
"近 1 日增速需要基线播放至少 "
f"{MIN_VIEW_FOR_GROWTH_RATE};当日发布不计入增速"
)
elif growth_days == 1:
reason = (
"近 1 日增长需要前一日快照;当日发布有一次成功采样即可;"
"更早发布的新入库需要当天至少两次成功采样"
)
else:
reason = ( reason = (
f"近 {growth_days} 日增长需要至少 {growth_days + 1} 天快照" f"近 {growth_days} 日增长需要至少 {growth_days + 1} 天快照"
) )
@@ -1149,7 +1023,6 @@ class DashboardQueries:
"available": available, "available": available,
"reason": reason, "reason": reason,
"duration_filter": duration_filter, "duration_filter": duration_filter,
"growth_rate_prior": rate_prior if sort == "growth_rate" else None,
} }
return self.db.read(run) return self.db.read(run)
@@ -1171,19 +1044,16 @@ class DashboardQueries:
video = dict(row) video = dict(row)
snapshots = self._snapshot_map(conn, [aid]) snapshots = self._snapshot_map(conn, [aid])
snapshot = snapshots.get(aid) snapshot = snapshots.get(aid)
previous = self._snapshot_map( previous = self._snapshot_map(conn, [aid], before=snapshot["stat_date"] if snapshot else None).get(aid)
conn, [aid], before=snapshot["stat_date"] if snapshot else None,
).get(aid)
_, _, heat, percentiles = self._global_heat(conn) _, _, heat, percentiles = self._global_heat(conn)
growth = self._growth_for(snapshot, previous)
item = self._item( item = self._item(
video, video,
snapshot, snapshot,
heat=heat.get(aid), heat=heat.get(aid),
percentile=percentiles.get(aid), percentile=percentiles.get(aid),
) )
item["growth"] = self._growth_result( item["growth"] = growth.as_dict()
video, snapshot, previous, days_back=1,
)
item["snapshot_count"] = int(conn.execute( item["snapshot_count"] = int(conn.execute(
"SELECT COUNT(*) FROM snapshot WHERE aid = ?", (aid,) "SELECT COUNT(*) FROM snapshot WHERE aid = ?", (aid,)
).fetchone()[0]) ).fetchone()[0])
@@ -1796,7 +1666,6 @@ class DashboardQueries:
"video_shrink_minutes": cfg.analytics.video_shrink_minutes, "video_shrink_minutes": cfg.analytics.video_shrink_minutes,
"shrink_minutes": cfg.analytics.shrink_minutes, "shrink_minutes": cfg.analytics.shrink_minutes,
"home_growth_days": cfg.analytics.home_growth_days, "home_growth_days": cfg.analytics.home_growth_days,
"growth_rate_prior": cfg.analytics.growth_rate_prior,
}, },
"heat_weights": { "heat_weights": {
metric: float(value) metric: float(value)
@@ -1978,27 +1847,9 @@ class DashboardQueries:
"fans": int(next_daily_run(now, cfg.schedule.snapshot_at)), "fans": int(next_daily_run(now, cfg.schedule.snapshot_at)),
"snapshot": int(next_daily_run(now, cfg.schedule.snapshot_at)), "snapshot": int(next_daily_run(now, cfg.schedule.snapshot_at)),
"snapshot_seed": int(next_daily_run(now, cfg.schedule.seed_at)), "snapshot_seed": int(next_daily_run(now, cfg.schedule.seed_at)),
"discover": int(interval_next_run(now, float(discover_interval))), "discover": int(now + discover_interval),
"snapshot_recent": int(interval_next_run(now, float(recent_interval))), "snapshot_recent": int(now + recent_interval),
} }
retry_seconds = 300
seed_retry_seconds = 600
# 优先 task:state.due_at(跨重启唯一时间门闩);无则回退 task:last 推导。
state_rows = conn.execute(
"SELECT k, v FROM kv WHERE k LIKE 'task:state:%'"
).fetchall()
for row in state_rows:
key = str(row["k"]).removeprefix("task:state:")
ui_name = "discover" if key == "discovery" else key
if ui_name not in next_runs:
continue
try:
state = json.loads(str(row["v"]))
due = float(state.get("due_at") or 0)
except (TypeError, ValueError, json.JSONDecodeError):
continue
if due > 0:
next_runs[ui_name] = int(due)
for name, key, at in ( for name, key, at in (
("fans", "fans", cfg.schedule.snapshot_at), ("fans", "fans", cfg.schedule.snapshot_at),
("snapshot", "snapshot", cfg.schedule.snapshot_at), ("snapshot", "snapshot", cfg.schedule.snapshot_at),
@@ -2006,58 +1857,23 @@ class DashboardQueries:
("discover", "discovery", None), ("discover", "discovery", None),
("snapshot_recent", "snapshot_recent", None), ("snapshot_recent", "snapshot_recent", None),
): ):
# 已有落盘 due_at 则不再用 last 覆盖
state_raw = self._kv(conn, f"task:state:{key}")
if state_raw:
try:
if float(json.loads(state_raw).get("due_at") or 0) > 0:
continue
except (TypeError, ValueError, json.JSONDecodeError):
pass
record = tasks.get(key) record = tasks.get(key)
if not (isinstance(record, dict) and record.get("finished_at")): if not (isinstance(record, dict) and record.get("finished_at")):
continue continue
finished_at = int(record["finished_at"]) finished_at = int(record["finished_at"])
summary = record.get("summary") if isinstance(record.get("summary"), dict) else {}
skipped = bool(record.get("skipped") or summary.get("skipped"))
failed = record.get("ok") is False
if name in {"discover", "snapshot_recent"}: if name in {"discover", "snapshot_recent"}:
# 上次完成推下一档;skip/失败短补跑;静默窗推到 01:00 后。 # 只按上次完成推下一档;进行中由 running 展示,不按墙钟抢跑。
interval = ( interval = (
recent_interval if name == "snapshot_recent" recent_interval if name == "snapshot_recent"
else discover_interval else discover_interval
) )
if skipped: next_runs[name] = finished_at + interval
next_runs[name] = int(
next_after_snapshot_quiet_window(float(finished_at))
)
elif failed:
next_runs[name] = int(finished_at + retry_seconds)
else:
next_runs[name] = int(
interval_next_run(float(finished_at), float(interval))
)
else:
if skipped or failed:
# 日更任务失败/skip:短间隔补跑,绝不按成功推到次日。
# snapshot_seed:全站快照占用 / no_source 时 10 分钟补跑。
gap = (
seed_retry_seconds if name == "snapshot_seed"
else retry_seconds
)
next_runs[name] = int(finished_at + gap)
else: else:
next_runs[name] = int(next_daily_run(finished_at, at)) next_runs[name] = int(next_daily_run(finished_at, at))
errors = [ errors = [
{"task": name, **record} {"task": name, **record}
for name, record in tasks.items() for name, record in tasks.items()
if isinstance(record, dict) if isinstance(record, dict) and record.get("ok") is False
and record.get("ok") is False
and not record.get("skipped")
and not (
isinstance(record.get("summary"), dict)
and record["summary"].get("skipped")
)
] ]
counts = {str(row["status"]): int(row["n"]) for row in conn.execute( counts = {str(row["status"]): int(row["n"]) for row in conn.execute(
"SELECT status, COUNT(*) AS n FROM video GROUP BY status" "SELECT status, COUNT(*) AS n FROM video GROUP BY status"
+5 -17
View File
@@ -1,6 +1,6 @@
import { import {
coverUrl, escapeAttr, escapeHtml, formatCount, formatDate, coverUrl, escapeAttr, escapeHtml, formatCount, formatDate,
formatDelta, formatDuration, formatPercent, formatSignedPercent, metricLabel, statusLabel, formatDelta, formatDuration, formatPercent, metricLabel, statusLabel,
} from "./format.js"; } from "./format.js";
export function cover(item, className = "thumb") { export function cover(item, className = "thumb") {
@@ -30,15 +30,9 @@ export function avatar(item, className = "avatar") {
loading="lazy" referrerpolicy="no-referrer"></div>`; loading="lazy" referrerpolicy="no-referrer"></div>`;
} }
function growthBaselineTag(growth) {
if (growth?.baseline === "zero") return "<small>今日</small>";
if (growth?.baseline === "open") return "<small>入库后</small>";
return "";
}
export function videoCard(item, { export function videoCard(item, {
rank = null, primary = "view", primaryLabel = "", primaryText = null, rank = null, primary = "view", primaryLabel = "", primaryText = null,
showStatus = false, growthField = "delta", showStatus = false,
} = {}) { } = {}) {
const metrics = item.metrics || {}; const metrics = item.metrics || {};
const hasMetric = Object.prototype.hasOwnProperty.call(metrics, primary); const hasMetric = Object.prototype.hasOwnProperty.call(metrics, primary);
@@ -49,13 +43,7 @@ export function videoCard(item, {
const primaryValue = typeof raw === "number" ? formatCount(raw) : raw; const primaryValue = typeof raw === "number" ? formatCount(raw) : raw;
const label = primaryLabel || metricLabel(primary); const label = primaryLabel || metricLabel(primary);
const growth = item.growth; const growth = item.growth;
const showRate = growthField === "rate"; const delta = growth?.available ? growth.delta : null;
const rate = showRate && growth?.available ? growth.rate : null;
const delta = !showRate && growth?.available ? growth.delta : null;
const change = showRate
? (rate === null || rate === undefined ? null : formatSignedPercent(rate))
: (delta === null || delta === undefined ? null : formatDelta(delta));
const changeUp = showRate ? Number(rate) >= 0 : Number(delta) >= 0;
const title = escapeHtml(item.title || "(无标题)"); const title = escapeHtml(item.title || "(无标题)");
const up = escapeHtml(item.up_name || "未知 UP"); const up = escapeHtml(item.up_name || "未知 UP");
const upMid = item.up_mid; const upMid = item.up_mid;
@@ -85,7 +73,7 @@ export function videoCard(item, {
<span class="primary-stat"> <span class="primary-stat">
<b>${escapeHtml(primaryValue ?? "—")}</b><small>${escapeHtml(label)}</small> <b>${escapeHtml(primaryValue ?? "—")}</b><small>${escapeHtml(label)}</small>
</span> </span>
${change !== null ? `<span class="delta ${changeUp ? "up" : "down"}">${change}${growthBaselineTag(growth)}</span>` : ""} ${delta !== null ? `<span class="delta ${delta >= 0 ? "up" : "down"}">${formatDelta(delta)}</span>` : ""}
<span class="secondary-stat">互动 ${formatPercent(item.interaction_rate)}</span> <span class="secondary-stat">互动 ${formatPercent(item.interaction_rate)}</span>
</div> </div>
</div> </div>
@@ -130,7 +118,7 @@ export function metricGrid(item) {
return `<div class="metric-grid"> return `<div class="metric-grid">
${["view", "like", "coin", "favorite", "share", "reply", "danmaku"].map((metric) => { ${["view", "like", "coin", "favorite", "share", "reply", "danmaku"].map((metric) => {
const extra = metric === "view" && growth?.available const extra = metric === "view" && growth?.available
? `<span class="metric-delta ${growth.delta >= 0 ? "up" : "down"}">${formatDelta(growth.delta)}${growthBaselineTag(growth)}</span>` ? `<span class="metric-delta ${growth.delta >= 0 ? "up" : "down"}">${formatDelta(growth.delta)}</span>`
: ""; : "";
return `<div class="metric-cell"> return `<div class="metric-cell">
<span class="metric-label">${escapeHtml(metricLabel(metric))}</span> <span class="metric-label">${escapeHtml(metricLabel(metric))}</span>
-6
View File
@@ -46,12 +46,6 @@ export function formatPercent(value, digits = 2) {
return `${(Number(value) * 100).toFixed(digits)}%`; return `${(Number(value) * 100).toFixed(digits)}%`;
} }
export function formatSignedPercent(value, digits = 2) {
const text = formatPercent(value, digits);
if (text === "—" || Number(value) <= 0) return text;
return `+${text}`;
}
export function formatDate(timestamp) { export function formatDate(timestamp) {
if (!timestamp) return "—"; if (!timestamp) return "—";
const date = new Date(Number(timestamp) * 1000); const date = new Date(Number(timestamp) * 1000);
+3 -3
View File
@@ -353,7 +353,7 @@ a.live-chip:hover { border-color: color-mix(in srgb, var(--accent) 55%, var(--bo
transition: border-color 0.15s cubic-bezier(0.22, 1, 0.36, 1), transition: border-color 0.15s cubic-bezier(0.22, 1, 0.36, 1),
background-color 0.15s cubic-bezier(0.22, 1, 0.36, 1); background-color 0.15s cubic-bezier(0.22, 1, 0.36, 1);
} }
.video-card { grid-template-columns: 160px minmax(0, 1fr); } .video-card { grid-template-columns: 128px minmax(0, 1fr); }
.up-card { grid-template-columns: 64px minmax(0, 1fr); align-items: center; } .up-card { grid-template-columns: 64px minmax(0, 1fr); align-items: center; }
.video-card:hover, .up-card:hover { .video-card:hover, .up-card:hover {
border-color: color-mix(in srgb, var(--accent) 45%, var(--border)); border-color: color-mix(in srgb, var(--accent) 45%, var(--border));
@@ -995,7 +995,7 @@ a.live-chip:hover { border-color: color-mix(in srgb, var(--accent) 55%, var(--bo
.up-stats { grid-template-columns: repeat(5, minmax(0, 1fr)); } .up-stats { grid-template-columns: repeat(5, minmax(0, 1fr)); }
.detail-hero { flex-direction: row; align-items: flex-start; } .detail-hero { flex-direction: row; align-items: flex-start; }
.hero-cover { width: 300px; flex: 0 0 300px; } .hero-cover { width: 300px; flex: 0 0 300px; }
.video-card { grid-template-columns: 160px minmax(0, 1fr); } .video-card { grid-template-columns: 144px minmax(0, 1fr); }
.secondary-stat.optional { display: inline; } .secondary-stat.optional { display: inline; }
} }
@@ -1048,7 +1048,7 @@ a.live-chip:hover { border-color: color-mix(in srgb, var(--accent) 55%, var(--bo
} }
@media (max-width: 380px) { @media (max-width: 380px) {
.video-card { gap: 8px; } .video-card { grid-template-columns: 104px minmax(0, 1fr); gap: 8px; }
.metric-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } .metric-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); }
.candidate-card { grid-template-columns: 80px minmax(0, 1fr); } .candidate-card { grid-template-columns: 80px minmax(0, 1fr); }
.candidate-thumb { width: 80px; } .candidate-thumb { width: 80px; }
+1 -2
View File
@@ -50,7 +50,6 @@ export async function renderAbout(root) {
<li><strong>播放 / 投币 / 收藏 / 评论</strong>:直接取当日快照;弹幕与转发不单列,已并入“互动”。</li> <li><strong>播放 / 投币 / 收藏 / 评论</strong>:直接取当日快照;弹幕与转发不单列,已并入“互动”。</li>
<li><strong>互动</strong>:点赞 + 投币 + 收藏。综合热度也使用全部七项指标,按配置权重加权。</li> <li><strong>互动</strong>:点赞 + 投币 + 收藏。综合热度也使用全部七项指标,按配置权重加权。</li>
<li><strong>互动率</strong>:(点赞 + 投币 + 收藏) / 播放;播放不足 100 不计算。</li> <li><strong>互动率</strong>:(点赞 + 投币 + 收藏) / 播放;播放不足 100 不计算。</li>
<li><strong>播放增速</strong>:卡片上的百分比是增量 / 基线播放。基线播放不足 1000 不进增速榜。排序用增量 / (基线 + ${analytics.growth_rate_prior})。当日发布没有基线,不进增速榜。</li>
<li><strong>综合热度</strong>:每项先 <code>log1p</code>,再按中位数与 IQR 做稳健标准化,最后加权求和。分数只用于同一批数据内排序,不能跨日期比较。</li> <li><strong>综合热度</strong>:每项先 <code>log1p</code>,再按中位数与 IQR 做稳健标准化,最后加权求和。分数只用于同一批数据内排序,不能跨日期比较。</li>
<li><strong>内容效率</strong>:播放 / 内容小时。内容小时 = 视频总时长(多 P 为各 P 之和)÷ 3600。榜单里的单条视频用收缩后的估计:<code>(播放 + κ·μ₀) / (小时 + κ)</code>,κ 默认 ${analytics.video_shrink_minutes} 分钟、μ₀ 为全库播放/小时;详情页保留原始播放/小时。</li> <li><strong>内容效率</strong>:播放 / 内容小时。内容小时 = 视频总时长(多 P 为各 P 之和)÷ 3600。榜单里的单条视频用收缩后的估计:<code>(播放 + κ·μ₀) / (小时 + κ)</code>,κ 默认 ${analytics.video_shrink_minutes} 分钟、μ₀ 为全库播放/小时;详情页保留原始播放/小时。</li>
<li><strong>互动效率</strong>:互动 / 内容小时,互动口径与收缩方式同上。</li> <li><strong>互动效率</strong>:互动 / 内容小时,互动口径与收缩方式同上。</li>
@@ -93,7 +92,7 @@ export async function renderAbout(root) {
<li>B 站不公开完播率与单 P 播放,内容小时是“供给时长”,不是“实际观看时长”。</li> <li>B 站不公开完播率与单 P 播放,内容小时是“供给时长”,不是“实际观看时长”。</li>
<li>多 P 可能让同一用户产生多次播放计数,本站不单独校正,属于已知偏差。</li> <li>多 P 可能让同一用户产生多次播放计数,本站不单独校正,属于已知偏差。</li>
<li>粉丝数可能包含不活跃账号;粉丝快照暂不参与任何榜单。</li> <li>粉丝数可能包含不活跃账号;粉丝快照暂不参与任何榜单。</li>
<li>近 7/30 日增长仍要跨日快照。近 1 日:当日发布按 0 起算;更早发布的新入库按入库时播放量起算,需当天至少两次采样。样本不足时显示提示,不伪造 0。</li> <li>样本不足(少于 2 天快照、单视频 UP)会显示提示,不伪造 0。</li>
</ul> </ul>
</section> </section>
+1 -1
View File
@@ -69,7 +69,7 @@ export async function renderHome(root) {
const growthEmptyTitle = growth.reason const growthEmptyTitle = growth.reason
|| `近 ${growthDays} 日增长暂不可用`; || `近 ${growthDays} 日增长暂不可用`;
const growthEmptyHint = growthDays <= 1 const growthEmptyHint = growthDays <= 1
? "当日发布按 0 起算;更早发布的新入库需要当天至少两次采样。" ? "当前快照不足,完成至少 2 天快照后即可显示播放增量。"
: `需要至少 ${growthDays + 1} 天快照才能计算近 ${growthDays} 日增长。`; : `需要至少 ${growthDays + 1} 天快照才能计算近 ${growthDays} 日增长。`;
const updatedLabel = formatClock(overview.updated_at); const updatedLabel = formatClock(overview.updated_at);
const dataDateLabel = formatDataDate(overview.data_date); const dataDateLabel = formatDataDate(overview.data_date);
@@ -166,7 +166,6 @@ function listHtml(data, scope, sort, startRank) {
return videoCard(item, { return videoCard(item, {
rank: startRank + index, primary: primary.primary, rank: startRank + index, primary: primary.primary,
primaryLabel: primary.label, primaryText: primary.text, primaryLabel: primary.label, primaryText: primary.text,
growthField: sort === "growth_rate" ? "rate" : "delta",
}); });
}).join("")}</div>`; }).join("")}</div>`;
} }
@@ -180,21 +179,6 @@ function durationNote(data) {
return `<div class="section-note">内容效率与互动效率:时长 ≥ ${filter.min_duration_seconds} 秒、发布满 ${filter.maturity_days} 天、播放 ≥ ${filter.min_view}${shrink};时长缺失不参与。</div>`; return `<div class="section-note">内容效率与互动效率:时长 ≥ ${filter.min_duration_seconds} 秒、发布满 ${filter.maturity_days} 天、播放 ≥ ${filter.min_view}${shrink};时长缺失不参与。</div>`;
} }
function growthNote(sort, growthDays, prior) {
if (sort === "growth_rate") {
const strength = Number(prior);
const smooth = Number.isFinite(strength)
? `百分比是真实相对增量;排序按平滑增速(基线 + ${strength})。`
: "百分比是真实相对增量;排序按平滑增速。";
const cold = Number(growthDays) === 1
? "基线播放不足 1000、以及当日发布,不进增速榜。"
: "基线播放不足 1000 不进增速榜。";
return `<div class="section-note">${smooth}${cold}</div>`;
}
if (sort !== "growth" || Number(growthDays) !== 1) return "";
return `<div class="section-note">近 1 日:当日发布按 0 起算;更早发布的新入库按入库时播放量起算,需要当天至少两次采样。</div>`;
}
function bindControls(root) { function bindControls(root) {
root.querySelector("#rank-sort")?.addEventListener("change", (event) => { root.querySelector("#rank-sort")?.addEventListener("change", (event) => {
replaceQuery({ sort: event.target.value, page: null }); replaceQuery({ sort: event.target.value, page: null });
@@ -261,7 +245,6 @@ async function renderScope(root, { query }, scope) {
</header> </header>
${controls({ scope, sort, within, year, growthDays, q, years })} ${controls({ scope, sort, within, year, growthDays, q, years })}
${durationNote(data)} ${durationNote(data)}
${growthNote(sort, growthDays, data.growth_rate_prior)}
${data.available === false ? `<div class="notice-row warning"> ${data.available === false ? `<div class="notice-row warning">
<strong>${sort.startsWith("growth") ? "增长数据尚不可用" : "榜单筛选后为空"}</strong> <strong>${sort.startsWith("growth") ? "增长数据尚不可用" : "榜单筛选后为空"}</strong>
<span>${escapeHtml(data.reason || "")}</span> <span>${escapeHtml(data.reason || "")}</span>
+4 -16
View File
@@ -5,16 +5,8 @@ import {
escapeHtml, formatCount, formatDateTime, formatNumber, escapeHtml, formatCount, formatDateTime, formatNumber,
} from "../format.js"; } from "../format.js";
function taskResultLabel(record) {
const summary = record.summary && typeof record.summary === "object" ? record.summary : {};
if (record.skipped || summary.skipped) {
return `<span class="muted">跳过</span>`;
}
if (record.ok) return "成功";
return `<span class="danger">失败</span>`;
}
function taskRows(tasks) { function taskRows(tasks) {
// Order comes from backend (_task_records started_at DESC); do not re-sort here.
const entries = Object.entries(tasks || {}); const entries = Object.entries(tasks || {});
if (!entries.length) return emptyState("暂无任务记录", "运行一次同步、快照或发现后会写入最近结果。"); if (!entries.length) return emptyState("暂无任务记录", "运行一次同步、快照或发现后会写入最近结果。");
return `<div class="table-wrap"> return `<div class="table-wrap">
@@ -24,16 +16,12 @@ function taskRows(tasks) {
${entries.map(([name, record]) => { ${entries.map(([name, record]) => {
const summary = record.summary && typeof record.summary === "object" const summary = record.summary && typeof record.summary === "object"
? Object.entries(record.summary) ? Object.entries(record.summary)
.filter(([key]) => [ .filter(([key]) => ["added", "removed", "candidates", "notified", "ok", "failed", "total", "completed", "backup_error"].includes(key))
"added", "removed", "candidates", "notified", "ok", "failed",
"total", "completed", "backup_error", "skipped", "already_completed",
"reason", "elapsed", "success",
].includes(key))
.map(([key, value]) => `${key}=${value}`).join(" ") .map(([key, value]) => `${key}=${value}`).join(" ")
: ""; : "";
return `<tr> return `<tr>
<td>${escapeHtml(name)}</td> <td>${escapeHtml(name)}</td>
<td>${taskResultLabel(record)}</td> <td>${record.ok ? "成功" : `<span class="danger">失败</span>`}</td>
<td>${escapeHtml(formatDateTime(record.finished_at))}</td> <td>${escapeHtml(formatDateTime(record.finished_at))}</td>
<td class="summary-cell">${escapeHtml(summary || record.error || "—")}</td> <td class="summary-cell">${escapeHtml(summary || record.error || "—")}</td>
</tr>`; </tr>`;
@@ -113,7 +101,7 @@ export async function renderSystem(root) {
<div><span>新视频小时同步</span><strong>${running.snapshot_recent ? "运行中" : escapeHtml(formatDateTime(next.snapshot_recent))}</strong></div> <div><span>新视频小时同步</span><strong>${running.snapshot_recent ? "运行中" : escapeHtml(formatDateTime(next.snapshot_recent))}</strong></div>
<div><span>新视频搜索</span><strong>${running.discover || running.discovery ? "运行中" : escapeHtml(formatDateTime(next.discover))}</strong></div> <div><span>新视频搜索</span><strong>${running.discover || running.discovery ? "运行中" : escapeHtml(formatDateTime(next.discover))}</strong></div>
</div> </div>
<p class="section-note">进行中显示「运行中」;空闲时按该任务上次完成时间推算下次运行。23:00–01:00(上海)为全站快照窗:发现 / 小时同步 / 候选清理会延后到 01:00 后;粉丝更新不避开。跳过或失败不会记成成功,会短间隔补跑。</p> <p class="section-note">进行中显示「运行中」;空闲时按该任务上次完成时间推算下次运行,不按墙钟抢跑。新视频小时同步与全站快照互斥。</p>
</section> </section>
<section class="panel"> <section class="panel">
+1 -1
View File
@@ -62,7 +62,7 @@ export async function renderVideo(root, { params, query }) {
<span>${escapeHtml(formatDuration(detail.duration))}</span> <span>${escapeHtml(formatDuration(detail.duration))}</span>
${detail.pages ? `<span>${formatNumber(detail.pages)} P</span>` : ""} ${detail.pages ? `<span>${formatNumber(detail.pages)} P</span>` : ""}
${badge(statusLabel(detail.status), `status-${detail.status}`)} ${badge(statusLabel(detail.status), `status-${detail.status}`)}
${samples < 2 && !detail.growth?.available ? badge("样本不足", "warning") : ""} ${samples < 2 ? badge("样本不足", "warning") : ""}
</div> </div>
${detail.blacklisted ? `<p class="section-note warning-text">该 UP 在黑名单中;黑名单只影响未来发现,不影响当前视频追踪。</p>` : ""} ${detail.blacklisted ? `<p class="section-note warning-text">该 UP 在黑名单中;黑名单只影响未来发现,不影响当前视频追踪。</p>` : ""}
${detail.up_follower_backfilled ? `<p class="section-note">UP 粉丝数为回填值,不是真实的入库时点。</p>` : ""} ${detail.up_follower_backfilled ? `<p class="section-note">UP 粉丝数为回填值,不是真实的入库时点。</p>` : ""}
-27
View File
@@ -12,7 +12,6 @@ from reaction_board.analytics import ( # noqa: E402
METRICS, METRICS,
age_days, age_days,
compute_growth, compute_growth,
smoothed_growth_rate,
content_hours, content_hours,
eb_rate, eb_rate,
effective_shrink_minutes, effective_shrink_minutes,
@@ -132,18 +131,6 @@ def test_compute_growth():
no_gap = compute_growth(100, 90, 0) no_gap = compute_growth(100, 90, 0)
assert no_gap.available is False assert no_gap.available is False
same_day = compute_growth(100, 90, 0, allow_zero_gap=True)
assert same_day.available is True
assert same_day.delta == 10
assert same_day.per_day is None
assert same_day.gap_days == 0
assert abs(same_day.rate - 10 / 90) < 1e-12
from_zero_same_day = compute_growth(50, 0, 0, allow_zero_gap=True)
assert from_zero_same_day.delta == 50
assert from_zero_same_day.rate == 50.0
assert from_zero_same_day.per_day is None
growth = compute_growth(1000, 800, 7) growth = compute_growth(1000, 800, 7)
assert growth.available is True assert growth.available is True
assert growth.delta == 200 assert growth.delta == 200
@@ -162,19 +149,6 @@ def test_compute_growth():
assert negative.per_day == -5.0 assert negative.per_day == -5.0
def test_smoothed_growth_rate():
# 80→512:真实增速 5.4;prior=500 时排序分 432/580
assert abs(smoothed_growth_rate(432, 80, 500) - 432 / 580) < 1e-12
assert abs(smoothed_growth_rate(432, 5000, 500) - 432 / 5500) < 1e-12
assert abs(smoothed_growth_rate(432, 80, 1000) - 432 / 1080) < 1e-12
# prior=0 时排序分退回真实相对增速(基线 ≥ 1)
assert smoothed_growth_rate(432, 80, 0) == 432 / 80
assert smoothed_growth_rate(None, 80, 500) is None
assert smoothed_growth_rate(432, None, 500) is None
assert smoothed_growth_rate(10, 0, 0) is None
assert smoothed_growth_rate(10, 80, -1) is None
def test_normalize_cover_and_hosts(): def test_normalize_cover_and_hosts():
assert normalize_cover(None) is None assert normalize_cover(None) is None
assert normalize_cover("") is None assert normalize_cover("") is None
@@ -265,7 +239,6 @@ def main_test():
test_score_percentiles() test_score_percentiles()
test_interaction_rate_thresholds() test_interaction_rate_thresholds()
test_compute_growth() test_compute_growth()
test_smoothed_growth_rate()
test_normalize_cover_and_hosts() test_normalize_cover_and_hosts()
test_content_efficiency_helpers() test_content_efficiency_helpers()
test_age_and_maturity() test_age_and_maturity()
+2 -618
View File
@@ -51,7 +51,7 @@ def test_init_and_status():
path = write_config(tmp) path = write_config(tmp)
code, output = run_cli(["init", "--config", str(path)]) code, output = run_cli(["init", "--config", str(path)])
assert code == 0 assert code == 0
assert '"schema_version": 8' in output assert '"schema_version": 7' in output
db = Database(Path(tmp) / "app.db") db = Database(Path(tmp) / "app.db")
db.initialize() db.initialize()
@@ -332,607 +332,6 @@ def test_backfill_followers_dry_run():
assert '"total": 0' in output assert '"total": 0' in output
def test_help_lists_snapshot_seed():
from reaction_board.cli import _build_parser
parser = _build_parser()
help_text = parser.format_help()
assert "snapshot-seed" in help_text
args = parser.parse_args(["snapshot-seed", "--date", "2026-10-31"])
assert args.command == "snapshot-seed"
assert args.date == "2026-10-31"
def test_snapshot_seed_cli_clones_offline():
from reaction_board.bilibili.models import FavResource
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp, cookie="", csrf="")
code, _ = run_cli(["init", "--config", str(path)])
assert code == 0
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
media_id = 11
repo.upsert_from_fav(
FavResource(
aid=1, bvid="BV1xx411c7mD", title="t", up_mid=9,
up_name="u", pubdate=1, duration=10, cover=None,
attr=0, fav_time=None,
),
media_id,
was_added=True,
)
yday, today = "2026-10-30", "2026-10-31"
repo.upsert_snapshot(
yday, 1,
metrics={"view": 10, "like": 1, "coin": 0, "favorite": 0,
"share": 0, "reply": 0, "danmaku": 0},
ok=True, err=None, captured_at=1_800_000_000,
)
repo.kv_set("snapshot:last_completed", yday)
repo.kv_set("snapshot:completed:" + yday, "1")
finally:
db.close()
code, output = run_cli([
"snapshot-seed", "--config", str(path), "--date", today,
])
assert code == 0, output
assert '"reason": "seeded"' in output
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
assert repo.kv_get("snapshot:seeded:" + today) == "1"
row = repo.get_snapshot(today, 1)
assert row is not None and int(row["view"]) == 10
assert row["src"] == "seed"
finally:
db.close()
def test_snapshot_seed_defers_when_snapshot_running():
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp, cookie="", csrf="")
code, _ = run_cli(["init", "--config", str(path)])
assert code == 0
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
repo.kv_set(
"task:running:snapshot",
json.dumps({"name": "snapshot", "started_at": 1}),
)
repo.kv_set("snapshot:last_completed", "2026-10-30")
finally:
db.close()
code, output = run_cli([
"snapshot-seed", "--config", str(path), "--date", "2026-10-31",
])
assert code == 1, output
assert "snapshot_running" in output
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
last = json.loads(repo.kv_get("task:last:snapshot_seed"))
assert last["ok"] is False
assert last["skipped"] is True
assert last["error"] == "snapshot_running"
assert repo.kv_get("task:running:snapshot_seed") is None
assert repo.kv_get("task:running:snapshot") is not None
finally:
db.close()
def test_snapshot_seed_no_source_is_non_success():
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp, cookie="", csrf="")
code, _ = run_cli(["init", "--config", str(path)])
assert code == 0
code, output = run_cli([
"snapshot-seed", "--config", str(path), "--date", "2026-10-31",
])
assert code == 1, output
assert "no_source" in output
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
last = json.loads(repo.kv_get("task:last:snapshot_seed"))
assert last["ok"] is False and last["skipped"] is True
assert last["error"] == "no_source"
finally:
db.close()
def test_defer_snapshot_seed_schedules_ten_minutes():
"""调度辅助:skip 记非成功,next_run = finished + 600;落盘 task:state=deferred。"""
from reaction_board.logging_json import JsonLogger
from reaction_board.task_state import STATUS_DEFERRED, load_state
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp, cookie="", csrf="")
code, _ = run_cli(["init", "--config", str(path)])
assert code == 0
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
logger = JsonLogger()
started = 1_800_000_000
finished = started + 5
ctrl = cli._defer_snapshot_seed(
repo, logger, started_at=started, reason="snapshot_running",
finished_at=finished,
)
assert ctrl.ok is False
assert ctrl.error == "snapshot_running"
expected_due = finished + cli._SEED_RETRY_SECONDS
assert abs(ctrl.next_run - expected_due) < 0.01
last = json.loads(repo.kv_get("task:last:snapshot_seed"))
assert last["skipped"] is True and last["ok"] is False
state = load_state(repo, "snapshot_seed")
assert state is not None
assert state.status == STATUS_DEFERRED
assert abs(state.due_at - expected_due) < 0.01
assert state.due_at > finished
assert state.error == "snapshot_running"
finally:
db.close()
def test_quiet_window_defer_persists_deferred_state():
"""H8: 静默窗 defer 落盘 status=deferred + due_at=窗后 01:00,不得记成功。"""
from datetime import datetime
from unittest import mock
from reaction_board.logging_json import JsonLogger
from reaction_board.task_state import STATUS_DEFERRED, load_state
from reaction_board.timeutil import (
SHANGHAI_TZ,
next_after_snapshot_quiet_window,
)
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp, cookie="", csrf="")
code, _ = run_cli(["init", "--config", str(path)])
assert code == 0
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
logger = JsonLogger()
# 静默窗内:2026-11-03 23:30 上海
quiet = datetime(2026, 11, 3, 23, 30, tzinfo=SHANGHAI_TZ).timestamp()
expected_due = next_after_snapshot_quiet_window(quiet)
assert expected_due > quiet
with mock.patch("reaction_board.cli.time.time", return_value=quiet):
ctrl = cli._quiet_window_defer(
repo, logger, "discovery", started_at=int(quiet) - 1,
)
assert ctrl.ok is False
assert ctrl.error == cli._QUIET_SKIP_REASON
assert abs(ctrl.next_run - expected_due) < 0.01
last = json.loads(repo.kv_get("task:last:discovery"))
assert last["ok"] is False and last["skipped"] is True
assert last["error"] == cli._QUIET_SKIP_REASON
# 不得推成成功完成类 kv
assert repo.kv_get("snapshot:last_completed") is None
assert repo.kv_get("discovery:last_completed") is None
state = load_state(repo, "discovery")
assert state is not None
assert state.status == STATUS_DEFERRED
assert abs(state.due_at - expected_due) < 0.01
assert state.due_at > quiet
assert state.error == cli._QUIET_SKIP_REASON
finally:
db.close()
def test_snapshot_seed_same_day_completed_succeeds():
"""当日已完成时手动 snapshot-seed 应成功 already_completed,不走 no_source。"""
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp, cookie="", csrf="")
code, _ = run_cli(["init", "--config", str(path)])
assert code == 0
today = "2026-10-31"
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
repo.kv_set("snapshot:last_completed", today)
repo.kv_set("snapshot:completed:" + today, "1")
finally:
db.close()
code, output = run_cli([
"snapshot-seed", "--config", str(path), "--date", today,
])
assert code == 0, output
assert '"reason": "already_completed"' in output
assert "no_source" not in output
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
last = json.loads(repo.kv_get("task:last:snapshot_seed"))
assert last["ok"] is True
finally:
db.close()
def test_candidate_cleanup_failure_retries_soon():
"""candidate_cleanup 异常应记失败任务并约 5 分钟补跑,而非落到 86400 间隔。"""
class FakeClient:
def check_login(self):
return LoginState(True, 1, "tester", 0, "OK")
class FakeDiscovery:
def __init__(self, *args, **kwargs):
pass
def run(self):
from reaction_board.services.discovery import DiscoverySummary
return DiscoverySummary(started_at=1, finished_at=1)
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp)
data = json.loads(path.read_text(encoding="utf-8"))
data["folders"] = []
data["track_fans"] = []
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
original_client = cli._client
original_discovery = cli.DiscoveryService
original_delete = Repository.delete_expired_candidates
captured = {}
def boom(self, *args, **kwargs):
raise RuntimeError("cleanup db boom")
def fake_client(credentials, *args, **kwargs):
return FakeClient()
# 只跑 cleanup:把其它任务 next_run 推远;通过 Scheduler 包装拦截。
original_scheduler = cli.Scheduler
class CaptureScheduler(original_scheduler):
def __init__(self, tasks, **kwargs):
far = 1_000_000_000_000.0
for task in tasks:
if task.name != "candidate_cleanup":
task.next_run = far
else:
task.next_run = 0.0
super().__init__(tasks, **kwargs)
captured["scheduler"] = self
cli._client = fake_client
cli.DiscoveryService = FakeDiscovery
cli.Scheduler = CaptureScheduler
Repository.delete_expired_candidates = boom
try:
code, output = run_cli(["run", "--once", "--config", str(path)])
finally:
cli._client = original_client
cli.DiscoveryService = original_discovery
cli.Scheduler = original_scheduler
Repository.delete_expired_candidates = original_delete
assert code == 0, output
from reaction_board.task_state import STATUS_FAILED, load_state
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
last = json.loads(repo.kv_get("task:last:candidate_cleanup"))
assert last["ok"] is False
assert "cleanup db boom" in last["error"]
state = load_state(repo, "candidate_cleanup")
assert state is not None
assert state.status == STATUS_FAILED
assert state.due_at > 0
delta_state = state.due_at - __import__("time").time()
assert 250 <= delta_state <= 360, delta_state
assert "cleanup db boom" in (state.error or "")
finally:
db.close()
scheduler = captured["scheduler"]
cleanup_task = next(t for t in scheduler.tasks if t.name == "candidate_cleanup")
# next_run ≈ finished + 300;允许几秒时钟误差
assert cleanup_task.next_run > 0
delta = cleanup_task.next_run - __import__("time").time()
assert 250 <= delta <= 360, delta
def test_initial_daily_next_run_uses_wall_clock_not_zero():
"""常驻启动:无状态时对齐墙钟;有 task:state.due_at 时认落盘。"""
from reaction_board.timeutil import next_daily_run, SHANGHAI_TZ
from reaction_board.task_state import mark_done
from datetime import datetime
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp, cookie="", csrf="")
assert run_cli(["init", "--config", str(path)])[0] == 0
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
# 2026-11-03 10:00 上海
now = datetime(2026, 11, 3, 10, 0, tzinfo=SHANGHAI_TZ).timestamp()
nxt = cli._initial_daily_next_run(
repo, "snapshot", "23:15", now=now,
)
assert nxt == next_daily_run(now, "23:15")
assert nxt > now
# 昨晚成功 → state.due_at=今晚 23:15
yday_done = datetime(2026, 11, 2, 23, 20, tzinfo=SHANGHAI_TZ).timestamp()
due = next_daily_run(yday_done, "23:15")
mark_done(repo, "snapshot", due_at=due, now=yday_done, stat_date="2026-11-02")
nxt2 = cli._initial_daily_next_run(
repo, "snapshot", "23:15", now=now,
)
assert nxt2 == due
assert nxt2 > now
finally:
db.close()
def test_reached_daily_slot_gates_force_seed():
from reaction_board.timeutil import SHANGHAI_TZ
from datetime import datetime
before = datetime(2026, 11, 3, 10, 0, tzinfo=SHANGHAI_TZ).timestamp()
at = datetime(2026, 11, 3, 23, 15, tzinfo=SHANGHAI_TZ).timestamp()
after = datetime(2026, 11, 3, 23, 16, tzinfo=SHANGHAI_TZ).timestamp()
assert cli._reached_daily_slot("23:15", before) is False
assert cli._reached_daily_slot("23:15", at) is True
assert cli._reached_daily_slot("23:15", after) is True
def test_restart_after_seed_schedules_snapshot_at_wall_clock():
"""午夜 seed 后重启:认 task:state pending due_at=当晚,首轮不得跑/force。"""
from reaction_board.scheduler import ScheduledTask, Scheduler
from reaction_board.timeutil import next_daily_run, SHANGHAI_TZ, shanghai_date
from reaction_board.task_state import ensure_pending, mark_done
from datetime import datetime
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp, cookie="", csrf="")
assert run_cli(["init", "--config", str(path)])[0] == 0
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
now = datetime(2026, 11, 3, 10, 30, tzinfo=SHANGHAI_TZ).timestamp()
today = shanghai_date(now)
repo.kv_set(f"snapshot:seeded:{today}", "1")
slot = datetime(2026, 11, 3, 23, 15, tzinfo=SHANGHAI_TZ).timestamp()
seed_done_at = datetime(2026, 11, 3, 0, 1, tzinfo=SHANGHAI_TZ).timestamp()
mark_done(
repo, "snapshot_seed",
due_at=next_daily_run(seed_done_at, "00:00"),
now=seed_done_at,
stat_date=today,
)
ensure_pending(
repo, "snapshot", due_at=slot, stat_date=today, now=seed_done_at,
)
ensure_pending(
repo, "fans", due_at=slot, stat_date=today, now=seed_done_at,
)
snapshot_at = "23:15"
seed_at = "00:00"
snapshot_next = cli._initial_daily_next_run(
repo, "snapshot", snapshot_at, now=now,
)
seed_next = cli._initial_daily_next_run(
repo, "snapshot_seed", seed_at, now=now,
)
fans_next = cli._initial_daily_next_run(
repo, "fans", snapshot_at, now=now,
)
assert snapshot_next == slot
assert fans_next == slot
assert seed_next == next_daily_run(seed_done_at, seed_at)
assert seed_next > now
force_calls = []
def run_snapshot():
force = (
repo.kv_get(f"snapshot:seeded:{today}") == "1"
and cli._reached_daily_slot(snapshot_at, clock[0])
)
force_calls.append(force)
if force and clock[0] < slot:
raise AssertionError("daytime restart must not force-seal seed")
return None
clock = [now]
scheduler = Scheduler(
[
ScheduledTask("snapshot_seed", 86400.0, lambda: None, next_run=seed_next),
ScheduledTask("snapshot", 86400.0, run_snapshot, next_run=snapshot_next),
ScheduledTask("fans", 86400.0, lambda: None, next_run=fans_next),
],
clock=lambda: clock[0],
)
assert scheduler.run_due() == []
assert force_calls == []
clock[0] = slot
results = scheduler.run_due()
assert len(results) >= 1
assert force_calls == [True]
finally:
db.close()
slot = datetime(2026, 11, 3, 23, 15, tzinfo=SHANGHAI_TZ).timestamp()
assert cli._reached_daily_slot("23:15", slot) is True
def test_daytime_seed_marker_does_not_force_without_slot():
"""即使 seeded=1,未到 snapshot_at 也不得 force(防守:即便误触发 run_snapshot)。"""
from reaction_board.timeutil import SHANGHAI_TZ
from datetime import datetime
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp, cookie="", csrf="")
assert run_cli(["init", "--config", str(path)])[0] == 0
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
now = datetime(2026, 11, 3, 12, 0, tzinfo=SHANGHAI_TZ).timestamp()
today = "2026-11-03"
repo.kv_set(f"snapshot:seeded:{today}", "1")
force_seed = (
repo.kv_get(f"snapshot:seeded:{today}") == "1"
and cli._reached_daily_slot("23:15", now)
)
assert force_seed is False
finally:
db.close()
def test_off_plan_completion_uses_started_at_not_finished_at():
"""规则2:完成记录 started_at 早于今日计划点 → off-plan;禁用 finished_at。"""
from datetime import datetime
from reaction_board.timeutil import SHANGHAI_TZ
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp, cookie="", csrf="")
assert run_cli(["init", "--config", str(path)])[0] == 0
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
slot = datetime(2026, 11, 3, 23, 15, tzinfo=SHANGHAI_TZ).timestamp()
# 白天开始、晚上才结束:started_at < slot,finished_at > slot
day_start = datetime(2026, 11, 3, 10, 0, tzinfo=SHANGHAI_TZ).timestamp()
day_end = datetime(2026, 11, 3, 23, 30, tzinfo=SHANGHAI_TZ).timestamp()
repo.kv_set(
"task:last:snapshot",
json.dumps({
"name": "snapshot",
"ok": True,
"skipped": False,
"started_at": int(day_start),
"finished_at": int(day_end),
"error": None,
"summary": {"stat_date": "2026-11-03"},
}, ensure_ascii=False),
)
assert cli._last_success_started_at(repo, "snapshot") == float(int(day_start))
assert cli._off_plan_completion(repo, "snapshot", "23:15", slot) is True
# 若误用 finished_at 会判成计划内;started_at 必须仍为 off-plan
assert day_end >= slot
assert day_start < slot
# 计划内:started_at == slot → 不可 off-plan
repo.kv_set(
"task:last:snapshot",
json.dumps({
"name": "snapshot",
"ok": True,
"skipped": False,
"started_at": int(slot),
"finished_at": int(slot + 600),
"error": None,
"summary": {"stat_date": "2026-11-03"},
}, ensure_ascii=False),
)
assert cli._off_plan_completion(repo, "snapshot", "23:15", slot) is False
assert cli._off_plan_completion(
repo, "snapshot", "23:15", slot + 60,
) is False
finally:
db.close()
def test_cross_midnight_force_seal_ignores_todays_slot_gate():
"""规则1:昨天 seed 未完成,跨日后即便未到今日 23:15 也 force。"""
from datetime import datetime
from reaction_board.timeutil import SHANGHAI_TZ
now = datetime(2026, 11, 3, 0, 10, tzinfo=SHANGHAI_TZ).timestamp()
assert cli._reached_daily_slot("23:15", now) is False
assert cli._should_force_daily_seal(
seeded=True,
stat_date="2026-11-02",
today="2026-11-03",
slot_hhmm="23:15",
now=now,
) is True
# 同日白天仍不得 force
day = datetime(2026, 11, 3, 12, 0, tzinfo=SHANGHAI_TZ).timestamp()
assert cli._should_force_daily_seal(
seeded=True,
stat_date="2026-11-03",
today="2026-11-03",
slot_hhmm="23:15",
now=day,
) is False
night = datetime(2026, 11, 3, 23, 15, tzinfo=SHANGHAI_TZ).timestamp()
assert cli._should_force_daily_seal(
seeded=True,
stat_date="2026-11-03",
today="2026-11-03",
slot_hhmm="23:15",
now=night,
) is True
def test_initial_daily_bootstraps_failed_last_without_state():
"""A: 无 state、仅 task:last 失败 → 启动 next_run 立刻可补,不退墙钟。"""
from datetime import datetime
from reaction_board.timeutil import SHANGHAI_TZ, next_daily_run
from reaction_board.task_state import STATUS_FAILED, load_state
with tempfile.TemporaryDirectory() as tmp:
path = write_config(tmp, cookie="", csrf="")
assert run_cli(["init", "--config", str(path)])[0] == 0
db = Database(Path(tmp) / "app.db")
db.initialize()
try:
repo = Repository(db)
now = datetime(2026, 11, 3, 18, 0, tzinfo=SHANGHAI_TZ).timestamp()
repo.kv_set(
"task:last:snapshot",
json.dumps({
"name": "snapshot",
"ok": False,
"skipped": False,
"started_at": int(now - 900),
"finished_at": int(now - 800),
"error": "net",
"summary": {"stat_date": "2026-11-03"},
}, ensure_ascii=False),
)
wall = next_daily_run(now, "23:15")
nxt = cli._initial_daily_next_run(
repo, "snapshot", "23:15", now=now,
)
assert nxt <= now
assert nxt != wall
state = load_state(repo, "snapshot")
assert state is not None
assert state.status == STATUS_FAILED
finally:
db.close()
def main_test(): def main_test():
test_init_and_status() test_init_and_status()
test_missing_config() test_missing_config()
@@ -946,22 +345,7 @@ def main_test():
test_snapshot_backup_failure_returns_error_and_retries() test_snapshot_backup_failure_returns_error_and_retries()
test_candidates_and_discover_commands() test_candidates_and_discover_commands()
test_backfill_followers_dry_run() test_backfill_followers_dry_run()
test_help_lists_snapshot_seed() print("OK: CLI init/sync/snapshot/auth-check/run/status/backfill/错误路径测试全部通过")
test_snapshot_seed_cli_clones_offline()
test_snapshot_seed_defers_when_snapshot_running()
test_snapshot_seed_no_source_is_non_success()
test_snapshot_seed_same_day_completed_succeeds()
test_defer_snapshot_seed_schedules_ten_minutes()
test_quiet_window_defer_persists_deferred_state()
test_candidate_cleanup_failure_retries_soon()
test_initial_daily_next_run_uses_wall_clock_not_zero()
test_reached_daily_slot_gates_force_seed()
test_restart_after_seed_schedules_snapshot_at_wall_clock()
test_daytime_seed_marker_does_not_force_without_slot()
test_off_plan_completion_uses_started_at_not_finished_at()
test_cross_midnight_force_seal_ignores_todays_slot_gate()
test_initial_daily_bootstraps_failed_last_without_state()
print("OK: CLI init/sync/snapshot/snapshot-seed/auth-check/run/status/backfill/错误路径测试全部通过")
if __name__ == "__main__": if __name__ == "__main__":
-4
View File
@@ -146,7 +146,6 @@ def test_analytics_defaults_and_validation():
assert cfg.analytics.video_shrink_minutes == 60 assert cfg.analytics.video_shrink_minutes == 60
assert cfg.analytics.shrink_minutes is None assert cfg.analytics.shrink_minutes is None
assert cfg.analytics.home_growth_days == "auto" assert cfg.analytics.home_growth_days == "auto"
assert cfg.analytics.growth_rate_prior == 1000
assert cfg.schedule.recent_snapshot_minutes == 60 assert cfg.schedule.recent_snapshot_minutes == 60
assert cfg.schedule.recent_video_days == 2 assert cfg.schedule.recent_video_days == 2
assert cfg.schedule.snapshot_at == "23:15" assert cfg.schedule.snapshot_at == "23:15"
@@ -160,7 +159,6 @@ def test_analytics_defaults_and_validation():
"video_shrink_minutes": 30, "video_shrink_minutes": 30,
"shrink_minutes": 90, "shrink_minutes": 90,
"home_growth_days": 1, "home_growth_days": 1,
"growth_rate_prior": 2000,
} }
data["schedule"] = { data["schedule"] = {
"discover_minutes": 30, "discover_minutes": 30,
@@ -178,7 +176,6 @@ def test_analytics_defaults_and_validation():
assert cfg.analytics.video_shrink_minutes == 30 assert cfg.analytics.video_shrink_minutes == 30
assert cfg.analytics.shrink_minutes == 90 assert cfg.analytics.shrink_minutes == 90
assert cfg.analytics.home_growth_days == 1 assert cfg.analytics.home_growth_days == 1
assert cfg.analytics.growth_rate_prior == 2000
assert cfg.schedule.recent_snapshot_minutes == 15 assert cfg.schedule.recent_snapshot_minutes == 15
assert cfg.schedule.recent_video_days == 3 assert cfg.schedule.recent_video_days == 3
@@ -197,7 +194,6 @@ def test_analytics_defaults_and_validation():
({"home_growth_days": 3}, "home_growth_days"), ({"home_growth_days": 3}, "home_growth_days"),
({"home_growth_days": "7"}, "home_growth_days"), ({"home_growth_days": "7"}, "home_growth_days"),
({"home_growth_days": 0}, "home_growth_days"), ({"home_growth_days": 0}, "home_growth_days"),
({"growth_rate_prior": -1}, "growth_rate_prior"),
): ):
data = base_config() data = base_config()
data["analytics"] = bad data["analytics"] = bad
+4 -4
View File
@@ -24,7 +24,7 @@ def test_write_read_transaction():
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
db = make_db(tmp) db = make_db(tmp)
try: try:
assert db.schema_version == 8 assert db.schema_version == 7
db.write(lambda c: c.execute("INSERT INTO kv(k, v) VALUES ('a', '1')")) db.write(lambda c: c.execute("INSERT INTO kv(k, v) VALUES ('a', '1')"))
assert db.read(lambda c: c.execute("SELECT v FROM kv WHERE k='a'").fetchone()[0]) == "1" assert db.read(lambda c: c.execute("SELECT v FROM kv WHERE k='a'").fetchone()[0]) == "1"
@@ -94,8 +94,8 @@ def test_close_and_reinitialize():
db2 = make_db(tmp) db2 = make_db(tmp)
try: try:
assert db2.schema_version == 8 assert db2.schema_version == 7
assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 8 assert db2.read(lambda c: c.execute("SELECT MAX(version) FROM schema_version").fetchone()[0]) == 7
finally: finally:
db2.close() db2.close()
@@ -123,7 +123,7 @@ def test_migration_backup_on_existing_db():
db = Database(path, backup_dir=Path(tmp) / "backup") db = Database(path, backup_dir=Path(tmp) / "backup")
try: try:
applied = db.initialize() applied = db.initialize()
assert applied == [1, 2, 3, 4, 5, 6, 7, 8] assert applied == [1, 2, 3, 4, 5, 6, 7]
assert len(list((Path(tmp) / "backup").glob("app-pre-v0-*.db"))) == 1 assert len(list((Path(tmp) / "backup").glob("app-pre-v0-*.db"))) == 1
finally: finally:
db.close() db.close()
+8 -46
View File
@@ -38,8 +38,8 @@ def test_fresh_and_idempotent():
conn = sqlite3.connect(Path(tmp) / "app.db") conn = sqlite3.connect(Path(tmp) / "app.db")
try: try:
applied = run_migrations(conn) applied = run_migrations(conn)
assert applied == [1, 2, 3, 4, 5, 6, 7, 8] assert applied == [1, 2, 3, 4, 5, 6, 7]
assert current_version(conn) == 8 assert current_version(conn) == 7
assert EXPECTED_TABLES <= table_names(conn) assert EXPECTED_TABLES <= table_names(conn)
assert "schema_version" in table_names(conn) assert "schema_version" in table_names(conn)
columns = { columns = {
@@ -70,14 +70,10 @@ def test_fresh_and_idempotent():
assert { assert {
"stat_date", "ip", "path", "hits", "first_at", "last_at", "stat_date", "ip", "path", "hits", "first_at", "last_at",
} <= visit_columns } <= visit_columns
snapshot_columns = {
row[1] for row in conn.execute("PRAGMA table_info(snapshot)")
}
assert {"view_open", "open_captured_at"} <= snapshot_columns
# 再跑一次不应重复应用 # 再跑一次不应重复应用
assert run_migrations(conn) == [] assert run_migrations(conn) == []
row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone() row = conn.execute("SELECT COUNT(*) FROM schema_version").fetchone()
assert row[0] == 8 assert row[0] == 7
finally: finally:
conn.close() conn.close()
@@ -92,15 +88,15 @@ def test_backup_before_upgrade():
conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')") conn.execute("INSERT INTO kv(k, v) VALUES ('hello', 'world')")
conn.commit() conn.commit()
v9 = Migration(9, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",)) v8 = Migration(8, "add probe", ("CREATE TABLE probe (id INTEGER PRIMARY KEY)",))
applied = run_migrations( applied = run_migrations(
conn, conn,
backup_dir=backup_dir, backup_dir=backup_dir,
now=1_800_000_000, now=1_800_000_000,
migrations=MIGRATIONS + (v9,), migrations=MIGRATIONS + (v8,),
) )
assert applied == [9] assert applied == [8]
backups = list(backup_dir.glob("app-pre-v8-*.db")) backups = list(backup_dir.glob("app-pre-v7-*.db"))
assert len(backups) == 1 assert len(backups) == 1
check = sqlite3.connect(backups[0]) check = sqlite3.connect(backups[0])
try: try:
@@ -185,46 +181,13 @@ def test_v5_normalizes_unfaved():
) )
conn.commit() conn.commit()
applied = run_migrations(conn) applied = run_migrations(conn)
assert applied == [5, 6, 7, 8] assert applied == [5, 6, 7]
rows = dict(conn.execute("SELECT aid, status FROM video").fetchall()) rows = dict(conn.execute("SELECT aid, status FROM video").fetchall())
assert rows == {1: "active", 2: "dropped"} assert rows == {1: "active", 2: "dropped"}
finally: finally:
conn.close() conn.close()
def test_v8_backfills_view_open():
with tempfile.TemporaryDirectory() as tmp:
conn = sqlite3.connect(Path(tmp) / "app.db")
try:
run_migrations(conn, migrations=MIGRATIONS[:7])
conn.execute(
"INSERT INTO video(aid, title, pubdate, status, first_seen) "
"VALUES (1, 't', 1, 'active', 1)"
)
conn.execute(
'INSERT INTO snapshot(stat_date, aid, "view", captured_at, ok) '
"VALUES ('2026-10-31', 1, 500, 100, 1)"
)
conn.execute(
'INSERT INTO snapshot(stat_date, aid, "view", captured_at, ok) '
"VALUES ('2026-10-30', 1, NULL, 90, 0)"
)
conn.commit()
assert run_migrations(conn) == [8]
ok_row = conn.execute(
"SELECT view_open, open_captured_at FROM snapshot "
"WHERE stat_date='2026-10-31'"
).fetchone()
assert ok_row[0] == 500 and ok_row[1] == 100
fail_row = conn.execute(
"SELECT view_open, open_captured_at FROM snapshot "
"WHERE stat_date='2026-10-30'"
).fetchone()
assert fail_row[0] is None and fail_row[1] is None
finally:
conn.close()
def main(): def main():
test_fresh_and_idempotent() test_fresh_and_idempotent()
test_backup_before_upgrade() test_backup_before_upgrade()
@@ -232,7 +195,6 @@ def main():
test_prune_backups() test_prune_backups()
test_validate_migration_order() test_validate_migration_order()
test_v5_normalizes_unfaved() test_v5_normalizes_unfaved()
test_v8_backfills_view_open()
print("OK: SQLite 迁移、备份与回滚测试全部通过") print("OK: SQLite 迁移、备份与回滚测试全部通过")
-54
View File
@@ -296,59 +296,6 @@ def test_pending_snapshot_aids():
db.close() db.close()
def test_upsert_snapshot_preserves_view_open():
with tempfile.TemporaryDirectory() as tmp:
db, repo = setup(tmp)
try:
metrics = {
"view": 10, "like": 1, "favorite": 0, "coin": 0,
"share": 0, "reply": 0, "danmaku": 0,
}
repo.upsert_snapshot(
"2026-10-31", 1, metrics=metrics, ok=True, err=None, captured_at=10,
)
first = repo.get_snapshot("2026-10-31", 1)
assert first["view"] == 10
assert first["view_open"] == 10
assert first["open_captured_at"] == 10
metrics["view"] = 40
repo.upsert_snapshot(
"2026-10-31", 1, metrics=metrics, ok=True, err=None, captured_at=20,
)
second = repo.get_snapshot("2026-10-31", 1)
assert second["view"] == 40
assert second["view_open"] == 10
assert second["open_captured_at"] == 10
assert second["captured_at"] == 20
repo.upsert_snapshot(
"2026-10-31", 2, metrics=None, ok=False, err="x", captured_at=1,
)
failed = repo.get_snapshot("2026-10-31", 2)
assert failed["view_open"] is None
metrics["view"] = 7
repo.upsert_snapshot(
"2026-10-31", 2, metrics=metrics, ok=True, err=None, captured_at=5,
)
recovered = repo.get_snapshot("2026-10-31", 2)
assert recovered["view"] == 7
assert recovered["view_open"] == 7
assert recovered["open_captured_at"] == 5
copied = repo.copy_ok_snapshots(
"2026-10-31", "2026-11-01", captured_at=30, src="seed",
)
assert copied == 2
seeded = repo.get_snapshot("2026-11-01", 1)
assert seeded["view"] == 40
assert seeded["view_open"] == 40
assert seeded["open_captured_at"] == 30
assert seeded["src"] == "seed"
finally:
db.close()
def test_up_follower_snapshot_and_backfill(): def test_up_follower_snapshot_and_backfill():
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
db, repo = setup(tmp) db, repo = setup(tmp)
@@ -420,7 +367,6 @@ def main():
test_primary_folder_is_order_independent() test_primary_folder_is_order_independent()
test_dropped_invalid_preserves_original_title() test_dropped_invalid_preserves_original_title()
test_pending_snapshot_aids() test_pending_snapshot_aids()
test_upsert_snapshot_preserves_view_open()
test_up_follower_snapshot_and_backfill() test_up_follower_snapshot_and_backfill()
print("OK: 收藏夹/视频状态机与黑名单测试全部通过") print("OK: 收藏夹/视频状态机与黑名单测试全部通过")
+1 -103
View File
@@ -8,7 +8,7 @@ import threading
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.stdout.reconfigure(encoding="utf-8") sys.stdout.reconfigure(encoding="utf-8")
from reaction_board.scheduler import ScheduledTask, Scheduler, TaskControl # noqa: E402 from reaction_board.scheduler import ScheduledTask, Scheduler # noqa: E402
def test_run_when_due_and_reschedule(): def test_run_when_due_and_reschedule():
@@ -128,111 +128,9 @@ def test_next_run_fn_controls_reschedule():
assert calls == [1_000.0, 1_123.0] assert calls == [1_000.0, 1_123.0]
def test_task_control_overrides_ok_and_next_run():
clock = [1_000.0]
task = ScheduledTask(
"defer",
60,
lambda: TaskControl(ok=False, next_run=1_500.0, error="quiet"),
)
scheduler = Scheduler([task], clock=lambda: clock[0])
results = scheduler.run_due()
assert len(results) == 1
assert results[0].ok is False
assert results[0].error == "quiet"
assert task.next_run == 1_500.0
# 未到 next_run 不重跑
clock[0] = 1_499.0
assert scheduler.run_due() == []
clock[0] = 1_500.0
task.fn = lambda: TaskControl(ok=True, next_run=2_000.0)
results = scheduler.run_due()
assert results[0].ok is True
assert task.next_run == 2_000.0
class _CaptureLogger:
def __init__(self):
self.events = []
def info(self, event, **fields):
self.events.append(("info", event, fields))
def warning(self, event, **fields):
self.events.append(("warning", event, fields))
def error(self, event, **fields):
self.events.append(("error", event, fields))
def test_task_control_ok_false_logs_fail_not_ok():
"""TaskControl(ok=False) 不得记 scheduler.task_ok(Skip ≠ success)。"""
clock = [1_000.0]
logger = _CaptureLogger()
task = ScheduledTask(
"quiet",
60,
lambda: TaskControl(ok=False, next_run=1_500.0, error="snapshot_quiet_window"),
)
scheduler = Scheduler([task], clock=lambda: clock[0], logger=logger)
results = scheduler.run_due()
assert results[0].ok is False
assert results[0].error == "snapshot_quiet_window"
assert ("info", "scheduler.task_ok", {"task": "quiet"}) not in logger.events
fail_events = [e for e in logger.events if e[1] == "scheduler.task_fail"]
assert len(fail_events) == 1
assert fail_events[0][0] == "warning"
assert fail_events[0][2]["task"] == "quiet"
assert fail_events[0][2]["error"] == "snapshot_quiet_window"
def test_task_control_ok_true_still_logs_task_ok():
clock = [1_000.0]
logger = _CaptureLogger()
task = ScheduledTask(
"ok_task",
60,
lambda: TaskControl(ok=True, next_run=1_200.0),
)
scheduler = Scheduler([task], clock=lambda: clock[0], logger=logger)
results = scheduler.run_due()
assert results[0].ok is True
assert ("info", "scheduler.task_ok", {"task": "ok_task"}) in logger.events
assert not any(e[1] == "scheduler.task_fail" for e in logger.events)
def test_raised_exception_logs_task_fail():
clock = [100.0]
logger = _CaptureLogger()
def boom():
raise RuntimeError("db down")
scheduler = Scheduler(
[ScheduledTask("bad", 10, boom)],
clock=lambda: clock[0],
logger=logger,
)
results = scheduler.run_due()
assert results[0].ok is False
assert results[0].error == "db down"
assert not any(e[1] == "scheduler.task_ok" for e in logger.events)
assert not any(e[1] == "scheduler.task_fail" for e in logger.events)
err_events = [e for e in logger.events if e[1] == "scheduler.task_error"]
assert len(err_events) == 1
assert err_events[0][0] == "error"
assert err_events[0][2]["error"] == "db down"
def main(): def main():
test_run_when_due_and_reschedule() test_run_when_due_and_reschedule()
test_next_run_fn_controls_reschedule() test_next_run_fn_controls_reschedule()
test_task_control_overrides_ok_and_next_run()
test_task_control_ok_false_logs_fail_not_ok()
test_task_control_ok_true_still_logs_task_ok()
test_raised_exception_logs_task_fail()
test_task_error_does_not_block_other_tasks() test_task_error_does_not_block_other_tasks()
test_run_forever_with_stop_event() test_run_forever_with_stop_event()
test_max_iterations_and_validation() test_max_iterations_and_validation()
+3 -156
View File
@@ -444,22 +444,19 @@ def test_run_recent_videos_refreshes_without_completion_markers():
1: view_for(1, view=111), 1: view_for(1, view=111),
2: view_for(2, view=222), 2: view_for(2, view=222),
3: view_for(3, view=333), 3: view_for(3, view=333),
4: view_for(4, view=444),
}) })
db, repo, service = make_service(tmp, transport, track_fans=()) db, repo, service = make_service(tmp, transport, track_fans=())
try: try:
# 1/2 在墙钟近 2 天内;3 为旧片;4 发布已久但当天入库,不进小时任务 # 1/2 在墙钟近 2 天内;3 为旧片
add_video(repo, 1) add_video(repo, 1)
add_video(repo, 2) add_video(repo, 2)
add_video(repo, 3) add_video(repo, 3)
add_video(repo, 4)
repo.db.write(lambda conn: conn.executemany( repo.db.write(lambda conn: conn.executemany(
"UPDATE video SET pubdate = ? WHERE aid = ?", "UPDATE video SET pubdate = ? WHERE aid = ?",
[ [
(clock - 3600, 1), (clock - 3600, 1),
(clock - 86400, 2), (clock - 86400, 2),
(clock - 10 * 86400, 3), (clock - 10 * 86400, 3),
(clock - 10 * 86400, 4),
], ],
)) ))
# 预置同日 ok=1,验证小时任务可覆盖刷新 # 预置同日 ok=1,验证小时任务可覆盖刷新
@@ -480,27 +477,23 @@ def test_run_recent_videos_refreshes_without_completion_markers():
)) == {1, 2} )) == {1, 2}
row = repo.db.read(lambda conn: dict(conn.execute( row = repo.db.read(lambda conn: dict(conn.execute(
"SELECT view, ok, captured_at, view_open FROM snapshot " "SELECT view, ok, captured_at FROM snapshot WHERE stat_date=? AND aid=1",
"WHERE stat_date=? AND aid=1",
(DATE,), (DATE,),
).fetchone())) ).fetchone()))
assert row["view"] == 111 assert row["view"] == 111
assert row["ok"] == 1 assert row["ok"] == 1
assert row["captured_at"] == clock assert row["captured_at"] == clock
assert row["view_open"] == 1
# 完成日 / 备份标记保持原样 # 完成日 / 备份标记保持原样
assert repo.kv_get(service.completed_key(DATE)) is None assert repo.kv_get(service.completed_key(DATE)) is None
assert repo.kv_get("snapshot:last_completed") is None assert repo.kv_get("snapshot:last_completed") is None
assert repo.kv_get(service.backup_key(DATE)) is None assert repo.kv_get(service.backup_key(DATE)) is None
assert transport.count("/web-interface/view") == 2 assert transport.count("/web-interface/view") == 2
assert repo.get_snapshot(DATE, 4) is None
# 全量任务随后才写完成日;且已 ok=1 的跳过 # 全量任务随后才写完成日;且已 ok=1 的跳过
full = service.run_videos(date=DATE) full = service.run_videos(date=DATE)
assert full.skipped >= 2 # 1、2 已被小时任务写过 assert full.skipped >= 2 # 1、2 已被小时任务写过
assert full.success >= 1 # 至少旧片 3、4 assert full.success >= 1 # 至少旧片 3
assert repo.get_snapshot(DATE, 4) is not None
assert repo.kv_get(service.completed_key(DATE)) == "1" assert repo.kv_get(service.completed_key(DATE)) == "1"
assert repo.kv_get("snapshot:last_completed") == DATE assert repo.kv_get("snapshot:last_completed") == DATE
finally: finally:
@@ -537,36 +530,6 @@ def test_full_snapshot_skips_ok_rows_including_hourly_refresh():
def test_seed_same_day_completed_is_already_completed_not_no_source():
"""全站完成后 last_completed==target:应 already_completed,不误判 no_source。"""
clock = 1_800_000_000
with tempfile.TemporaryDirectory() as tmp:
transport = SnapshotTransport({1: view_for(1)})
db, repo, service = make_service(tmp, transport, track_fans=())
try:
add_video(repo, 1)
today = "2026-10-31"
repo.upsert_snapshot(
today, 1,
metrics={"view": 10, "like": 1, "coin": 0, "favorite": 0,
"share": 0, "reply": 0, "danmaku": 0},
ok=True, err=None, captured_at=clock,
)
# 正常收尾态:last_completed 已推进到当日,且 completed 标记在位
repo.kv_set("snapshot:last_completed", today)
repo.kv_set(service.completed_key(today), "1")
summary = service.run_seed_from_last_completed(date=today)
assert summary.reason == "already_completed"
assert summary.already_completed is True
assert summary.completed is True
assert summary.reason != "no_source"
assert transport.count("/web-interface/view") == 0
finally:
db.close()
def test_seed_from_last_completed_does_not_mark_completed(): def test_seed_from_last_completed_does_not_mark_completed():
clock = 1_800_000_000 clock = 1_800_000_000
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
@@ -599,7 +562,6 @@ def test_seed_from_last_completed_does_not_mark_completed():
row = repo.get_snapshot(today, 1) row = repo.get_snapshot(today, 1)
assert row is not None assert row is not None
assert int(row["view"]) == 10 assert int(row["view"]) == 10
assert int(row["view_open"]) == 10
assert row["src"] == "seed" assert row["src"] == "seed"
fan = repo.get_up_stat(today, 37754047) fan = repo.get_up_stat(today, 37754047)
assert fan is not None and int(fan["follower"]) == 100 assert fan is not None and int(fan["follower"]) == 100
@@ -661,117 +623,6 @@ def test_seeded_day_skips_without_force_and_seals_with_force():
db.close() db.close()
def test_seal_keep_scraped_skips_view_progress_punches_seed():
"""晚间 seal(keep_scraped):已 view 的 ok=1 作进度跳过,seed 壳仍打穿。"""
clock = 1_800_000_000
with tempfile.TemporaryDirectory() as tmp:
transport = SnapshotTransport({
1: view_for(1, view=111),
2: view_for(2, view=222),
})
db, repo, service = make_service(tmp, transport)
try:
add_video(repo, 1)
add_video(repo, 2)
today = DATE
# 两条 seed 壳
for aid, view in ((1, 10), (2, 20)):
repo.upsert_snapshot(
today, aid,
metrics={"view": view, "like": 1, "coin": 0, "favorite": 0,
"share": 0, "reply": 0, "danmaku": 0},
ok=True, err=None, captured_at=clock, src="seed",
)
# 模拟 seal 中途:aid=1 已真实采集
repo.upsert_snapshot(
today, 1,
metrics={"view": 111, "like": 1, "coin": 0, "favorite": 0,
"share": 0, "reply": 0, "danmaku": 0},
ok=True, err=None, captured_at=clock + 1, src="view",
)
repo.kv_set(service.seeded_key(today), "1")
sealed = service.run_if_needed(date=today, force=True)
assert sealed.completed is True
assert sealed.skipped >= 1 # aid=1 进度跳过
assert sealed.success >= 1 # aid=2 seed 打穿
# 只应请求 aid=2 的 view(aid=1 已是 view 进度)
assert transport.count("/web-interface/view") == 1
assert int(repo.get_snapshot(today, 1)["view"]) == 111
assert repo.get_snapshot(today, 1)["src"] == "view"
assert int(repo.get_snapshot(today, 2)["view"]) == 222
assert repo.get_snapshot(today, 2)["src"] == "view"
finally:
db.close()
def test_force_keep_scraped_false_rescrapes_view_progress():
"""C: CLI --force 路径 keep_scraped=False 须重采已有 view 进度行。"""
clock = 1_800_000_000
with tempfile.TemporaryDirectory() as tmp:
transport = SnapshotTransport({
1: view_for(1, view=999),
2: view_for(2, view=222),
})
db, repo, service = make_service(tmp, transport)
try:
add_video(repo, 1)
add_video(repo, 2)
today = DATE
for aid, view, src in ((1, 111, "view"), (2, 20, "seed")):
repo.upsert_snapshot(
today, aid,
metrics={"view": view, "like": 1, "coin": 0, "favorite": 0,
"share": 0, "reply": 0, "danmaku": 0},
ok=True, err=None, captured_at=clock, src=src,
)
repo.kv_set(service.seeded_key(today), "1")
forced = service.run_if_needed(
date=today, force=True, keep_scraped=False,
)
assert forced.completed is True
# 两条都重采
assert transport.count("/web-interface/view") == 2
assert int(repo.get_snapshot(today, 1)["view"]) == 999
finally:
db.close()
def test_force_default_keep_scraped_skips_view_progress():
"""C: force 默认 keep_scraped=True(seal)跳过已 view 行。"""
clock = 1_800_000_000
with tempfile.TemporaryDirectory() as tmp:
transport = SnapshotTransport({
1: view_for(1, view=999),
2: view_for(2, view=222),
})
db, repo, service = make_service(tmp, transport)
try:
add_video(repo, 1)
add_video(repo, 2)
today = DATE
repo.upsert_snapshot(
today, 1,
metrics={"view": 111, "like": 1, "coin": 0, "favorite": 0,
"share": 0, "reply": 0, "danmaku": 0},
ok=True, err=None, captured_at=clock, src="view",
)
repo.upsert_snapshot(
today, 2,
metrics={"view": 20, "like": 1, "coin": 0, "favorite": 0,
"share": 0, "reply": 0, "danmaku": 0},
ok=True, err=None, captured_at=clock, src="seed",
)
repo.kv_set(service.seeded_key(today), "1")
sealed = service.run_if_needed(date=today, force=True)
assert sealed.completed is True
assert transport.count("/web-interface/view") == 1
assert int(repo.get_snapshot(today, 1)["view"]) == 111
finally:
db.close()
def main(): def main():
test_snapshot_success_resume_and_force() test_snapshot_success_resume_and_force()
test_resume_skips_ok_and_retries_failed() test_resume_skips_ok_and_retries_failed()
@@ -787,12 +638,8 @@ def main():
test_backup_failure_retries_without_api() test_backup_failure_retries_without_api()
test_run_recent_videos_refreshes_without_completion_markers() test_run_recent_videos_refreshes_without_completion_markers()
test_full_snapshot_skips_ok_rows_including_hourly_refresh() test_full_snapshot_skips_ok_rows_including_hourly_refresh()
test_seed_same_day_completed_is_already_completed_not_no_source()
test_seed_from_last_completed_does_not_mark_completed() test_seed_from_last_completed_does_not_mark_completed()
test_seeded_day_skips_without_force_and_seals_with_force() test_seeded_day_skips_without_force_and_seals_with_force()
test_seal_keep_scraped_skips_view_progress_punches_seed()
test_force_keep_scraped_false_rescrapes_view_progress()
test_force_default_keep_scraped_skips_view_progress()
print("OK: 每日快照、断点续跑、失败占位与粉丝采集测试全部通过") print("OK: 每日快照、断点续跑、失败占位与粉丝采集测试全部通过")
-398
View File
@@ -1,398 +0,0 @@
# -*- coding: utf-8 -*-
"""task:state 持久化与启动续跑: uv run python tests/test_task_state.py"""
import json
import os
import sys
import tempfile
from datetime import datetime
from pathlib import Path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.stdout.reconfigure(encoding="utf-8")
from reaction_board.database import Database # noqa: E402
from reaction_board.repository import Repository # noqa: E402
from reaction_board.scheduler import ScheduledTask, Scheduler # noqa: E402
from reaction_board.task_state import ( # noqa: E402
STATUS_DEFERRED,
STATUS_DONE,
STATUS_FAILED,
STATUS_PENDING,
STATUS_RUNNING,
bootstrap_from_last,
ensure_pending,
load_state,
mark_deferred,
mark_done,
mark_failed,
mark_running,
recover_stale_running,
resume_stat_date,
startup_next_run,
)
from reaction_board.timeutil import SHANGHAI_TZ, next_daily_run # noqa: E402
def _repo(tmp: str) -> tuple[Database, Repository]:
db = Database(Path(tmp) / "app.db")
db.initialize()
return db, Repository(db)
def test_restart_mid_run_resumes_same_stat_date():
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 23, 40, tzinfo=SHANGHAI_TZ).timestamp()
mark_running(
repo, "snapshot", stat_date="2026-11-03", now=now,
progress={"done": 10, "total": 100},
)
# 模拟进程崩溃:running 残留
recovered = recover_stale_running(repo, "snapshot", now=now + 60)
assert recovered is not None
assert recovered.status == STATUS_PENDING
assert recovered.stat_date == "2026-11-03"
assert recovered.progress == {"done": 10, "total": 100}
assert recovered.due_at <= now + 60
assert resume_stat_date(repo, "snapshot", fallback="2026-11-04") == "2026-11-03"
nxt = startup_next_run(
repo, "snapshot",
wall_due_at=next_daily_run(now + 60, "23:15"),
now=now + 60,
)
assert nxt <= now + 60
finally:
db.close()
def test_seeded_before_snapshot_at_does_not_force_run():
"""seed 后挂 pending;白天重启认落盘 due_at(刻意 ≠ 墙钟)且未到不跑。"""
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 10, 30, tzinfo=SHANGHAI_TZ).timestamp()
# 刻意偏离墙钟 23:15,证明认落盘而非 fallback
slot = datetime(2026, 11, 3, 22, 0, tzinfo=SHANGHAI_TZ).timestamp()
wall = next_daily_run(now, "23:15")
assert slot != wall
ensure_pending(
repo, "snapshot", due_at=slot, stat_date="2026-11-03", now=now,
)
repo.kv_set("snapshot:seeded:2026-11-03", "1")
nxt = startup_next_run(
repo, "snapshot",
wall_due_at=wall,
now=now,
)
assert nxt == slot
assert nxt != wall
assert nxt > now
calls = []
def run_snapshot():
calls.append("ran")
return None
clock = [now]
sched = Scheduler(
[ScheduledTask("snapshot", 86400.0, run_snapshot, next_run=nxt)],
clock=lambda: clock[0],
)
assert sched.run_due() == []
assert calls == []
finally:
db.close()
def test_failed_future_due_at_does_not_run_early():
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 12, 0, tzinfo=SHANGHAI_TZ).timestamp()
due = now + 600
mark_failed(
repo, "snapshot", due_at=due, now=now,
stat_date="2026-11-03", error="boom",
)
nxt = startup_next_run(
repo, "snapshot",
wall_due_at=next_daily_run(now, "23:15"),
now=now,
)
assert nxt == due
clock = [now]
calls = []
sched = Scheduler(
[
ScheduledTask(
"snapshot", 86400.0, lambda: calls.append(1), next_run=nxt,
)
],
clock=lambda: clock[0],
)
assert sched.run_due() == []
assert calls == []
finally:
db.close()
def test_due_at_passed_runs_including_failed_retry():
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 12, 10, tzinfo=SHANGHAI_TZ).timestamp()
due = now - 30 # 已过重试点
mark_failed(
repo, "snapshot", due_at=due, now=due - 300,
stat_date="2026-11-03", error="boom",
)
nxt = startup_next_run(
repo, "snapshot",
wall_due_at=next_daily_run(now, "23:15"),
now=now,
)
# 不得退回今晚墙钟;应立刻可跑
assert nxt == due
assert nxt <= now
wall = next_daily_run(now, "23:15")
assert nxt != wall
calls = []
clock = [now]
sched = Scheduler(
[
ScheduledTask(
"snapshot", 86400.0, lambda: calls.append(1), next_run=nxt,
)
],
clock=lambda: clock[0],
)
results = sched.run_due()
assert len(results) == 1
assert calls == [1]
finally:
db.close()
def test_done_writes_next_due_and_blocks_until_then():
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 23, 40, tzinfo=SHANGHAI_TZ).timestamp()
next_due = next_daily_run(now, "23:15")
mark_done(repo, "snapshot", due_at=next_due, now=now, stat_date="2026-11-03")
state = load_state(repo, "snapshot")
assert state is not None
assert state.status == STATUS_DONE
nxt = startup_next_run(
repo, "snapshot", wall_due_at=next_due, now=now + 60,
)
assert nxt == next_due
assert nxt > now + 60
finally:
db.close()
def test_no_state_falls_back_to_wall_clock():
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 10, 0, tzinfo=SHANGHAI_TZ).timestamp()
wall = next_daily_run(now, "23:15")
nxt = startup_next_run(repo, "snapshot", wall_due_at=wall, now=now)
assert nxt == wall
finally:
db.close()
def test_fail_or_defer_preserves_progress():
"""H7: mark_failed / mark_deferred 不传 progress 时保留已有 progress 与 stat_date。"""
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 12, 0, tzinfo=SHANGHAI_TZ).timestamp()
progress = {"done": 42, "total": 100, "cursor": "aid:99"}
mark_running(
repo, "snapshot", stat_date="2026-11-03", now=now,
progress=progress,
)
before = load_state(repo, "snapshot")
assert before is not None
assert before.progress == progress
assert before.stat_date == "2026-11-03"
due_fail = now + 300
mark_failed(
repo, "snapshot", due_at=due_fail, now=now + 1,
progress=None, error="boom",
)
failed = load_state(repo, "snapshot")
assert failed is not None
assert failed.status == STATUS_FAILED
assert failed.due_at == due_fail
assert failed.progress == progress
assert failed.stat_date == "2026-11-03"
assert failed.error == "boom"
due_defer = now + 600
mark_deferred(
repo, "snapshot", due_at=due_defer, now=now + 2,
progress=None, error="snapshot_quiet_window",
)
deferred = load_state(repo, "snapshot")
assert deferred is not None
assert deferred.status == STATUS_DEFERRED
assert deferred.due_at == due_defer
assert deferred.progress == progress
assert deferred.stat_date == "2026-11-03"
assert deferred.error == "snapshot_quiet_window"
finally:
db.close()
def test_bootstrap_from_last_fail_due_soon():
"""A: 仅有 task:last 失败、无 state → 回填 failed 且 due_at 可立刻补跑。"""
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 12, 0, tzinfo=SHANGHAI_TZ).timestamp()
wall = next_daily_run(now, "23:15")
repo.kv_set(
"task:last:snapshot",
json.dumps({
"name": "snapshot",
"ok": False,
"skipped": False,
"started_at": int(now - 600),
"finished_at": int(now - 500),
"error": "boom",
"summary": {"stat_date": "2026-11-03"},
}, ensure_ascii=False),
)
assert load_state(repo, "snapshot") is None
boot = bootstrap_from_last(
repo, "snapshot", wall_due_at=wall, now=now, retry_seconds=300.0,
)
assert boot is not None
assert boot.status == STATUS_FAILED
assert boot.stat_date == "2026-11-03"
assert boot.due_at <= now
nxt = startup_next_run(
repo, "snapshot", wall_due_at=wall, now=now, slot_hhmm="23:15",
)
assert nxt <= now
assert nxt != wall
finally:
db.close()
def test_ensure_pending_does_not_glue_yesterday_stat_date():
"""B: 昨天未完成保留;不得把今晚 due 粘到昨天的 stat_date 上。"""
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 0, 10, tzinfo=SHANGHAI_TZ).timestamp()
yday_due = now - 60
ensure_pending(
repo, "snapshot", due_at=yday_due,
stat_date="2026-11-02", now=now - 120,
)
tonight = datetime(2026, 11, 3, 23, 15, tzinfo=SHANGHAI_TZ).timestamp()
# seed 想挂今晚:不得冲掉昨天未完成
kept = ensure_pending(
repo, "snapshot", due_at=tonight,
stat_date="2026-11-03", now=now,
)
assert kept.stat_date == "2026-11-02"
assert kept.due_at == yday_due
assert resume_stat_date(
repo, "snapshot", fallback="2026-11-03",
) == "2026-11-02"
finally:
db.close()
def test_cross_midnight_mid_seal_resumes_yesterday():
"""规则1 / B: 23:15 崩溃 → 00:10 重启续跑昨天,due 立刻可跑。"""
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
crash = datetime(2026, 11, 2, 23, 40, tzinfo=SHANGHAI_TZ).timestamp()
mark_running(
repo, "snapshot", stat_date="2026-11-02", now=crash,
progress={"done": 3, "total": 10},
)
repo.kv_set("snapshot:seeded:2026-11-02", "1")
restart = datetime(2026, 11, 3, 0, 10, tzinfo=SHANGHAI_TZ).timestamp()
wall = next_daily_run(restart, "23:15")
nxt = startup_next_run(
repo, "snapshot", wall_due_at=wall, now=restart, slot_hhmm="23:15",
)
assert nxt <= restart
assert resume_stat_date(
repo, "snapshot", fallback="2026-11-03",
) == "2026-11-02"
state = load_state(repo, "snapshot")
assert state is not None
assert state.status == STATUS_PENDING
assert state.progress == {"done": 3, "total": 10}
finally:
db.close()
def test_scheduler_due_reader_picks_up_enqueued_seal():
"""D: 同进程 seed 写盘 due_at=now 后,下一轮 run_due 须跑 seal。"""
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 23, 20, tzinfo=SHANGHAI_TZ).timestamp()
# snapshot 原 next_run 挂次日;盘上被 enqueue 成 now
tomorrow = next_daily_run(now, "23:15")
calls = []
def run_snap():
calls.append("snap")
return None
def due_reader(name: str):
st = load_state(repo, name)
return float(st.due_at) if st and st.due_at else None
clock = [now]
sched = Scheduler(
[ScheduledTask("snapshot", 86400.0, run_snap, next_run=tomorrow)],
clock=lambda: clock[0],
due_reader=due_reader,
)
assert sched.run_due() == []
assert calls == []
ensure_pending(
repo, "snapshot", due_at=now, stat_date="2026-11-03", now=now,
)
results = sched.run_due()
assert len(results) == 1
assert calls == ["snap"]
finally:
db.close()
def main_test():
test_restart_mid_run_resumes_same_stat_date()
test_seeded_before_snapshot_at_does_not_force_run()
test_failed_future_due_at_does_not_run_early()
test_due_at_passed_runs_including_failed_retry()
test_done_writes_next_due_and_blocks_until_then()
test_no_state_falls_back_to_wall_clock()
test_fail_or_defer_preserves_progress()
test_bootstrap_from_last_fail_due_soon()
test_ensure_pending_does_not_glue_yesterday_stat_date()
test_cross_midnight_mid_seal_resumes_yesterday()
test_scheduler_due_reader_picks_up_enqueued_seal()
print("OK: task_state persistence / resume / due_at gates")
if __name__ == "__main__":
main_test()
-32
View File
@@ -11,10 +11,7 @@ sys.stdout.reconfigure(encoding="utf-8")
from reaction_board.timeutil import ( # noqa: E402 from reaction_board.timeutil import ( # noqa: E402
SHANGHAI_TZ, SHANGHAI_TZ,
day_bounds, day_bounds,
in_snapshot_quiet_window,
interval_next_run,
is_date, is_date,
next_after_snapshot_quiet_window,
next_daily_run, next_daily_run,
parse_hhmm, parse_hhmm,
shanghai_date, shanghai_date,
@@ -75,40 +72,11 @@ def test_day_bounds_and_date_validation():
raise AssertionError("非法日期应失败") raise AssertionError("非法日期应失败")
def test_snapshot_quiet_window():
assert in_snapshot_quiet_window(ts("2026-11-02 23:00:00"))
assert in_snapshot_quiet_window(ts("2026-11-02 23:59:59"))
assert in_snapshot_quiet_window(ts("2026-11-03 00:00:00"))
assert in_snapshot_quiet_window(ts("2026-11-03 00:59:59"))
assert not in_snapshot_quiet_window(ts("2026-11-03 01:00:00"))
assert not in_snapshot_quiet_window(ts("2026-11-02 22:59:59"))
assert not in_snapshot_quiet_window(ts("2026-11-02 12:00:00"))
assert next_after_snapshot_quiet_window(ts("2026-11-02 22:00:00")) == ts(
"2026-11-02 22:00:00"
)
assert next_after_snapshot_quiet_window(ts("2026-11-02 23:15:00")) == ts(
"2026-11-03 01:00:00"
)
assert next_after_snapshot_quiet_window(ts("2026-11-03 00:30:00")) == ts(
"2026-11-03 01:00:00"
)
# 22:50 + 60min = 23:50 → 推到次日 01:00
assert interval_next_run(ts("2026-11-02 22:50:00"), 3600) == ts(
"2026-11-03 01:00:00"
)
assert interval_next_run(ts("2026-11-02 12:00:00"), 3600) == ts(
"2026-11-02 13:00:00"
)
def main(): def main():
test_shanghai_date_boundary() test_shanghai_date_boundary()
test_parse_hhmm() test_parse_hhmm()
test_next_daily_run() test_next_daily_run()
test_day_bounds_and_date_validation() test_day_bounds_and_date_validation()
test_snapshot_quiet_window()
print("OK: 上海时区与每日时间计算测试全部通过") print("OK: 上海时区与每日时间计算测试全部通过")
+10 -282
View File
@@ -15,7 +15,7 @@ from reaction_board.repository import Repository # noqa: E402
from reaction_board.timeutil import day_bounds # noqa: E402 from reaction_board.timeutil import day_bounds # noqa: E402
from reaction_board.web.cache import DASHBOARD_CACHE # noqa: E402 from reaction_board.web.cache import DASHBOARD_CACHE # noqa: E402
from reaction_board.web.queries import DashboardQueries # noqa: E402 from reaction_board.web.queries import DashboardQueries # noqa: E402
from tests.web_fixtures import DATES, FOLDER, build_db, resource, write_config # noqa: E402 from tests.web_fixtures import DATES, build_db, write_config # noqa: E402
FIXED_NOW = 1_793_459_000 # 2026-10-31 前后 FIXED_NOW = 1_793_459_000 # 2026-10-31 前后
@@ -46,7 +46,7 @@ def test_overview():
assert overview["counts"]["candidates_pending"] == 1 assert overview["counts"]["candidates_pending"] == 1
assert overview["totals"]["view"] == 1200 + 5200 + 120 assert overview["totals"]["view"] == 1200 + 5200 + 120
assert overview["interactions"] > 0 assert overview["interactions"] > 0
assert overview["view_delta"] == (100 + 100 + 120) # 101/102 跨日 + 103 当日发布 assert overview["view_delta"] == (100 + 100 + 0) # 101/102 的日增量
assert overview["snapshot_health"]["total"] == 3 assert overview["snapshot_health"]["total"] == 3
assert overview["folders"][0]["media_count"] == 3 assert overview["folders"][0]["media_count"] == 3
assert overview["folders"][0]["ratio"] < 1 assert overview["folders"][0]["ratio"] < 1
@@ -484,7 +484,7 @@ def test_about_payload():
assert about["snapshot_days"] == 3 assert about["snapshot_days"] == 3
assert about["snapshot_first_date"] == DATES[0] assert about["snapshot_first_date"] == DATES[0]
assert about["fans_start_date"] == DATES[0] assert about["fans_start_date"] == DATES[0]
assert about["schema_version"] == 8 assert about["schema_version"] == 7
assert about["analytics"] == { assert about["analytics"] == {
"min_duration_seconds": 600, "min_duration_seconds": 600,
"maturity_days": 2, "maturity_days": 2,
@@ -492,7 +492,6 @@ def test_about_payload():
"video_shrink_minutes": 60, "video_shrink_minutes": 60,
"shrink_minutes": None, "shrink_minutes": None,
"home_growth_days": "auto", "home_growth_days": "auto",
"growth_rate_prior": 1000,
} }
assert about["heat_weights"]["view"] == 0.30 assert about["heat_weights"]["view"] == 0.30
assert about["followers"]["videos_total"] == 3 assert about["followers"]["videos_total"] == 3
@@ -578,13 +577,9 @@ def test_growth_auto_falls_back_to_one_day():
assert auto["available"] is True assert auto["available"] is True
assert auto["growth_days"] == 1 assert auto["growth_days"] == 1
assert auto["days"] == 1 assert auto["days"] == 1
assert {item["aid"] for item in auto["items"]} == {101, 102, 103} assert {item["aid"] for item in auto["items"]} == {101, 102}
first = next(item for item in auto["items"] if item["aid"] == 101) first = next(item for item in auto["items"] if item["aid"] == 101)
assert first["growth"]["delta"] == 100 # 相邻日 assert first["growth"]["delta"] == 100 # 相邻日
assert first["growth"]["baseline"] == "snapshot"
today = next(item for item in auto["items"] if item["aid"] == 103)
assert today["growth"]["baseline"] == "zero"
assert today["growth"]["delta"] == 120
forced7 = q.list_videos(sort="growth", growth_days=7) forced7 = q.list_videos(sort="growth", growth_days=7)
assert forced7["available"] is False assert forced7["available"] is False
@@ -610,15 +605,12 @@ def test_growth_auto_falls_back_to_one_day():
def test_snapshot_recent_next_run_follows_completion(): def test_snapshot_recent_next_run_follows_completion():
import json import json
from datetime import datetime
from reaction_board.timeutil import SHANGHAI_TZ, interval_next_run
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp) db, q = setup(tmp)
try: try:
repo = Repository(db) repo = Repository(db)
# 午后完成:间隔后仍在静默窗外,应严格 finished+3600 finished = FIXED_NOW - 10
finished = int(datetime(2026, 10, 31, 15, 0, 0, tzinfo=SHANGHAI_TZ).timestamp())
repo.kv_set( repo.kv_set(
"task:last:snapshot_recent", "task:last:snapshot_recent",
json.dumps({ json.dumps({
@@ -630,128 +622,6 @@ def test_snapshot_recent_next_run_follows_completion():
DASHBOARD_CACHE.clear() DASHBOARD_CACHE.clear()
system = q.system() system = q.system()
assert system["next_runs"]["snapshot_recent"] == finished + 3600 assert system["next_runs"]["snapshot_recent"] == finished + 3600
assert system["next_runs"]["snapshot_recent"] == int(
interval_next_run(float(finished), 3600)
)
finally:
db.close()
def test_snapshot_recent_next_run_defers_quiet_window_and_retry_on_skip():
import json
from datetime import datetime
from reaction_board.timeutil import SHANGHAI_TZ, next_after_snapshot_quiet_window
with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp)
try:
repo = Repository(db)
# 22:50 成功 → 下一档 23:50 落入静默窗 → 推到 01:00
finished = int(datetime(2026, 10, 31, 22, 50, 0, tzinfo=SHANGHAI_TZ).timestamp())
repo.kv_set(
"task:last:snapshot_recent",
json.dumps({
"name": "snapshot_recent", "ok": True,
"started_at": finished - 5, "finished_at": finished,
"error": None, "summary": {},
}),
)
DASHBOARD_CACHE.clear()
system = q.system()
expected = int(datetime(2026, 11, 1, 1, 0, 0, tzinfo=SHANGHAI_TZ).timestamp())
assert system["next_runs"]["snapshot_recent"] == expected
# skip 记录:补跑到窗后 01:00,不得当成功推进
quiet = int(datetime(2026, 10, 31, 23, 20, 0, tzinfo=SHANGHAI_TZ).timestamp())
repo.kv_set(
"task:last:snapshot_recent",
json.dumps({
"name": "snapshot_recent", "ok": False, "skipped": True,
"started_at": quiet, "finished_at": quiet,
"error": "snapshot_quiet_window",
"summary": {"skipped": True, "reason": "snapshot_quiet_window"},
}),
)
DASHBOARD_CACHE.clear()
system = q.system()
assert system["next_runs"]["snapshot_recent"] == int(
next_after_snapshot_quiet_window(float(quiet))
)
assert not any(e["task"] == "snapshot_recent" for e in system["errors"])
finally:
db.close()
def test_snapshot_failed_next_run_retries_soon_not_next_day():
import json
from datetime import datetime
from reaction_board.timeutil import SHANGHAI_TZ, next_daily_run
with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp)
try:
repo = Repository(db)
finished = int(datetime(2026, 10, 31, 23, 15, 0, tzinfo=SHANGHAI_TZ).timestamp())
repo.kv_set(
"task:last:snapshot",
json.dumps({
"name": "snapshot", "ok": False,
"started_at": finished - 1, "finished_at": finished,
"error": "boom", "summary": {"completed": False},
}),
)
DASHBOARD_CACHE.clear()
system = q.system()
assert system["next_runs"]["snapshot"] == finished + 300
assert system["next_runs"]["snapshot"] != int(
next_daily_run(finished, "23:15")
)
assert any(e["task"] == "snapshot" for e in system["errors"])
finally:
db.close()
def test_snapshot_seed_skip_next_run_retries_ten_minutes():
import json
from datetime import datetime
from reaction_board.timeutil import SHANGHAI_TZ, next_daily_run
with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp)
try:
repo = Repository(db)
finished = int(datetime(2026, 11, 1, 0, 0, 0, tzinfo=SHANGHAI_TZ).timestamp())
repo.kv_set(
"task:last:snapshot_seed",
json.dumps({
"name": "snapshot_seed", "ok": False, "skipped": True,
"started_at": finished - 1, "finished_at": finished,
"error": "snapshot_running",
"summary": {"skipped": True, "reason": "snapshot_running"},
}),
)
DASHBOARD_CACHE.clear()
system = q.system()
assert system["next_runs"]["snapshot_seed"] == finished + 600
assert system["next_runs"]["snapshot_seed"] != int(
next_daily_run(finished, "00:00")
)
# skip 不算 errors
assert not any(e["task"] == "snapshot_seed" for e in system["errors"])
# no_source 同样 10 分钟
repo.kv_set(
"task:last:snapshot_seed",
json.dumps({
"name": "snapshot_seed", "ok": False, "skipped": True,
"started_at": finished - 1, "finished_at": finished,
"error": "no_source",
"summary": {"skipped": True, "reason": "no_source"},
}),
)
DASHBOARD_CACHE.clear()
system = q.system()
assert system["next_runs"]["snapshot_seed"] == finished + 600
finally: finally:
db.close() db.close()
@@ -797,10 +667,9 @@ def test_overview_adjacent_previous_with_seed_and_dual_fields():
+ overview["counts"]["unfaved"] + overview["counts"]["unfaved"]
+ overview["counts"]["removed"] + overview["counts"]["removed"]
) )
# 101/102:当前=seed(=完成日数值),相邻 previous=完成日本身 → 增量 0 # 当前=seed(=完成日数值),相邻 previous=完成日本身 → 增量 0
# 103:当日发布,近 1 日相对 0,计入当前播放
assert overview["view_delta_available"] is True assert overview["view_delta_available"] is True
assert overview["view_delta"] == 120 assert overview["view_delta"] == 0
growth = q.list_videos(sort="growth", growth_days=1) growth = q.list_videos(sort="growth", growth_days=1)
assert growth["available"] is True assert growth["available"] is True
@@ -809,141 +678,6 @@ def test_overview_adjacent_previous_with_seed_and_dual_fields():
assert item["growth"]["available"] is True assert item["growth"]["available"] is True
assert item["growth"]["delta"] == 0 assert item["growth"]["delta"] == 0
assert item["growth"]["gap_days"] == 1 assert item["growth"]["gap_days"] == 1
assert item["growth"]["baseline"] == "snapshot"
finally:
db.close()
def _view_metrics(view: int) -> dict[str, int]:
return {
"view": view, "like": 0, "coin": 0, "favorite": 0,
"share": 0, "reply": 0, "danmaku": 0,
}
def test_same_day_growth_baselines():
"""近 1 日:当日发布相对 0;更早发布用 view_open;7 日不用这两条退路。"""
with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp)
try:
repo = Repository(db)
today = DATES[-1]
start = int(day_bounds(today)[0])
old_pub = int(day_bounds("2026-09-01")[0])
repo.upsert_snapshot(
today, 103, metrics=_view_metrics(200),
ok=True, err=None, captured_at=start + 3600,
)
repo.upsert_from_fav(
resource(105, up_mid=1, up_name="UP A", pubdate=old_pub),
FOLDER.media_id, was_added=True, now=start + 10,
)
repo.upsert_snapshot(
today, 105, metrics=_view_metrics(500),
ok=True, err=None, captured_at=start + 10,
)
repo.upsert_snapshot(
today, 105, metrics=_view_metrics(800),
ok=True, err=None, captured_at=start + 20,
)
repo.upsert_from_fav(
resource(106, up_mid=1, up_name="UP A", pubdate=old_pub),
FOLDER.media_id, was_added=True, now=start + 30,
)
repo.upsert_snapshot(
today, 106, metrics=_view_metrics(900),
ok=True, err=None, captured_at=start + 30,
)
repo.upsert_from_fav(
resource(107, up_mid=1, up_name="UP A", pubdate=old_pub),
FOLDER.media_id, was_added=True, now=start + 40,
)
repo.upsert_snapshot(
today, 107, metrics=_view_metrics(50),
ok=True, err=None, captured_at=start + 40,
)
db.write(lambda conn: conn.execute(
"UPDATE video SET pubdate = 0 WHERE aid = 107"
))
DASHBOARD_CACHE.clear()
one = q.list_videos(sort="growth", growth_days=1, page_size=20)
by_aid = {item["aid"]: item for item in one["items"]}
assert 103 in by_aid and 105 in by_aid
assert 106 not in by_aid and 107 not in by_aid
assert by_aid[103]["growth"]["baseline"] == "zero"
assert by_aid[103]["growth"]["delta"] == 200
assert by_aid[103]["growth"]["rate"] is None
assert by_aid[103]["growth"]["gap_days"] == 0
assert by_aid[105]["growth"]["baseline"] == "open"
assert by_aid[105]["growth"]["delta"] == 300
assert by_aid[101]["growth"]["baseline"] == "snapshot"
assert by_aid[101]["growth"]["delta"] == 100
rate = q.list_videos(sort="growth_rate", growth_days=1, page_size=20)
rate_aids = {item["aid"] for item in rate["items"]}
assert 103 not in rate_aids and 105 not in rate_aids
assert 101 in rate_aids
seven = q.list_videos(sort="growth", growth_days=7)
seven_aids = {item["aid"] for item in seven["items"]}
assert 103 not in seven_aids
assert 105 not in seven_aids
detail103 = q.video_detail(103)
assert detail103["growth"]["baseline"] == "zero"
assert detail103["growth"]["delta"] == 200
detail105 = q.video_detail(105)
assert detail105["growth"]["baseline"] == "open"
assert detail105["growth"]["delta"] == 300
assert q.video_detail(106)["growth"]["available"] is False
assert q.video_detail(107)["growth"]["available"] is False
overview = q.overview()
assert overview["view_delta"] == 100 + 100 + 200 + 300
finally:
db.close()
def test_growth_rate_sort_uses_prior():
"""基线不足 1000 不进增速榜;达标后按基线 + prior 排序,不按 raw rate。"""
with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp)
try:
repo = Repository(db)
latest = DATES[-1]
previous_day = DATES[-2]
start = int(day_bounds(latest)[0])
pub = int(day_bounds("2026-09-01")[0])
# 301:80→512,真实 +540%,基线低于 1000,只留在增量榜。
# 304:1000→2500,真实增速高于 302,平滑后应排在 302 后面。
for aid, views in ((301, (80, 512)), (302, (5000, 10000)), (304, (1000, 2500))):
repo.upsert_from_fav(
resource(aid, up_mid=1, up_name="UP A", pubdate=pub),
FOLDER.media_id, was_added=True, now=start,
)
repo.upsert_snapshot(
previous_day, aid, metrics=_view_metrics(views[0]),
ok=True, err=None, captured_at=start - 86400,
)
repo.upsert_snapshot(
latest, aid, metrics=_view_metrics(views[1]),
ok=True, err=None, captured_at=start + 60,
)
DASHBOARD_CACHE.clear()
ranked = q.list_videos(sort="growth_rate", growth_days=1, page_size=20)
order = [item["aid"] for item in ranked["items"]]
assert ranked["growth_rate_prior"] == 1000
assert 301 not in order
assert order.index(302) < order.index(304)
modest = next(item for item in ranked["items"] if item["aid"] == 304)
assert abs(modest["growth"]["rate"] - 1500 / 1000) < 1e-12
growth = q.list_videos(sort="growth", growth_days=1, page_size=20)
grown = next(item for item in growth["items"] if item["aid"] == 301)
assert grown["growth"]["delta"] == 432
assert abs(grown["growth"]["rate"] - 432 / 80) < 1e-12
finally: finally:
db.close() db.close()
@@ -1007,12 +741,7 @@ def main_test():
test_system_tasks_sorted_by_started_at_desc() test_system_tasks_sorted_by_started_at_desc()
test_growth_auto_falls_back_to_one_day() test_growth_auto_falls_back_to_one_day()
test_snapshot_recent_next_run_follows_completion() test_snapshot_recent_next_run_follows_completion()
test_snapshot_recent_next_run_defers_quiet_window_and_retry_on_skip()
test_snapshot_failed_next_run_retries_soon_not_next_day()
test_snapshot_seed_skip_next_run_retries_ten_minutes()
test_overview_adjacent_previous_with_seed_and_dual_fields() test_overview_adjacent_previous_with_seed_and_dual_fields()
test_same_day_growth_baselines()
test_growth_rate_sort_uses_prior()
print("OK: Web 查询层总览/榜单/详情/UP/时长效率/粉丝/候选/系统/访问统计测试全部通过") print("OK: Web 查询层总览/榜单/详情/UP/时长效率/粉丝/候选/系统/访问统计测试全部通过")
@@ -1076,15 +805,13 @@ def test_system_status_uses_completed_date_and_running_flags():
def test_system_next_runs_follow_last_completion_not_wall_clock(): def test_system_next_runs_follow_last_completion_not_wall_clock():
import json import json
from datetime import datetime
from reaction_board.timeutil import SHANGHAI_TZ
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
db, q = setup(tmp) db, q = setup(tmp)
try: try:
repo = Repository(db) repo = Repository(db)
# 午后完成:下次 = finished + 60min,不跳档、不误入静默窗推算 # 发现任务刚完成不久:下次 = finished + 60min,即使 now 已超过也不跳档
finished = int(datetime(2026, 10, 31, 15, 0, 0, tzinfo=SHANGHAI_TZ).timestamp()) finished = FIXED_NOW - 30
repo.kv_set( repo.kv_set(
"task:last:discovery", "task:last:discovery",
json.dumps({ json.dumps({
@@ -1096,6 +823,7 @@ def test_system_next_runs_follow_last_completion_not_wall_clock():
DASHBOARD_CACHE.clear() DASHBOARD_CACHE.clear()
system = q.system() system = q.system()
assert system["next_runs"]["discover"] == finished + 3600 assert system["next_runs"]["discover"] == finished + 3600
# 仍小于/等于 now 也可以(表示本轮后的下一档已到期或未到),但绝不能跳到 +7200
assert system["next_runs"]["discover"] != finished + 7200 assert system["next_runs"]["discover"] != finished + 7200
finally: finally:
db.close() db.close()
+1 -1
View File
@@ -75,7 +75,7 @@ def test_readonly_routes_and_static():
assert status == 200 and payload["ok"] is True assert status == 200 and payload["ok"] is True
status, _, payload = running.json("/api/about") status, _, payload = running.json("/api/about")
assert status == 200 assert status == 200
assert payload["data"]["schema_version"] == 8 assert payload["data"]["schema_version"] == 7
assert payload["data"]["analytics"]["min_duration_seconds"] == 600 assert payload["data"]["analytics"]["min_duration_seconds"] == 600
status, _, overview = running.json("/api/overview") status, _, overview = running.json("/api/overview")
+1 -1
View File
@@ -138,7 +138,7 @@ def build_db(tmp: str | Path) -> Path:
day, 37754047, follower=1000 + index * 10, following=5, day, 37754047, follower=1000 + index * 10, following=5,
ok=True, captured_at=ts(day) + 60, ok=True, captured_at=ts(day) + 60,
) )
# 103 只有最后一天快照(发布日=快照日,近 1 日相对 0);104 另加一条失败快照 # 103 只有最后一天快照,用于验证样本不足;104 另加一条失败快照
repo.upsert_snapshot( repo.upsert_snapshot(
DATES[-1], 103, DATES[-1], 103,
metrics={"view": 120, "like": 5, "coin": 1, "favorite": 2, "share": 0, "reply": 1, "danmaku": 1}, metrics={"view": 120, "like": 5, "coin": 1, "favorite": 2, "share": 0, "reply": 1, "danmaku": 1},
+2 -4
View File
@@ -237,8 +237,8 @@ def _insert_snapshot(
""" """
INSERT OR REPLACE INTO snapshot( INSERT OR REPLACE INTO snapshot(
stat_date, aid, "view", "like", favorite, coin, share, reply, stat_date, aid, "view", "like", favorite, coin, share, reply,
danmaku, captured_at, ok, err, src, view_open, open_captured_at danmaku, captured_at, ok, err, src
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'preview', ?, ?) ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'preview')
""", """,
( (
stat_date, stat_date,
@@ -253,8 +253,6 @@ def _insert_snapshot(
int(captured_at), int(captured_at),
1 if ok else 0, 1 if ok else 0,
err, err,
values.get("view") if ok else None,
int(captured_at) if ok else None,
), ),
) )