From 8a9cb214080303c51e5e479cc5ded66661d6a2d6 Mon Sep 17 00:00:00 2001 From: bot-ci Date: Mon, 2 Nov 2026 12:38:55 +0800 Subject: [PATCH] feat: EOD 23:15 seal + midnight seed shell for video/fans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move daily snapshot/fans to 23:15 and add 00:01 seed that clones last_completed ok=1 rows into today without advancing completed. Evening runs force through seed markers; growth previous is per-aid adjacent. Home shows short-update clock + 全站数据 date, and 收录视频 counts. --- DEVELOPMENT.md | 11 +- README.md | 2 +- config.docker.example.json | 3 +- config.example.json | 3 +- project.md | 12 +-- reaction_board/cli.py | 44 +++++++- reaction_board/config.py | 10 +- reaction_board/repository.py | 79 +++++++++++++- reaction_board/services/snapshot.py | 125 +++++++++++++++++++++- reaction_board/web/queries.py | 76 +++++++++++-- reaction_board/web/static/format.js | 16 +++ reaction_board/web/static/views/home.js | 15 ++- reaction_board/web/static/views/system.js | 1 + tests/test_config.py | 6 +- tests/test_snapshot.py | 97 ++++++++++++++++- tests/test_web_queries.py | 56 ++++++++++ tests/web_fixtures.py | 3 +- 17 files changed, 520 insertions(+), 39 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 8e6549e..9af139e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -294,7 +294,8 @@ PRAGMA synchronous = NORMAL; "up_blacklist_seed": [261052419, 1908537643, 1701121088, 542262], "schedule": { "discover_minutes": 60, - "snapshot_at": "00:01", + "snapshot_at": "23:15", + "seed_at": "00:01", "candidate_ttl_days": 7 }, "notify": { "min_score": 2, "max_items_per_message": 10 } @@ -308,7 +309,7 @@ PRAGMA synchronous = NORMAL; ### 5.2 安全规则- Windows 本地开发时应限制 `config.json` 的文件访问权限；部署到 Linux 时权限必须为 `600`； - 配置文件每 60 秒检查一次 mtime，凭据、分类规则和权重变更后无需重启； - Cookie / CSRF 变化时由 `ClientProvider` 重建 B 站请求客户端，并保留该角色的限速与重试策略； -- `schedule.discover_minutes` 与 `schedule.snapshot_at` 在启动时固化，修改后需重启 `run`； +- `schedule.discover_minutes`、`schedule.snapshot_at` 与 `schedule.seed_at` 在启动时固化，修改后需重启 `run`； - Telegram 操作必须同时验证 `from.id` 和 `chat.id` 是否为配置的私聊用户； - 仅允许环境变量临时覆盖敏感配置，不能将真实凭据硬编码进源代码； - `GET` 路由绝不进行入库、忽略、拉黑等写操作。 @@ -434,7 +435,7 @@ P3 实现说明： ### 7.4 每日快照与粉丝采集 -每天 `00:01`（`Asia/Shanghai`）执行： +每天 `23:15`（`Asia/Shanghai`）执行全量落盘；`00:01` 用上一完成日克隆当日壳（seed）： - 对 `status IN ('active', 'unfaved')` 的视频逐条请求 `view?aid=`； - 成功写入播放、点赞、收藏、投币、转发、评论、弹幕七项数据，并把 `view` 的 `owner.face` 写入 `video.up_face`（UP 头像，供 UP 列表/详情展示）； @@ -450,7 +451,9 @@ P3 实现说明： P1 实现说明： - 时区使用固定 UTC+8（中国无夏令时），避免 Windows 缺少 `tzdata`； -- 当日全部视频与粉丝采集成功后在 `kv` 写入 `snapshot:completed:=1`；启动时未完成会补跑； +- 午夜 `seed_at`（默认 `00:01`）把 `snapshot:last_completed` 的 ok=1 行克隆为当日壳，写入 `snapshot:seeded:=1` / `fans:seeded:=1`，**不**推进完成日； +- 晚间 `snapshot_at`（默认 `23:15`）对 seeded 日 force 重采；成功后写 `snapshot:completed:=1`、推进 `last_completed` 并清除 seeded； +- 当日全部视频与粉丝采集成功后在 `kv` 写入 `snapshot:completed:=1`；启动时未完成会补跑（纯 seed 日不误标完成）； - 完成标记不是唯一门槛：每次检查 `pending_snapshot_aids(date)`，新入库或从 `dropped` 恢复的视频会立即补采当天快照； - `ok=0` 的当日记录会在下次补跑时重试；`ok=1` 直接跳过；粉丝同样跳过已成功的 `(stat_date, mid)`； - 开始采集前先写 `snapshot:completed:=0`，强制重采中断后不会残留旧的完成标记； diff --git a/README.md b/README.md index eb872d4..83a58ba 100644 --- a/README.md +++ b/README.md @@ -329,7 +329,7 @@ Copy-Item config.example.json config.json - `web`：绑定地址、端口、访问令牌与 `web.track`（访问统计开关、可信代理网段、上报头顺序、落库周期与保留期）； - `schedule` / `notify` / `classify` / `heat_weights`：任务节奏、通知阈值、分类规则和热度权重。 -配置每 60 秒检查一次 mtime，凭据、分类规则等变更无需重启即可在下次任务读取；但 `schedule.discover_minutes` 与 `schedule.snapshot_at` 在启动时固化，修改后需重启 `run` 进程。 +配置每 60 秒检查一次 mtime，凭据、分类规则等变更无需重启即可在下次任务读取；但 `schedule.discover_minutes`、`schedule.snapshot_at` 与 `schedule.seed_at` 在启动时固化，修改后需重启 `run` 进程。 完整配置口径以 [project.md](project.md) 和 [DEVELOPMENT.md](DEVELOPMENT.md) 为准。 diff --git a/config.docker.example.json b/config.docker.example.json index 2a0eef9..223ddcb 100644 --- a/config.docker.example.json +++ b/config.docker.example.json @@ -63,7 +63,8 @@ ], "schedule": { "discover_minutes": 60, - "snapshot_at": "00:01", + "snapshot_at": "23:15", + "seed_at": "00:01", "candidate_ttl_days": 7, "recent_snapshot_minutes": 60, "recent_video_days": 2 diff --git a/config.example.json b/config.example.json index 6069938..4597c7c 100644 --- a/config.example.json +++ b/config.example.json @@ -73,7 +73,8 @@ ], "schedule": { "discover_minutes": 60, - "snapshot_at": "00:01", + "snapshot_at": "23:15", + "seed_at": "00:01", "candidate_ttl_days": 7, "recent_snapshot_minutes": 60, "recent_video_days": 2 diff --git a/project.md b/project.md index f2ea3c4..da89cbb 100644 --- a/project.md +++ b/project.md @@ -117,7 +117,7 @@ Telegram 私聊发链接/BV 号 ───┘ 145178393, 66389997, 293976264, 3026109 ], "schedule": { "discover_minutes": 60, - "snapshot_at": "00:01", "candidate_ttl_days": 7 }, + "snapshot_at": "23:15", "seed_at": "00:01", "candidate_ttl_days": 7 }, "discover": { "keywords": ["咻咻满"], "pages": 3, "window_days": 7, "candidate_min_score": 4 }, "notify": { "min_score": 4, "max_items_per_message": 10 }, @@ -377,7 +377,7 @@ for folder in 配置的年度夹（2025、2026、…）: ### 5.1 任务 ``` -每天 00:01:00（Asia/Shanghai）： +每天 23:15（Asia/Shanghai）全量落盘；00:01 克隆当日壳： videos = db.where(status='active') # 跨所有年度夹 for v in videos: if exists(snapshot[today, v.aid]) and ok: continue # 断点续跑 @@ -397,13 +397,13 @@ for folder in 配置的年度夹（2025、2026、…）: | 1600（2027 年末） | 1600 | 约 33 分钟 | | 3000（2029 年） | 3000 | 约 60 分钟 | -- 00:01 启动，最晚 01:00 前完成，不影响白天使用； +- 23:15 启动全量落盘；00:01 仅本地克隆壳，不影响白天使用； - 若单条 0.1s + sleep 1.2s ≈ 1.3s，实际按 1.2s 估算偏保守。 ### 5.3 幂等与口径 - 唯一键 `(stat_date, aid)`，`INSERT ... ON CONFLICT DO UPDATE`； -- `stat_date` 用本地日期，不用 UTC（避免 00:01 落到前一天）； +- `stat_date` 用本地日期，不用 UTC（避免日界落到前一天）； - 失败重试 3 次后写 `ok=0` 占位，**不写假数据**；看板显示"缺失"而不是 0； - 完成标记之外还要检查当天待采集合：新入库/恢复视频立即补采，`ok=0` 项在续跑时重试； - 强制重采开始前先写未完成标记，中断后仍可续跑； @@ -412,7 +412,7 @@ for folder in 配置的年度夹（2025、2026、…）: ### 5.4 粉丝数追踪（新增） ``` -每天 00:01 与视频快照同一个任务里，追加 1 次请求： +每天 23:15 与视频快照同一个调度窗口里，追加粉丝请求： r = GET x/relation/stat?vmid=37754047 # 匿名可用，实测 0.1s upsert(up_stat, today, mid=37754047, follower=r.follower, following=r.following) ``` @@ -782,7 +782,7 @@ config.docker.example.json 服务器配置样例（0.0.0.0 + /data 绝对路径 **进程结构** ``` -scheduler 线程：discovery(每 60min) / snapshot(每日 00:01) / candidate_cleanup(每日) +scheduler 线程：discovery(每 60min) / snapshot_seed(每日 00:01) / snapshot+fans(每日 23:15) / snapshot_recent(小时) / candidate_cleanup(每日) web 线程：ThreadingHTTPServer :8080 telegram：getUpdates long polling（回调 + 私聊文本入库） storage：SQLite WAL，单写线程 diff --git a/reaction_board/cli.py b/reaction_board/cli.py index 9d1761b..83c11d7 100644 --- a/reaction_board/cli.py +++ b/reaction_board/cli.py @@ -566,7 +566,12 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int: return None _mark_task_running(repo, "snapshot", started_at=started) try: - summary = snapshot_service.run_if_needed() + # 晚间全量：若当日是午夜 seed 壳，必须 force 打穿 ok=1 克隆行。 + today = shanghai_date() + force_seed = repo.kv_get( + SnapshotService.seeded_key(today) + ) == "1" + summary = snapshot_service.run_if_needed(force=force_seed) except Exception as exc: # noqa: BLE001 _record_task( repo, "snapshot", ok=False, started_at=started, @@ -621,7 +626,11 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int: started = int(time.time()) _mark_task_running(repo, "fans", started_at=started) try: - summary = snapshot_service.run_fans_if_needed() + today = shanghai_date() + force_seed = repo.kv_get( + SnapshotService.fans_seeded_key(today) + ) == "1" + summary = snapshot_service.run_fans_if_needed(force=force_seed) except Exception as exc: # noqa: BLE001 _record_task( repo, "fans", ok=False, started_at=started, @@ -642,6 +651,27 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int: logger.info("fans.summary", **payload) return summary + def run_snapshot_seed(): + started = int(time.time()) + _mark_task_running(repo, "snapshot_seed", started_at=started) + try: + summary = snapshot_service.run_seed_from_last_completed() + except Exception as exc: # noqa: BLE001 + _record_task( + repo, "snapshot_seed", ok=False, started_at=started, + finished_at=int(time.time()), error=str(exc), + ) + raise + payload = summary.as_dict() + _record_task( + repo, "snapshot_seed", ok=True, started_at=summary.started_at, + finished_at=summary.finished_at or int(time.time()), + payload=payload, + ) + last["snapshot_seed"] = summary + logger.info("snapshot.seed_summary", **payload) + return summary + def run_config_watch(): before = store.revision store.get() @@ -651,9 +681,18 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int: hh, mm = parse_hhmm(cfg.schedule.snapshot_at) snapshot_at = f"{hh:02d}:{mm:02d}" + sh, sm = parse_hhmm(cfg.schedule.seed_at) + seed_at = f"{sh:02d}:{sm:02d}" recent_minutes = int(store.get().schedule.recent_snapshot_minutes) scheduler = Scheduler( [ + ScheduledTask( + "snapshot_seed", + 86400.0, + run_snapshot_seed, + next_run=0.0, + next_run_fn=lambda started: next_daily_run(started, seed_at), + ), ScheduledTask( "snapshot", 86400.0, @@ -707,6 +746,7 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int: logger.info( "scheduler.start", snapshot_at=snapshot_at, + seed_at=seed_at, ) scheduler.run_forever() diff --git a/reaction_board/config.py b/reaction_board/config.py index f7cc615..e71dde7 100644 --- a/reaction_board/config.py +++ b/reaction_board/config.py @@ -83,7 +83,8 @@ DEFAULT_CONFIG: dict[str, Any] = { "up_blacklist_seed": [], "schedule": { "discover_minutes": 60, - "snapshot_at": "00:01", + "snapshot_at": "23:15", + "seed_at": "00:01", "candidate_ttl_days": 7, "recent_snapshot_minutes": 60, "recent_video_days": 2, @@ -179,6 +180,7 @@ class FolderConfig: class ScheduleConfig: discover_minutes: int snapshot_at: str + seed_at: str candidate_ttl_days: int recent_snapshot_minutes: int recent_video_days: int @@ -449,14 +451,18 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig: sched_raw = data["schedule"] if not isinstance(sched_raw, Mapping): raise ConfigError("schedule 必须是对象") - snapshot_at = _as_str(sched_raw.get("snapshot_at", "00:01"), "schedule.snapshot_at") + snapshot_at = _as_str(sched_raw.get("snapshot_at", "23:15"), "schedule.snapshot_at") if not _SNAPSHOT_AT_RE.match(snapshot_at): raise ConfigError("schedule.snapshot_at 必须是 HH:MM（24 小时制）") + seed_at = _as_str(sched_raw.get("seed_at", "00:01"), "schedule.seed_at") + if not _SNAPSHOT_AT_RE.match(seed_at): + raise ConfigError("schedule.seed_at 必须是 HH:MM（24 小时制）") schedule = ScheduleConfig( discover_minutes=_as_int( sched_raw.get("discover_minutes", 60), "schedule.discover_minutes", minimum=1 ), snapshot_at=snapshot_at, + seed_at=seed_at, candidate_ttl_days=_as_int( sched_raw.get("candidate_ttl_days", 7), "schedule.candidate_ttl_days", diff --git a/reaction_board/repository.py b/reaction_board/repository.py index edbaa6f..f0df923 100644 --- a/reaction_board/repository.py +++ b/reaction_board/repository.py @@ -10,7 +10,7 @@ from __future__ import annotations import sqlite3 import time from dataclasses import dataclass -from typing import Any, Iterable, Mapping +from typing import Any, Iterable, Mapping, Sequence from .config import FolderConfig from .database import Database @@ -727,6 +727,83 @@ class Repository: ) self.db.write(run) + def copy_ok_snapshots( + self, + source_date: str, + target_date: str, + *, + captured_at: int, + src: str = "seed", + ) -> int: + """将 source_date 的 ok=1 行复制到 target_date；已有 target 行则跳过。""" + def run(conn: sqlite3.Connection) -> int: + cur = conn.execute( + """ + INSERT INTO snapshot( + stat_date, aid, "view", "like", favorite, coin, + share, reply, danmaku, captured_at, ok, err, src + ) + SELECT + ?, aid, "view", "like", favorite, coin, + share, reply, danmaku, ?, 1, NULL, ? + FROM snapshot + WHERE stat_date = ? AND ok = 1 + AND aid NOT IN ( + SELECT aid FROM snapshot WHERE stat_date = ? + ) + """, + (target_date, int(captured_at), src, source_date, target_date), + ) + return int(cur.rowcount or 0) + return self.db.write(run) + + def copy_ok_up_stats( + self, + source_date: str, + target_date: str, + *, + captured_at: int, + mids: Sequence[int] | None = None, + ) -> int: + """将 source_date 的 ok=1 粉丝行复制到 target_date；已有 target 行则跳过。""" + def run(conn: sqlite3.Connection) -> int: + if mids is not None: + mid_list = [int(m) for m in mids] + if not mid_list: + return 0 + placeholders = ",".join("?" for _ in mid_list) + cur = conn.execute( + f""" + INSERT INTO up_stat( + stat_date, mid, follower, following, captured_at, ok + ) + SELECT ?, mid, follower, following, ?, 1 + FROM up_stat + WHERE stat_date = ? AND ok = 1 AND mid IN ({placeholders}) + AND mid NOT IN ( + SELECT mid FROM up_stat WHERE stat_date = ? + ) + """, + (target_date, int(captured_at), source_date, *mid_list, target_date), + ) + else: + cur = conn.execute( + """ + INSERT INTO up_stat( + stat_date, mid, follower, following, captured_at, ok + ) + SELECT ?, mid, follower, following, ?, 1 + FROM up_stat + WHERE stat_date = ? AND ok = 1 + AND mid NOT IN ( + SELECT mid FROM up_stat WHERE stat_date = ? + ) + """, + (target_date, int(captured_at), source_date, target_date), + ) + return int(cur.rowcount or 0) + return self.db.write(run) + def snapshot_counts(self, stat_date: str) -> dict[str, int]: def run(conn: sqlite3.Connection) -> dict[str, int]: rows = conn.execute( diff --git a/reaction_board/services/snapshot.py b/reaction_board/services/snapshot.py index bb06690..59e40c9 100644 --- a/reaction_board/services/snapshot.py +++ b/reaction_board/services/snapshot.py @@ -5,6 +5,8 @@ * 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``； * 成功写入七项指标；``ok=1`` 的同日快照直接跳过（断点续跑；含小时任务 1 小时内刚刷新的）； +* 午夜 ``seed_at`` 用上一完成日 ok=1 行克隆出当日壳（``snapshot:seeded`` / ``fans:seeded``），不推进完成日； +* 晚间 ``snapshot_at`` 全量必须 force 打穿 seed，成功后才写完成日并清除 seeded； * ``run_recent_videos`` 只刷新近 N 天新片的当日行，不改完成日 / 备份； * 视频不可访问时标记 ``removed``； * 其他错误最多重试 3 次，仍失败写 ``ok=0`` 占位，不写假零值； @@ -129,18 +131,43 @@ class SnapshotService: def backup_error_key(stat_date: str) -> str: return f"snapshot:backup_error:{stat_date}" + @staticmethod + def seeded_key(stat_date: str) -> str: + return f"snapshot:seeded:{stat_date}" + + @staticmethod + def fans_seeded_key(stat_date: str) -> str: + return f"fans:seeded:{stat_date}" + def run_if_needed( self, *, date: str | None = None, force: bool = False ) -> SnapshotSummary: - """视频快照（含 UP 头像补齐）；粉丝采集请用 ``run_fans_if_needed``。""" + """视频快照（含 UP 头像补齐）；粉丝采集请用 ``run_fans_if_needed``。 + + 当日若仅有午夜 seed 壳且未 ``force``，直接返回等待晚间落盘， + 避免把「全员 skip」误标成 completed。 + """ now = self._now() stat_date = date or shanghai_date(now) if not is_date(stat_date): raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}") + seeded = self.repo.kv_get(self.seeded_key(stat_date)) == "1" if force: return self.run_videos(date=stat_date, force=True) completed = self.repo.kv_get(self.completed_key(stat_date)) == "1" + if seeded and not completed: + self._log( + "info", "snapshot.skip", + stat_date=stat_date, reason="seeded_pending_seal", + ) + return SnapshotSummary( + stat_date=stat_date, + started_at=now, + finished_at=now, + completed=False, + already_completed=False, + ) pending = self.repo.pending_snapshot_aids(stat_date) if completed and not pending: if self.repo.kv_get(self.backup_key(stat_date)) != "1": @@ -256,6 +283,8 @@ class SnapshotService: self.repo.kv_set("snapshot:last_attempt", stat_date) if summary.completed: self.repo.kv_set("snapshot:last_completed", stat_date) + # 真落盘后清掉午夜 seed 标记。 + self.repo.kv_delete(self.seeded_key(stat_date)) elif self.repo.kv_get("snapshot:last_completed") == stat_date: self.repo.kv_set("snapshot:last_completed", "") @@ -378,7 +407,24 @@ class SnapshotService: ) self._log("info", "fans.skip", stat_date=stat_date, reason="no_track_fans") return summary - if not force and not self._pending_fan_mids(stat_date, cfg.track_fans): + seeded = self.repo.kv_get(self.fans_seeded_key(stat_date)) == "1" + if force or seeded: + # 晚间/显式 force，或存在 seed 壳时必须重采（seeded 时由调度传入 force）。 + if seeded and not force: + self._log( + "info", "fans.skip", + stat_date=stat_date, reason="seeded_pending_seal", + ) + return SnapshotSummary( + stat_date=stat_date, + started_at=now, + finished_at=now, + completed=False, + already_completed=False, + fans_skipped=len(cfg.track_fans), + ) + return self.run_fans(date=stat_date, force=True) + if not self._pending_fan_mids(stat_date, cfg.track_fans): self._log("info", "fans.skip", stat_date=stat_date, reason="already_completed") return SnapshotSummary( stat_date=stat_date, @@ -456,6 +502,11 @@ class SnapshotService: # 独立粉丝任务用 completed 表示粉丝段成败；合并进视频摘要时不改动视频完成标记。 if standalone: summary.completed = fans_ok + if fans_ok: + self.repo.kv_delete(self.fans_seeded_key(stat_date)) + elif fans_ok: + # CLI 合并跑：粉丝段成功也清 seed。 + self.repo.kv_delete(self.fans_seeded_key(stat_date)) summary.finished_at = self._now() self._log( "info" if fans_ok else "warning", @@ -468,6 +519,76 @@ class SnapshotService: ) return summary + + def run_seed_from_last_completed( + self, *, date: str | None = None + ) -> SnapshotSummary: + """用上一完成日的 ok=1 行克隆出当日壳（视频 + 粉丝）。 + + 不写 ``snapshot:completed`` / ``snapshot:last_completed``； + 仅打 ``snapshot:seeded`` / ``fans:seeded``，供白天展示与晚间 force 重采。 + """ + cfg = self.config_store.get() + now = self._now() + stat_date = date or shanghai_date(now) + if not is_date(stat_date): + raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}") + + summary = SnapshotSummary(stat_date=stat_date, started_at=now) + source = self.repo.kv_get("snapshot:last_completed") + if not source or source >= stat_date: + self._log( + "info", "snapshot.seed_skip", + stat_date=stat_date, reason="no_source", source=source, + ) + summary.finished_at = self._now() + summary.completed = True + return summary + if self.repo.kv_get(self.completed_key(stat_date)) == "1": + self._log( + "info", "snapshot.seed_skip", + stat_date=stat_date, reason="already_completed", + ) + summary.finished_at = self._now() + summary.completed = True + summary.already_completed = True + return summary + if self.repo.kv_get(self.seeded_key(stat_date)) == "1": + self._log( + "info", "snapshot.seed_skip", + stat_date=stat_date, reason="already_seeded", source=source, + ) + summary.finished_at = self._now() + summary.completed = True + summary.already_completed = True + return summary + + copied = self.repo.copy_ok_snapshots( + source, stat_date, captured_at=now, src="seed", + ) + fans_copied = 0 + if cfg.track_fans: + fans_copied = self.repo.copy_ok_up_stats( + source, stat_date, captured_at=now, mids=cfg.track_fans, + ) + self.repo.kv_set(self.fans_seeded_key(stat_date), "1") + self.repo.kv_set(self.seeded_key(stat_date), "1") + summary.total = copied + summary.success = copied + summary.fans_ok = fans_copied + if copied or fans_copied: + self.repo.bump_cache_version("videos") + if fans_copied: + self.repo.bump_cache_version("fans") + summary.finished_at = self._now() + summary.completed = True # seed 任务本身成功；不等于快照落盘完成 + self._log( + "info", "snapshot.seed_done", + stat_date=stat_date, source=source, + copied=copied, fans_copied=fans_copied, + ) + return summary + def _pending_fan_mids(self, stat_date: str, mids: tuple[int, ...] | list[int]) -> list[int]: pending: list[int] = [] for mid in mids: diff --git a/reaction_board/web/queries.py b/reaction_board/web/queries.py index 23937e1..5db1331 100644 --- a/reaction_board/web/queries.py +++ b/reaction_board/web/queries.py @@ -284,6 +284,37 @@ class DashboardQueries: result[int(data["aid"])] = data return result + @classmethod + def _snapshot_previous_relative( + cls, + conn: sqlite3.Connection, + current: Mapping[int, Mapping[str, Any]], + *, + days_back: int, + ) -> dict[int, dict[str, Any]]: + """相对各片当前快照日取 previous：days_back=1 为相邻成功日，否则 <= current-N。""" + if not current or days_back < 1: + return {} + by_date: dict[str, list[int]] = {} + for aid, snap in current.items(): + day = snap.get("stat_date") + if not day: + continue + by_date.setdefault(str(day), []).append(int(aid)) + result: dict[int, dict[str, Any]] = {} + for day, aids in by_date.items(): + if days_back <= 1: + prev = cls._snapshot_map( + conn, aids, before=day, before_inclusive=False, + ) + else: + baseline = _date_minus(day, days_back) + prev = cls._snapshot_map( + conn, aids, before=baseline, before_inclusive=True, + ) + result.update(prev) + return result + @staticmethod def _videos( conn: sqlite3.Connection, @@ -443,7 +474,8 @@ class DashboardQueries: videos = self._videos(conn) aids = [int(video["aid"]) for video in videos] snapshots = self._snapshot_map(conn, aids) - previous = self._snapshot_map(conn, aids, before=latest or self._today()) + # 播放增量：按片相对当前点的相邻成功日，避免 seed/recent 拉到 T-2。 + previous = self._snapshot_previous_relative(conn, snapshots, days_back=1) totals = {metric: 0 for metric in METRICS} for video in videos: metrics = _metric_map(snapshots.get(int(video["aid"]))) @@ -521,6 +553,7 @@ class DashboardQueries: reverse=True, ) tasks = self._task_records(conn) + updated_at = self._latest_short_update_at(tasks) auth = { "ok": self._kv(conn, "auth:ok"), "message": self._kv(conn, "auth:message"), @@ -535,6 +568,7 @@ class DashboardQueries: "generated_at": int(self._now()), "today": self._today(), "data_date": latest, + "updated_at": updated_at, "snapshot_days": snapshot_days, "first_seen_date": first_seen_date, "is_first_day": is_first_day, @@ -542,6 +576,7 @@ class DashboardQueries: "added_today": added_today, "counts": { **by_status, + "total": sum(by_status.values()), "active_ups": int(conn.execute( "SELECT COUNT(DISTINCT up_mid) FROM video " "WHERE status IN ('active','unfaved') AND up_mid IS NOT NULL" @@ -571,6 +606,22 @@ class DashboardQueries: row = conn.execute("SELECT v FROM kv WHERE k = ?", (key,)).fetchone() return None if row is None else str(row["v"]) + @staticmethod + def _latest_short_update_at(tasks: Mapping[str, Any]) -> int | None: + """首页「更新」时刻：优先短更新任务，其次全日更 / 粉丝。""" + for name in ("snapshot_recent", "snapshot", "fans"): + raw = tasks.get(name) + if not isinstance(raw, Mapping): + continue + finished = raw.get("finished_at") + if finished is None: + continue + try: + return int(finished) + except (TypeError, ValueError): + continue + return None + def _task_records(self, conn: sqlite3.Connection) -> dict[str, Any]: rows = conn.execute( "SELECT k, v FROM kv WHERE k LIKE 'task:last:%' ORDER BY k" @@ -802,8 +853,10 @@ class DashboardQueries: previous: dict[int, dict[str, Any]] = {} if sort in {"growth", "growth_rate", "interaction"}: - baseline_date = _date_minus(latest, growth_days) - previous = self._snapshot_map(conn, aids, before=baseline_date, before_inclusive=True) + # 相对各片当前快照日回退 growth_days，避免 seed/recent 相对 last_completed 拉长窗口。 + previous = self._snapshot_previous_relative( + conn, snapshots, days_back=growth_days, + ) elif sort in {"heat", "view", "like", "coin", "favorite", "share", "reply", "danmaku"}: previous = {} @@ -1780,14 +1833,16 @@ class DashboardQueries: next_runs = { "fans": int(next_daily_run(now, cfg.schedule.snapshot_at)), "snapshot": int(next_daily_run(now, cfg.schedule.snapshot_at)), + "snapshot_seed": int(next_daily_run(now, cfg.schedule.seed_at)), "discover": int(now + discover_interval), "snapshot_recent": int(now + recent_interval), } - for name, key in ( - ("fans", "fans"), - ("snapshot", "snapshot"), - ("discover", "discovery"), - ("snapshot_recent", "snapshot_recent"), + for name, key, at in ( + ("fans", "fans", cfg.schedule.snapshot_at), + ("snapshot", "snapshot", cfg.schedule.snapshot_at), + ("snapshot_seed", "snapshot_seed", cfg.schedule.seed_at), + ("discover", "discovery", None), + ("snapshot_recent", "snapshot_recent", None), ): record = tasks.get(key) if not (isinstance(record, dict) and record.get("finished_at")): @@ -1801,9 +1856,7 @@ class DashboardQueries: ) next_runs[name] = finished_at + interval else: - next_runs[name] = int( - next_daily_run(finished_at, cfg.schedule.snapshot_at) - ) + next_runs[name] = int(next_daily_run(finished_at, at)) errors = [ {"task": name, **record} for name, record in tasks.items() @@ -1840,6 +1893,7 @@ class DashboardQueries: "schedule": { "discover_minutes": cfg.schedule.discover_minutes, "snapshot_at": cfg.schedule.snapshot_at, + "seed_at": cfg.schedule.seed_at, "candidate_ttl_days": cfg.schedule.candidate_ttl_days, "recent_snapshot_minutes": cfg.schedule.recent_snapshot_minutes, "recent_video_days": cfg.schedule.recent_video_days, diff --git a/reaction_board/web/static/format.js b/reaction_board/web/static/format.js index b931a60..27c3603 100644 --- a/reaction_board/web/static/format.js +++ b/reaction_board/web/static/format.js @@ -55,6 +55,22 @@ export function formatDate(timestamp) { }).format(date); } + +export function formatClock(timestamp) { + if (!timestamp) return "—"; + const date = new Date(Number(timestamp) * 1000); + if (Number.isNaN(date.getTime())) return "—"; + return new Intl.DateTimeFormat("zh-CN", { + hour: "2-digit", minute: "2-digit", hour12: false, + timeZone: "Asia/Shanghai", + }).format(date); +} + +export function formatDataDate(value) { + if (!value) return "—"; + return String(value).replace(/-/g, "/"); +} + export function formatDateTime(timestamp) { if (!timestamp) return "—"; const date = new Date(Number(timestamp) * 1000); diff --git a/reaction_board/web/static/views/home.js b/reaction_board/web/static/views/home.js index 8753661..fbccdd4 100644 --- a/reaction_board/web/static/views/home.js +++ b/reaction_board/web/static/views/home.js @@ -1,8 +1,8 @@ import { api } from "../api.js"; import { emptyState, mediaCard } from "../components.js"; import { - escapeAttr, escapeHtml, formatCount, formatDelta, - formatPercent, formatScore, + escapeAttr, escapeHtml, formatClock, formatCount, formatDataDate, + formatDelta, formatPercent, formatScore, } from "../format.js"; import { canAccessProtected } from "../session.js"; @@ -55,6 +55,11 @@ export async function renderHome(root) { ? formatCount(overview.initial_import_count) : formatCount(overview.added_today); const activeCount = overview.counts?.active || 0; + const totalCount = overview.counts?.total + ?? ((overview.counts?.active || 0) + + (overview.counts?.unfaved || 0) + + (overview.counts?.removed || 0) + + (overview.counts?.dropped || 0)); const pending = overview.counts?.candidates_pending || 0; // 候选提示仅对已登录（或未启用鉴权）用户可见，访客不展示运行期信息。 const showCandidates = pending > 0 && canAccessProtected(); @@ -66,12 +71,14 @@ export async function renderHome(root) { const growthEmptyHint = growthDays <= 1 ? "当前快照不足，完成至少 2 天快照后即可显示播放增量。" : `需要至少 ${growthDays + 1} 天快照才能计算近 ${growthDays} 日增长。`; + const updatedLabel = formatClock(overview.updated_at); + const dataDateLabel = formatDataDate(overview.data_date); root.innerHTML = `

最新动态

-

收录视频 ${formatCount(activeCount)}

+

更新：${escapeHtml(updatedLabel)} / 全站数据：${escapeHtml(dataDateLabel)}

@@ -86,7 +93,7 @@ export async function renderHome(root) { ${statChip("7 日粉丝变化", fans?.delta_7d === null || fans?.delta_7d === undefined ? "—" : formatDelta(fans.delta_7d), { href: "/fans", })} - ${statChip("数据日期", escapeHtml(overview.data_date || "—"))} + ${statChip("收录视频", `${formatCount(activeCount)} / ${formatCount(totalCount)}`)} ${showCandidates ? ` diff --git a/reaction_board/web/static/views/system.js b/reaction_board/web/static/views/system.js index 29349a6..f4920d7 100644 --- a/reaction_board/web/static/views/system.js +++ b/reaction_board/web/static/views/system.js @@ -94,6 +94,7 @@ export async function renderSystem(root) {
${sectionHeader("预计下次运行")}
+
午夜落盘克隆${running.snapshot_seed ? "运行中" : escapeHtml(formatDateTime(next.snapshot_seed))}
咻咻满粉丝数更新${running.fans ? "运行中" : escapeHtml(formatDateTime(next.fans))}
视频数据快照 + UP 信息${running.snapshot ? "运行中" : escapeHtml(formatDateTime(next.snapshot))}
新视频小时同步${running.snapshot_recent ? "运行中" : escapeHtml(formatDateTime(next.snapshot_recent))}
diff --git a/tests/test_config.py b/tests/test_config.py index f68928d..4f220fd 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -154,7 +154,8 @@ def test_analytics_defaults_and_validation(): } data["schedule"] = { "discover_minutes": 30, - "snapshot_at": "00:01", + "snapshot_at": "23:15", + "seed_at": "00:01", "candidate_ttl_days": 7, "recent_snapshot_minutes": 15, "recent_video_days": 3, @@ -276,7 +277,8 @@ def test_schedule_recent_validation(): data = base_config() data["schedule"] = { "discover_minutes": 60, - "snapshot_at": "00:01", + "snapshot_at": "23:15", + "seed_at": "00:01", "candidate_ttl_days": 7, **bad, } diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index e0a2795..02856ee 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -501,7 +501,7 @@ def test_run_recent_videos_refreshes_without_completion_markers(): def test_full_snapshot_skips_ok_rows_including_hourly_refresh(): - """00:01 全量：同日 ok=1（含小时任务刚刷）直接跳过。""" + """晚间全量（未 force）：同日 ok=1（含小时任务刚刷）直接跳过。""" clock = 1_800_000_000 with tempfile.TemporaryDirectory() as tmp: transport = SnapshotTransport({1: view_for(1), 2: view_for(2)}) @@ -530,6 +530,99 @@ def test_full_snapshot_skips_ok_rows_including_hourly_refresh(): +def test_seed_from_last_completed_does_not_mark_completed(): + clock = 1_800_000_000 + with tempfile.TemporaryDirectory() as tmp: + transport = SnapshotTransport({1: view_for(1)}) + db, repo, service = make_service(tmp, transport, track_fans=(37754047,)) + try: + add_video(repo, 1) + yday = "2026-10-30" + today = "2026-10-31" + repo.upsert_snapshot( + yday, 1, + metrics={"view": 10, "like": 1, "coin": 0, "favorite": 0, + "share": 0, "reply": 0, "danmaku": 0}, + ok=True, err=None, captured_at=clock - 86400, + ) + repo.upsert_up_stat( + yday, 37754047, follower=100, following=1, ok=True, + captured_at=clock - 86400, + ) + repo.kv_set("snapshot:last_completed", yday) + repo.kv_set(service.completed_key(yday), "1") + + summary = service.run_seed_from_last_completed(date=today) + assert summary.success == 1 + assert summary.fans_ok == 1 + assert repo.kv_get(service.seeded_key(today)) == "1" + assert repo.kv_get(service.fans_seeded_key(today)) == "1" + assert repo.kv_get(service.completed_key(today)) is None + assert repo.kv_get("snapshot:last_completed") == yday + row = repo.get_snapshot(today, 1) + assert row is not None + assert int(row["view"]) == 10 + assert row["src"] == "seed" + fan = repo.get_up_stat(today, 37754047) + assert fan is not None and int(fan["follower"]) == 100 + + # 幂等：再 seed 不重复 + again = service.run_seed_from_last_completed(date=today) + assert again.already_completed is True + assert transport.count("/web-interface/view") == 0 + finally: + db.close() + + +def test_seeded_day_skips_without_force_and_seals_with_force(): + clock = 1_800_000_000 + with tempfile.TemporaryDirectory() as tmp: + fan_payload = dict(RELATION_PAYLOAD) + fan_payload["data"] = dict(RELATION_PAYLOAD["data"]) + fan_payload["data"]["follower"] = 999 + transport = SnapshotTransport({1: view_for(1, view=555)}, fan=fan_payload) + db, repo, service = make_service(tmp, transport, track_fans=(37754047,)) + try: + add_video(repo, 1) + yday = "2026-10-30" + today = DATE + repo.upsert_snapshot( + yday, 1, + metrics={"view": 10, "like": 1, "coin": 0, "favorite": 0, + "share": 0, "reply": 0, "danmaku": 0}, + ok=True, err=None, captured_at=clock - 86400, + ) + repo.upsert_up_stat( + yday, 37754047, follower=100, following=1, ok=True, + captured_at=clock - 86400, + ) + repo.kv_set("snapshot:last_completed", yday) + service.run_seed_from_last_completed(date=today) + + pending = service.run_if_needed(date=today) + assert pending.completed is False + assert pending.already_completed is False + assert repo.kv_get(service.seeded_key(today)) == "1" + assert transport.count("/web-interface/view") == 0 + + sealed = service.run_if_needed(date=today, force=True) + assert sealed.completed is True + assert sealed.success >= 1 + assert repo.kv_get(service.completed_key(today)) == "1" + assert repo.kv_get("snapshot:last_completed") == today + assert repo.kv_get(service.seeded_key(today)) is None + row = repo.get_snapshot(today, 1) + assert int(row["view"]) == 555 # 被真实 view 覆盖 + assert row["src"] == "view" + + fans = service.run_fans_if_needed(date=today, force=True) + assert fans.fans_ok >= 1 + assert repo.kv_get(service.fans_seeded_key(today)) is None + assert int(repo.get_up_stat(today, 37754047)["follower"]) == 999 + finally: + db.close() + + def main(): test_snapshot_success_resume_and_force() test_resume_skips_ok_and_retries_failed() @@ -545,6 +638,8 @@ def main(): test_backup_failure_retries_without_api() test_run_recent_videos_refreshes_without_completion_markers() test_full_snapshot_skips_ok_rows_including_hourly_refresh() + test_seed_from_last_completed_does_not_mark_completed() + test_seeded_day_skips_without_force_and_seals_with_force() print("OK: 每日快照、断点续跑、失败占位与粉丝采集测试全部通过") diff --git a/tests/test_web_queries.py b/tests/test_web_queries.py index f157eb3..5ab401b 100644 --- a/tests/test_web_queries.py +++ b/tests/test_web_queries.py @@ -626,6 +626,61 @@ def test_snapshot_recent_next_run_follows_completion(): db.close() + +def test_overview_adjacent_previous_with_seed_and_dual_fields(): + """seed 当日行存在时，播放增量按片相邻成功日；overview 暴露更新时刻与总数。""" + import json + + with tempfile.TemporaryDirectory() as tmp: + db, q = setup(tmp) + try: + repo = Repository(db) + today = "2026-11-01" + # 克隆上一完成日到今天（不推进 last_completed） + for aid in (101, 102): + prev = repo.get_snapshot(DATES[-1], aid) + metrics = { + k: int(prev[k]) for k in ( + "view", "like", "coin", "favorite", "share", "reply", "danmaku" + ) + } + repo.upsert_snapshot( + today, aid, metrics=metrics, ok=True, err=None, + captured_at=FIXED_NOW, src="seed", + ) + repo.kv_set("snapshot:seeded:" + today, "1") + # 小时短更新任务记录 + repo.kv_set( + "task:last:snapshot_recent", + json.dumps({ + "name": "snapshot_recent", "ok": True, + "started_at": FIXED_NOW - 60, "finished_at": FIXED_NOW - 30, + "error": None, "summary": {}, + }), + ) + DASHBOARD_CACHE.clear() + overview = q.overview() + assert overview["data_date"] == DATES[-1] + assert overview["updated_at"] == FIXED_NOW - 30 + assert overview["counts"]["total"] == ( + overview["counts"]["active"] + + overview["counts"]["unfaved"] + + overview["counts"]["removed"] + ) + # 当前=seed(=完成日数值)，相邻 previous=完成日本身 → 增量 0 + assert overview["view_delta_available"] is True + assert overview["view_delta"] == 0 + + growth = q.list_videos(sort="growth", growth_days=1) + assert growth["available"] is True + for item in growth["items"]: + if item["aid"] in (101, 102): + assert item["growth"]["available"] is True + assert item["growth"]["delta"] == 0 + assert item["growth"]["gap_days"] == 1 + finally: + db.close() + def main_test(): test_overview() test_video_listing_filters_and_sorting() @@ -643,6 +698,7 @@ def main_test(): test_system_next_runs_follow_last_completion_not_wall_clock() test_growth_auto_falls_back_to_one_day() test_snapshot_recent_next_run_follows_completion() + test_overview_adjacent_previous_with_seed_and_dual_fields() print("OK: Web 查询层总览/榜单/详情/UP/时长效率/粉丝/候选/系统/访问统计测试全部通过") diff --git a/tests/web_fixtures.py b/tests/web_fixtures.py index 73bf99c..789cb2e 100644 --- a/tests/web_fixtures.py +++ b/tests/web_fixtures.py @@ -48,7 +48,8 @@ def write_config( "up_blacklist_seed": [], "schedule": { "discover_minutes": 60, - "snapshot_at": "00:01", + "snapshot_at": "23:15", + "seed_at": "00:01", "candidate_ttl_days": 7, "recent_snapshot_minutes": 60, "recent_video_days": 2,