From 9163f47b76835eb74e2f8a1ef72ef752eaf4af09 Mon Sep 17 00:00:00 2001 From: bot-ci Date: Mon, 2 Nov 2026 10:00:02 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20home=20growth=20auto(7=E2=86=921)?= =?UTF-8?q?=20and=20hourly=20recent-video=20snapshot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add analytics.home_growth_days (auto|1|7); API growth_days=auto tries 7 then 1 - Homepage shows active window and growth-board link; clearer empty reason - New snapshot_recent scheduler task (schedule.recent_snapshot_minutes/days) - run_recent_videos refreshes new aids only; never touches completed/backup - Full 00:01 job keeps ok=1 skip (covers hourly refresh within 1h); mutex with recent --- config.docker.example.json | 7 +- config.example.json | 7 +- reaction_board/cli.py | 56 +++++++++++++ reaction_board/config.py | 24 ++++++ reaction_board/repository.py | 13 +++ reaction_board/services/snapshot.py | 97 ++++++++++++++++++++++- reaction_board/web/queries.py | 41 ++++++++-- reaction_board/web/server.py | 26 +++++- reaction_board/web/static/style.css | 3 +- reaction_board/web/static/views/home.js | 16 +++- reaction_board/web/static/views/system.js | 6 +- tests/test_config.py | 48 +++++++++++ tests/test_snapshot.py | 96 ++++++++++++++++++++++ tests/test_web_queries.py | 76 +++++++++++++++++- tests/web_fixtures.py | 2 + 15 files changed, 496 insertions(+), 22 deletions(-) diff --git a/config.docker.example.json b/config.docker.example.json index 7de7b0a..2a0eef9 100644 --- a/config.docker.example.json +++ b/config.docker.example.json @@ -64,7 +64,9 @@ "schedule": { "discover_minutes": 60, "snapshot_at": "00:01", - "candidate_ttl_days": 7 + "candidate_ttl_days": 7, + "recent_snapshot_minutes": 60, + "recent_video_days": 2 }, "discover": { "keywords": ["咻咻满"], @@ -107,6 +109,7 @@ "maturity_days": 2, "min_view_for_ranking": 2000, "video_shrink_minutes": 60, - "shrink_minutes": null + "shrink_minutes": null, + "home_growth_days": "auto" } } \ No newline at end of file diff --git a/config.example.json b/config.example.json index e81459f..6069938 100644 --- a/config.example.json +++ b/config.example.json @@ -74,7 +74,9 @@ "schedule": { "discover_minutes": 60, "snapshot_at": "00:01", - "candidate_ttl_days": 7 + "candidate_ttl_days": 7, + "recent_snapshot_minutes": 60, + "recent_video_days": 2 }, "discover": { "keywords": ["咻咻满"], @@ -110,6 +112,7 @@ "maturity_days": 2, "min_view_for_ranking": 2000, "video_shrink_minutes": 60, - "shrink_minutes": null + "shrink_minutes": null, + "home_growth_days": "auto" } } diff --git a/reaction_board/cli.py b/reaction_board/cli.py index c580f69..2e6372e 100644 --- a/reaction_board/cli.py +++ b/reaction_board/cli.py @@ -163,6 +163,13 @@ def _clear_task_running(repo: Repository, name: str) -> None: pass +def _is_task_running(repo: Repository, name: str) -> bool: + try: + return bool(repo.kv_get(_task_running_key(name))) + except Exception: # noqa: BLE001 + return False + + def _record_task( repo: Repository, name: str, @@ -547,6 +554,16 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int: def run_snapshot(): started = int(time.time()) + if _is_task_running(repo, "snapshot_recent"): + logger.info( + "snapshot.skip", reason="snapshot_recent_running", + ) + _record_task( + repo, "snapshot", ok=True, started_at=started, + finished_at=int(time.time()), + payload={"skipped": True, "reason": "snapshot_recent_running"}, + ) + return None _mark_task_running(repo, "snapshot", started_at=started) try: summary = snapshot_service.run_if_needed() @@ -567,6 +584,39 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int: logger.info("snapshot.summary", **payload) return summary + def run_snapshot_recent(): + started = int(time.time()) + if _is_task_running(repo, "snapshot"): + logger.info( + "snapshot.recent_skip", reason="snapshot_running", + ) + _record_task( + repo, "snapshot_recent", ok=True, started_at=started, + finished_at=int(time.time()), + payload={"skipped": True, "reason": "snapshot_running"}, + ) + return None + _mark_task_running(repo, "snapshot_recent", started_at=started) + try: + summary = snapshot_service.run_recent_videos() + except Exception as exc: # noqa: BLE001 + _record_task( + repo, "snapshot_recent", ok=False, started_at=started, + finished_at=int(time.time()), error=str(exc), + ) + raise + payload = summary.as_dict() + ok = summary.failed == 0 + error = "；".join(summary.errors) if summary.errors else None + _record_task( + repo, "snapshot_recent", ok=ok, started_at=summary.started_at, + finished_at=summary.finished_at or int(time.time()), + payload=payload, error=error, + ) + last["snapshot_recent"] = summary + logger.info("snapshot.recent_summary", **payload) + return summary + def run_fans(): started = int(time.time()) _mark_task_running(repo, "fans", started_at=started) @@ -601,6 +651,7 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int: hh, mm = parse_hhmm(cfg.schedule.snapshot_at) snapshot_at = f"{hh:02d}:{mm:02d}" + recent_minutes = int(store.get().schedule.recent_snapshot_minutes) scheduler = Scheduler( [ ScheduledTask( @@ -617,6 +668,11 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int: next_run=0.0, next_run_fn=lambda started: next_daily_run(started, snapshot_at), ), + ScheduledTask( + "snapshot_recent", + float(recent_minutes * 60), + run_snapshot_recent, + ), ScheduledTask( "discovery", float(cfg.schedule.discover_minutes * 60), diff --git a/reaction_board/config.py b/reaction_board/config.py index c76f7e0..f7cc615 100644 --- a/reaction_board/config.py +++ b/reaction_board/config.py @@ -49,6 +49,7 @@ DEFAULT_ANALYTICS: dict[str, Any] = { "min_view_for_ranking": 2000, "video_shrink_minutes": 60, "shrink_minutes": None, + "home_growth_days": "auto", } DEFAULT_CONFIG: dict[str, Any] = { @@ -84,6 +85,8 @@ DEFAULT_CONFIG: dict[str, Any] = { "discover_minutes": 60, "snapshot_at": "00:01", "candidate_ttl_days": 7, + "recent_snapshot_minutes": 60, + "recent_video_days": 2, }, "discover": { "keywords": ["咻咻满"], @@ -177,6 +180,8 @@ class ScheduleConfig: discover_minutes: int snapshot_at: str candidate_ttl_days: int + recent_snapshot_minutes: int + recent_video_days: int @dataclass(frozen=True) @@ -200,6 +205,7 @@ class AnalyticsConfig: min_view_for_ranking: int video_shrink_minutes: int shrink_minutes: int | None + home_growth_days: str | int @dataclass(frozen=True) @@ -456,6 +462,16 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig: "schedule.candidate_ttl_days", minimum=1, ), + recent_snapshot_minutes=_as_int( + sched_raw.get("recent_snapshot_minutes", 60), + "schedule.recent_snapshot_minutes", + minimum=1, + ), + recent_video_days=_as_int( + sched_raw.get("recent_video_days", 2), + "schedule.recent_video_days", + minimum=1, + ), ) notify_raw = data["notify"] @@ -533,6 +549,13 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig: if not isinstance(analytics_raw, Mapping): raise ConfigError("analytics 必须是对象") shrink_raw = analytics_raw.get("shrink_minutes") + home_growth_raw = analytics_raw.get("home_growth_days", "auto") + if home_growth_raw == "auto": + home_growth_days: str | int = "auto" + elif home_growth_raw in (1, 7): + home_growth_days = int(home_growth_raw) + else: + raise ConfigError('analytics.home_growth_days 必须是 "auto"、1 或 7') analytics = AnalyticsConfig( min_duration_seconds=_as_int( analytics_raw.get("min_duration_seconds", 600), @@ -559,6 +582,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig: shrink_raw, "analytics.shrink_minutes", minimum=1 ) ), + home_growth_days=home_growth_days, ) return AppConfig( diff --git a/reaction_board/repository.py b/reaction_board/repository.py index b4d2f3b..edbaa6f 100644 --- a/reaction_board/repository.py +++ b/reaction_board/repository.py @@ -637,6 +637,19 @@ class Repository: return [int(row["aid"]) for row in rows] return self.db.read(run) + def video_aids_recent_for_snapshot(self, *, since_pubdate: int) -> list[int]: + """近 N 天新视频：按 pubdate 墙钟窗口（since_pubdate = now - days*86400）。""" + def run(conn: sqlite3.Connection) -> list[int]: + rows = conn.execute( + "SELECT aid FROM video " + "WHERE status IN ('active', 'unfaved') " + "AND pubdate IS NOT NULL AND pubdate >= ? " + "ORDER BY aid", + (int(since_pubdate),), + ).fetchall() + return [int(row["aid"]) for row in rows] + return self.db.read(run) + def pending_snapshot_aids(self, stat_date: str) -> list[int]: """当天仍缺成功快照的追踪视频；用于新入库/恢复后的同日补采。""" def run(conn: sqlite3.Connection) -> list[int]: diff --git a/reaction_board/services/snapshot.py b/reaction_board/services/snapshot.py index 362e71a..bb06690 100644 --- a/reaction_board/services/snapshot.py +++ b/reaction_board/services/snapshot.py @@ -4,7 +4,8 @@ 规则（DEVELOPMENT.md §7.4）： * 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``； -* 成功写入七项指标；``ok=1`` 的同日快照直接跳过（断点续跑）； +* 成功写入七项指标；``ok=1`` 的同日快照直接跳过（断点续跑；含小时任务 1 小时内刚刷新的）； +* ``run_recent_videos`` 只刷新近 N 天新片的当日行，不改完成日 / 备份； * 视频不可访问时标记 ``removed``； * 其他错误最多重试 3 次，仍失败写 ``ok=0`` 占位，不写假零值； * 对 ``track_fans`` 逐个请求 ``relation/stat``（独立任务，不影响视频完成日）； @@ -198,6 +199,8 @@ class SnapshotService: for index, aid in enumerate(videos, 1): existing = self.repo.get_snapshot(stat_date, aid) if existing is not None and int(existing.get("ok") or 0) == 1 and not force: + # 断点续跑：同日已成功则跳过。小时任务刷新过的行也是 ok=1， + # 自然覆盖「1 小时内 captured_at 已更新则不再次更新」的要求。 summary.skipped += 1 continue @@ -269,6 +272,98 @@ class SnapshotService: ) return summary + def run_recent_videos( + self, + *, + date: str | None = None, + recent_days: int | None = None, + ) -> SnapshotSummary: + """刷新近 N 天新视频的当日快照；不碰完成日标记与日备份。 + + 「近 N 天」按 pubdate 墙钟窗口：``now - recent_video_days * 86400``， + 不是上海日历日差。允许覆盖同日已有 ``ok=1`` 行以便小时级刷新。 + """ + cfg = self.config_store.get() + client = self._client() + now = self._now() + stat_date = date or shanghai_date(now) + if not is_date(stat_date): + raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}") + days = ( + int(cfg.schedule.recent_video_days) + if recent_days is None else int(recent_days) + ) + if days < 1: + raise ValueError("recent_days 必须 >= 1") + # 墙钟窗口，而非按上海日历截断到日界。 + since_pubdate = now - days * 86400 + + summary = SnapshotSummary(stat_date=stat_date, started_at=now) + self._log( + "info", "snapshot.recent_start", + stat_date=stat_date, recent_days=days, since_pubdate=since_pubdate, + ) + videos = self.repo.video_aids_recent_for_snapshot(since_pubdate=since_pubdate) + summary.total = len(videos) + for index, aid in enumerate(videos, 1): + stat, error, unavailable = self._fetch_view(client, aid) + if unavailable: + summary.removed += 1 + self.repo.set_video_status(aid, "removed", now=now) + self._log( + "warning", "snapshot.recent_removed", + stat_date=stat_date, aid=aid, error=error, + ) + elif stat is not None: + self.repo.upsert_snapshot( + stat_date, + aid, + metrics=stat.metrics(), + ok=True, + err=None, + captured_at=self._now(), + ) + if getattr(stat, "up_face", None): + self.repo.apply_video_up_face(aid, stat.up_face) + summary.success += 1 + else: + summary.failed += 1 + self.repo.upsert_snapshot( + stat_date, + aid, + metrics=None, + ok=False, + err=error, + captured_at=self._now(), + ) + summary.errors.append(f"aid {aid}: {error}") + self._log( + "error", "snapshot.recent_failed", + stat_date=stat_date, aid=aid, error=error, + ) + + if self.progress_every > 0 and ( + index % self.progress_every == 0 or index == summary.total + ): + self._log( + "info", "snapshot.recent_progress", + stat_date=stat_date, done=index, total=summary.total, + success=summary.success, failed=summary.failed, + removed=summary.removed, + ) + + # 刻意不写 snapshot:completed / last_completed / 日备份：完成日只归全量任务。 + summary.completed = summary.failed == 0 + if summary.success or summary.removed or summary.failed: + self.repo.bump_cache_version("videos") + summary.finished_at = self._now() + self._log( + "info" if summary.completed else "warning", + "snapshot.recent_done", + **summary.as_dict(), + ) + return summary + def run_fans_if_needed( self, *, date: str | None = None, force: bool = False ) -> SnapshotSummary: diff --git a/reaction_board/web/queries.py b/reaction_board/web/queries.py index 4efd913..68f17cd 100644 --- a/reaction_board/web/queries.py +++ b/reaction_board/web/queries.py @@ -699,7 +699,7 @@ class DashboardQueries: sort: str = "heat", within_days: int | None = None, year: int | None = None, - growth_days: int = 7, + growth_days: int | str = 7, folder_id: int | None = None, q: str | None = None, page: int = 1, @@ -734,7 +734,7 @@ class DashboardQueries: sort: str = "heat", within_days: int | None = None, year: int | None = None, - growth_days: int = 7, + growth_days: int | str = 7, folder_id: int | None = None, q: str | None = None, page: int = 1, @@ -742,6 +742,25 @@ class DashboardQueries: ) -> dict[str, Any]: if sort not in VIDEO_SORTS: raise ValueError(f"不支持的排序: {sort}") + if growth_days == "auto": + preferred = self.config_store.get().analytics.home_growth_days + if preferred in (1, 7): + growth_days = int(preferred) + else: + trial = self._list_videos_uncached( + sort=sort, + within_days=within_days, + year=year, + growth_days=7, + folder_id=folder_id, + q=q, + page=page, + page_size=page_size, + ) + if trial.get("available") and int(trial.get("total") or 0) > 0: + return trial + growth_days = 1 + growth_days = int(growth_days) if growth_days < 1 or growth_days > 365: raise ValueError("growth_days 必须在 1~365 之间") page = max(1, int(page)) @@ -900,7 +919,9 @@ class DashboardQueries: reason = None if sort in {"growth", "growth_rate"} and not growth_available: available = False - reason = "需要至少两天的快照才能计算增长" + reason = ( + f"近 {growth_days} 日增长需要至少 {growth_days + 1} 天快照" + ) duration_filter = None if sort in duration_sorts: duration_filter = { @@ -1571,6 +1592,7 @@ class DashboardQueries: "min_view_for_ranking": cfg.analytics.min_view_for_ranking, "video_shrink_minutes": cfg.analytics.video_shrink_minutes, "shrink_minutes": cfg.analytics.shrink_minutes, + "home_growth_days": cfg.analytics.home_growth_days, }, "heat_weights": { metric: float(value) @@ -1747,23 +1769,30 @@ class DashboardQueries: running = self._task_running(conn) now = self._now() discover_interval = int(cfg.schedule.discover_minutes) * 60 + recent_interval = int(cfg.schedule.recent_snapshot_minutes) * 60 next_runs = { "fans": int(next_daily_run(now, cfg.schedule.snapshot_at)), "snapshot": int(next_daily_run(now, cfg.schedule.snapshot_at)), "discover": int(now + discover_interval), + "snapshot_recent": int(now + recent_interval), } for name, key in ( ("fans", "fans"), ("snapshot", "snapshot"), ("discover", "discovery"), + ("snapshot_recent", "snapshot_recent"), ): record = tasks.get(key) if not (isinstance(record, dict) and record.get("finished_at")): continue finished_at = int(record["finished_at"]) - if name == "discover": + if name in {"discover", "snapshot_recent"}: # 只按上次完成推下一档；进行中由 running 展示，不按墙钟抢跑。 - next_runs[name] = finished_at + discover_interval + interval = ( + recent_interval if name == "snapshot_recent" + else discover_interval + ) + next_runs[name] = finished_at + interval else: next_runs[name] = int( next_daily_run(finished_at, cfg.schedule.snapshot_at) @@ -1805,6 +1834,8 @@ class DashboardQueries: "discover_minutes": cfg.schedule.discover_minutes, "snapshot_at": cfg.schedule.snapshot_at, "candidate_ttl_days": cfg.schedule.candidate_ttl_days, + "recent_snapshot_minutes": cfg.schedule.recent_snapshot_minutes, + "recent_video_days": cfg.schedule.recent_video_days, }, "telegram_enabled": bool(cfg.telegram.enabled), } diff --git a/reaction_board/web/server.py b/reaction_board/web/server.py index 1d299b2..45dc580 100644 --- a/reaction_board/web/server.py +++ b/reaction_board/web/server.py @@ -373,7 +373,7 @@ border-color:#3a2c22;color:#f3e9dc}button{background:#e9b97c;color:#2a1a10}} sort=_param(query, "sort", "heat"), within_days=_optional_int(query, "within_days"), year=_year_param(query), - growth_days=_int_param(query, "growth_days", 7, 1, 365), + growth_days=_growth_days_param(query), folder_id=_optional_int(query, "folder"), q=_optional_str(query, "q"), page=_int_param(query, "page", 1, 1, 1_000_000), @@ -422,7 +422,7 @@ border-color:#3a2c22;color:#f3e9dc}button{background:#e9b97c;color:#2a1a10}} sort=_param(query, "sort", "heat"), within_days=_optional_int(query, "within_days"), year=_year_param(query), - growth_days=_int_param(query, "growth_days", 7, 1, 365), + growth_days=_growth_days_param(query), folder_id=_optional_int(query, "folder"), q=_optional_str(query, "q"), page=_int_param(query, "page", 1, 1, 1_000_000), @@ -649,6 +649,28 @@ def _year_param(query: Mapping[str, list[str]]) -> int | None: return value +def _growth_days_param(query: Mapping[str, list[str]]) -> int | str: + """榜单传 1~365；首页可传 auto（由查询层按 home_growth_days 解析）。""" + raw = _optional_str(query, "growth_days") + if raw is None: + return 7 + if raw == "auto": + return "auto" + try: + value = int(raw) + except ValueError as exc: + raise ApiError( + HTTPStatus.BAD_REQUEST, "bad_request", + "growth_days 必须是整数或 auto", + ) from exc + if value < 1 or value > 365: + raise ApiError( + HTTPStatus.BAD_REQUEST, "bad_request", + "growth_days 必须在 1~365 之间", + ) + return value + + def _int_param( query: Mapping[str, list[str]], name: str, diff --git a/reaction_board/web/static/style.css b/reaction_board/web/static/style.css index a66736c..7c3bffc 100644 --- a/reaction_board/web/static/style.css +++ b/reaction_board/web/static/style.css @@ -510,6 +510,7 @@ a.live-chip:hover { border-color: color-mix(in srgb, var(--accent) 55%, var(--bo .status-grid strong { display: block; margin-top: 2px; font-size: 12.5px; font-weight: 600; } /* 预计下次运行等 3 项：PC 与移动端都保持一行 */ .status-grid.status-row-3 { grid-template-columns: repeat(3, minmax(0, 1fr)); } +.status-grid.status-row-4 { grid-template-columns: repeat(2, minmax(0, 1fr)); } .segmented { display: inline-flex; @@ -993,4 +994,4 @@ a.live-chip:hover { border-color: color-mix(in srgb, var(--accent) 55%, var(--bo .about-page .panel h2 { font-size: 15px; margin-bottom: 8px; } .about-list { margin: 0; padding-left: 18px; display: grid; gap: 6px; font-size: 13px; line-height: 1.55; } .about-list code { padding: 1px 4px; border-radius: 4px; background: var(--accent-soft); font-size: 12px; } -.about-list.about-weights { list-style: none; padding-left: 0; grid-template-columns: repeat(2, minmax(0, 1fr)); } +.about-list.about-weights { list-style: none; padding-left: 0; grid-template-columns: repeat(2, minmax(0, 1fr)); } diff --git a/reaction_board/web/static/views/home.js b/reaction_board/web/static/views/home.js index 5dbdabc..714215f 100644 --- a/reaction_board/web/static/views/home.js +++ b/reaction_board/web/static/views/home.js @@ -47,7 +47,8 @@ export async function renderHome(root) { api("/api/overview"), api("/api/videos", { sort: "published", within_days: 14, limit: 6 }), api("/api/videos", { sort: "heat", limit: 6 }), - api("/api/videos", { sort: "growth", growth_days: 7, limit: 6 }), + // auto：服务端按 analytics.home_growth_days 解析；默认先试 7 天再降到 1 天 + api("/api/videos", { sort: "growth", growth_days: "auto", limit: 6 }), ]); const fans = overview.fans; @@ -59,6 +60,13 @@ export async function renderHome(root) { const pending = overview.counts?.candidates_pending || 0; // 候选提示仅对已登录（或未启用鉴权）用户可见，访客不展示运行期信息。 const showCandidates = pending > 0 && canAccessProtected(); + const growthDays = Number(growth.growth_days) || 1; + const growthLabel = `近 ${growthDays} 日`; + const growthEmptyTitle = growth.reason + || `近 ${growthDays} 日增长暂不可用`; + const growthEmptyHint = growthDays <= 1 + ? "当前快照不足，完成至少 2 天快照后即可显示播放增量。" + : `需要至少 ${growthDays + 1} 天快照才能计算近 ${growthDays} 日增长。`; root.innerHTML = `
@@ -101,12 +109,12 @@ export async function renderHome(root) {
-

增长动态

- 增长榜 +

增长动态 · ${escapeHtml(growthLabel)}

+ 增长榜
${growth.available && growth.items.length ? `
${growth.items.map((item, index) => growthCard(item, index)).join("")}
` - : emptyState("需要至少 2 天快照", growth.reason || "当前只有 1 个数据点，明天开始显示播放增量。")} + : emptyState(growthEmptyTitle, growthEmptyHint)}
diff --git a/reaction_board/web/static/views/system.js b/reaction_board/web/static/views/system.js index 85b7fd5..29349a6 100644 --- a/reaction_board/web/static/views/system.js +++ b/reaction_board/web/static/views/system.js @@ -38,6 +38,7 @@ export async function renderSystem(root) { const health = system.snapshot_health || { total: 0, ok: 0, failed: 0 }; const counts = system.counts || {}; const next = system.next_runs || {}; + const running = system.running || {}; const visitExtra = visitStats ? `近 ${visitStats.days} 天 · ${visitStats.pending ? "含未落库缓冲" : "已落库"}` @@ -92,12 +93,13 @@ export async function renderSystem(root) {
${sectionHeader("预计下次运行")} -
+
咻咻满粉丝数更新${running.fans ? "运行中" : escapeHtml(formatDateTime(next.fans))}
视频数据快照 + UP 信息${running.snapshot ? "运行中" : escapeHtml(formatDateTime(next.snapshot))}
+
新视频小时同步${running.snapshot_recent ? "运行中" : escapeHtml(formatDateTime(next.snapshot_recent))}
新视频搜索${running.discover || running.discovery ? "运行中" : escapeHtml(formatDateTime(next.discover))}
-

进行中显示「运行中」；空闲时按该任务上次完成时间推算下次运行，不按墙钟抢跑。

+

进行中显示「运行中」；空闲时按该任务上次完成时间推算下次运行，不按墙钟抢跑。新视频小时同步与全站快照互斥。

diff --git a/tests/test_config.py b/tests/test_config.py index 5863793..f68928d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -139,6 +139,9 @@ def test_analytics_defaults_and_validation(): assert cfg.analytics.min_view_for_ranking == 2000 assert cfg.analytics.video_shrink_minutes == 60 assert cfg.analytics.shrink_minutes is None + assert cfg.analytics.home_growth_days == "auto" + assert cfg.schedule.recent_snapshot_minutes == 60 + assert cfg.schedule.recent_video_days == 2 data = base_config() data["analytics"] = { @@ -147,6 +150,14 @@ def test_analytics_defaults_and_validation(): "min_view_for_ranking": 100, "video_shrink_minutes": 30, "shrink_minutes": 90, + "home_growth_days": 1, + } + data["schedule"] = { + "discover_minutes": 30, + "snapshot_at": "00:01", + "candidate_ttl_days": 7, + "recent_snapshot_minutes": 15, + "recent_video_days": 3, } write(path, data) cfg = load_config(path, env={}) @@ -155,6 +166,14 @@ def test_analytics_defaults_and_validation(): assert cfg.analytics.min_view_for_ranking == 100 assert cfg.analytics.video_shrink_minutes == 30 assert cfg.analytics.shrink_minutes == 90 + assert cfg.analytics.home_growth_days == 1 + assert cfg.schedule.recent_snapshot_minutes == 15 + assert cfg.schedule.recent_video_days == 3 + + data = base_config() + data["analytics"] = {"home_growth_days": 7} + write(path, data) + assert load_config(path, env={}).analytics.home_growth_days == 7 for bad, hint in ( ({"min_duration_seconds": -1}, "min_duration_seconds"), @@ -163,6 +182,9 @@ def test_analytics_defaults_and_validation(): ({"video_shrink_minutes": -1}, "video_shrink_minutes"), ({"shrink_minutes": 0}, "shrink_minutes"), ({"shrink_minutes": "auto"}, "shrink_minutes"), + ({"home_growth_days": 3}, "home_growth_days"), + ({"home_growth_days": "7"}, "home_growth_days"), + ({"home_growth_days": 0}, "home_growth_days"), ): data = base_config() data["analytics"] = bad @@ -243,12 +265,38 @@ def test_web_track_defaults_and_validation(): raise AssertionError("web.track 必须是对象") +def test_schedule_recent_validation(): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "config.json" + for bad, hint in ( + ({"recent_snapshot_minutes": 0}, "recent_snapshot_minutes"), + ({"recent_video_days": 0}, "recent_video_days"), + ({"recent_snapshot_minutes": -5}, "recent_snapshot_minutes"), + ): + data = base_config() + data["schedule"] = { + "discover_minutes": 60, + "snapshot_at": "00:01", + "candidate_ttl_days": 7, + **bad, + } + write(path, data) + try: + load_config(path, env={}) + except ConfigError as exc: + assert hint in str(exc), (bad, str(exc)) + else: + raise AssertionError(f"非法 schedule 必须拒绝: {bad}") + + + def main(): test_dual_accounts_and_env_override() test_flat_and_missing_account_rejected() test_validation_require_read_auth_and_hot_reload() test_invalid_values() test_analytics_defaults_and_validation() + test_schedule_recent_validation() test_web_track_defaults_and_validation() print("OK: 双账号配置加载、拒绝 flat 格式、ENV 覆盖、热加载与 analytics/web.track 测试全部通过") diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index be18b35..e0a2795 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -436,6 +436,100 @@ def test_backup_failure_retries_without_api(): db.close() +def test_run_recent_videos_refreshes_without_completion_markers(): + """小时任务只刷新近窗新片，不写完成日 / 备份。""" + clock = 1_800_000_000 + with tempfile.TemporaryDirectory() as tmp: + transport = SnapshotTransport({ + 1: view_for(1, view=111), + 2: view_for(2, view=222), + 3: view_for(3, view=333), + }) + db, repo, service = make_service(tmp, transport, track_fans=()) + try: + # 1/2 在墙钟近 2 天内；3 为旧片 + add_video(repo, 1) + add_video(repo, 2) + add_video(repo, 3) + repo.db.write(lambda conn: conn.executemany( + "UPDATE video SET pubdate = ? WHERE aid = ?", + [ + (clock - 3600, 1), + (clock - 86400, 2), + (clock - 10 * 86400, 3), + ], + )) + # 预置同日 ok=1，验证小时任务可覆盖刷新 + repo.upsert_snapshot( + DATE, 1, + metrics={"view": 1, "like": 0, "coin": 0, "favorite": 0, + "share": 0, "reply": 0, "danmaku": 0}, + ok=True, err=None, captured_at=clock - 100, + ) + assert repo.kv_get(service.completed_key(DATE)) is None + + summary = service.run_recent_videos(date=DATE, recent_days=2) + assert summary.total == 2 + assert summary.success == 2 + assert summary.failed == 0 + assert set(repo.video_aids_recent_for_snapshot( + since_pubdate=clock - 2 * 86400 + )) == {1, 2} + + row = repo.db.read(lambda conn: dict(conn.execute( + "SELECT view, ok, captured_at FROM snapshot WHERE stat_date=? AND aid=1", + (DATE,), + ).fetchone())) + assert row["view"] == 111 + assert row["ok"] == 1 + assert row["captured_at"] == clock + + # 完成日 / 备份标记保持原样 + assert repo.kv_get(service.completed_key(DATE)) is None + assert repo.kv_get("snapshot:last_completed") is None + assert repo.kv_get(service.backup_key(DATE)) is None + assert transport.count("/web-interface/view") == 2 + + # 全量任务随后才写完成日；且已 ok=1 的跳过 + full = service.run_videos(date=DATE) + assert full.skipped >= 2 # 1、2 已被小时任务写过 + assert full.success >= 1 # 至少旧片 3 + assert repo.kv_get(service.completed_key(DATE)) == "1" + assert repo.kv_get("snapshot:last_completed") == DATE + finally: + db.close() + + +def test_full_snapshot_skips_ok_rows_including_hourly_refresh(): + """00:01 全量：同日 ok=1（含小时任务刚刷）直接跳过。""" + clock = 1_800_000_000 + with tempfile.TemporaryDirectory() as tmp: + transport = SnapshotTransport({1: view_for(1), 2: view_for(2)}) + db, repo, service = make_service(tmp, transport, track_fans=()) + try: + add_video(repo, 1) + add_video(repo, 2) + repo.upsert_snapshot( + DATE, 1, + metrics={"view": 9, "like": 0, "coin": 0, "favorite": 0, + "share": 0, "reply": 0, "danmaku": 0}, + ok=True, err=None, captured_at=clock - 30, # 「1 小时内」 + ) + result = service.run_videos(date=DATE) + assert result.skipped == 1 + assert result.success == 1 + # aid=1 保持小时任务写入的 view=9，未被全量覆盖 + row = repo.db.read(lambda conn: dict(conn.execute( + "SELECT view FROM snapshot WHERE stat_date=? AND aid=1", + (DATE,), + ).fetchone())) + assert row["view"] == 9 + assert transport.count("/web-interface/view") == 1 + finally: + db.close() + + + def main(): test_snapshot_success_resume_and_force() test_resume_skips_ok_and_retries_failed() @@ -449,6 +543,8 @@ def main(): test_fan_resume_preserves_successful_value() test_force_interrupt_leaves_incomplete_marker() test_backup_failure_retries_without_api() + test_run_recent_videos_refreshes_without_completion_markers() + test_full_snapshot_skips_ok_rows_including_hourly_refresh() print("OK: 每日快照、断点续跑、失败占位与粉丝采集测试全部通过") diff --git a/tests/test_web_queries.py b/tests/test_web_queries.py index c905446..a5e1ca1 100644 --- a/tests/test_web_queries.py +++ b/tests/test_web_queries.py @@ -124,7 +124,8 @@ def test_growth_and_cold_start(): cold = q.list_videos(sort="growth", growth_days=2) assert cold["available"] is False assert cold["items"] == [] - assert "两天" in cold["reason"] + assert "至少 3 天快照" in cold["reason"] + assert cold["growth_days"] == 2 finally: db.close() @@ -361,10 +362,15 @@ def test_fans_candidates_system_and_export(): assert system["errors"] == [] assert system["preview"] is False assert system["running"] == {} - assert set(system["next_runs"]) >= {"fans", "snapshot", "discover"} + assert set(system["next_runs"]) >= { + "fans", "snapshot", "discover", "snapshot_recent", + } assert system["next_runs"]["snapshot"] > 0 assert system["next_runs"]["fans"] > 0 assert system["next_runs"]["discover"] > 0 + assert system["next_runs"]["snapshot_recent"] > 0 + assert system["schedule"]["recent_snapshot_minutes"] == 60 + assert system["schedule"]["recent_video_days"] == 2 finally: db.close() @@ -483,6 +489,7 @@ def test_about_payload(): "min_view_for_ranking": 2000, "video_shrink_minutes": 60, "shrink_minutes": None, + "home_growth_days": "auto", } assert about["heat_weights"]["view"] == 0.30 assert about["followers"]["videos_total"] == 3 @@ -553,6 +560,65 @@ def test_stats_query(): db.close() + +def test_growth_auto_falls_back_to_one_day(): + """3 天快照撑不起 7 日增长；auto 应降到 1 日。""" + import json + import os + import time + + with tempfile.TemporaryDirectory() as tmp: + db, q = setup(tmp) + try: + auto = q.list_videos(sort="growth", growth_days="auto", page_size=6) + assert auto["available"] is True + assert auto["growth_days"] == 1 + assert {item["aid"] for item in auto["items"]} == {101, 102} + first = next(item for item in auto["items"] if item["aid"] == 101) + assert first["growth"]["delta"] == 100 # 相邻日 + + forced7 = q.list_videos(sort="growth", growth_days=7) + assert forced7["available"] is False + assert "至少 8 天快照" in forced7["reason"] + + # 配置钉死 7 天：auto 不再降级 + path = Path(tmp) / "config.json" + data = json.loads(path.read_text(encoding="utf-8")) + data.setdefault("analytics", {})["home_growth_days"] = 7 + path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") + target = max(path.stat().st_mtime_ns, time.time_ns()) + 1_000_000 + os.utime(path, ns=(target, target)) + DASHBOARD_CACHE.clear() + pinned = q.list_videos(sort="growth", growth_days="auto") + assert pinned["growth_days"] == 7 + assert pinned["available"] is False + finally: + db.close() + + +def test_snapshot_recent_next_run_follows_completion(): + import json + + with tempfile.TemporaryDirectory() as tmp: + db, q = setup(tmp) + try: + repo = Repository(db) + finished = FIXED_NOW - 10 + repo.kv_set( + "task:last:snapshot_recent", + json.dumps({ + "name": "snapshot_recent", "ok": True, + "started_at": finished - 5, "finished_at": finished, + "error": None, "summary": {}, + }), + ) + DASHBOARD_CACHE.clear() + system = q.system() + assert system["next_runs"]["snapshot_recent"] == finished + 3600 + finally: + db.close() + + def main_test(): test_overview() test_video_listing_filters_and_sorting() @@ -568,6 +634,8 @@ def main_test(): test_fans_candidates_system_and_export() test_system_status_uses_completed_date_and_running_flags() test_system_next_runs_follow_last_completion_not_wall_clock() + test_growth_auto_falls_back_to_one_day() + test_snapshot_recent_next_run_follows_completion() print("OK: Web 查询层总览/榜单/详情/UP/时长效率/粉丝/候选/系统/访问统计测试全部通过") @@ -605,7 +673,9 @@ def test_system_status_uses_completed_date_and_running_flags(): system = q.system() assert "snapshot" in system["running"] assert system["running"]["snapshot"]["running"] is True - assert set(system["next_runs"]) >= {"fans", "snapshot", "discover"} + assert set(system["next_runs"]) >= { + "fans", "snapshot", "discover", "snapshot_recent", + } # 失败记录保留，完成日不变 repo.kv_set( diff --git a/tests/web_fixtures.py b/tests/web_fixtures.py index 9fcd9a8..73bf99c 100644 --- a/tests/web_fixtures.py +++ b/tests/web_fixtures.py @@ -50,6 +50,8 @@ def write_config( "discover_minutes": 60, "snapshot_at": "00:01", "candidate_ttl_days": 7, + "recent_snapshot_minutes": 60, + "recent_video_days": 2, }, }, ensure_ascii=False, indent=2), encoding="utf-8") return path -- 2.54.0 From 297d9279d83789a227c83806414eec5450c06430 Mon Sep 17 00:00:00 2001 From: bot-ci Date: Mon, 2 Nov 2026 10:05:00 +0800 Subject: [PATCH 2/2] fix(home): server-owned growth window; expose days MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Home omits growth_days; backend auto (or home_growth_days 1/7) picks the window. Response includes days alongside growth_days. Growth榜 link uses the returned days. Non-growth sorts treat omitted/auto as 7. --- reaction_board/web/queries.py | 43 ++++++++++++++----------- reaction_board/web/server.py | 7 ++-- reaction_board/web/static/views/home.js | 7 ++-- tests/test_web_queries.py | 11 +++++-- tests/test_web_server.py | 12 ++++++- 5 files changed, 52 insertions(+), 28 deletions(-) diff --git a/reaction_board/web/queries.py b/reaction_board/web/queries.py index 68f17cd..23937e1 100644 --- a/reaction_board/web/queries.py +++ b/reaction_board/web/queries.py @@ -699,7 +699,7 @@ class DashboardQueries: sort: str = "heat", within_days: int | None = None, year: int | None = None, - growth_days: int | str = 7, + growth_days: int | str = "auto", folder_id: int | None = None, q: str | None = None, page: int = 1, @@ -734,7 +734,7 @@ class DashboardQueries: sort: str = "heat", within_days: int | None = None, year: int | None = None, - growth_days: int | str = 7, + growth_days: int | str = "auto", folder_id: int | None = None, q: str | None = None, page: int = 1, @@ -742,24 +742,29 @@ class DashboardQueries: ) -> dict[str, Any]: if sort not in VIDEO_SORTS: raise ValueError(f"不支持的排序: {sort}") + # auto：仅增长类排序由服务端选窗（配置 home_growth_days；默认先 7 再降 1）。 + # 首页应省略 growth_days，勿自行探查。其它排序把 auto 当作 7（兼容旧默认）。 if growth_days == "auto": - preferred = self.config_store.get().analytics.home_growth_days - if preferred in (1, 7): - growth_days = int(preferred) + if sort in {"growth", "growth_rate"}: + preferred = self.config_store.get().analytics.home_growth_days + if preferred in (1, 7): + growth_days = int(preferred) + else: + trial = self._list_videos_uncached( + sort=sort, + within_days=within_days, + year=year, + growth_days=7, + folder_id=folder_id, + q=q, + page=page, + page_size=page_size, + ) + if trial.get("available") and int(trial.get("total") or 0) > 0: + return trial + growth_days = 1 else: - trial = self._list_videos_uncached( - sort=sort, - within_days=within_days, - year=year, - growth_days=7, - folder_id=folder_id, - q=q, - page=page, - page_size=page_size, - ) - if trial.get("available") and int(trial.get("total") or 0) > 0: - return trial - growth_days = 1 + growth_days = 7 growth_days = int(growth_days) if growth_days < 1 or growth_days > 365: raise ValueError("growth_days 必须在 1~365 之间") @@ -942,6 +947,8 @@ class DashboardQueries: "within_days": within_days, "year": year, "growth_days": growth_days, + # days：与 growth_days 同步；首页 auto 时为 1|7，供前端链接/展示 + "days": growth_days, "page": page, "page_size": page_size, "total": total, diff --git a/reaction_board/web/server.py b/reaction_board/web/server.py index 45dc580..ba8f3f6 100644 --- a/reaction_board/web/server.py +++ b/reaction_board/web/server.py @@ -650,11 +650,10 @@ def _year_param(query: Mapping[str, list[str]]) -> int | None: def _growth_days_param(query: Mapping[str, list[str]]) -> int | str: - """榜单传 1~365；首页可传 auto（由查询层按 home_growth_days 解析）。""" + """省略或 auto：查询层按 home_growth_days 解析（默认 7→1）。 + 榜单应显式传 1~365。""" raw = _optional_str(query, "growth_days") - if raw is None: - return 7 - if raw == "auto": + if raw is None or raw == "auto": return "auto" try: value = int(raw) diff --git a/reaction_board/web/static/views/home.js b/reaction_board/web/static/views/home.js index 714215f..05acf2a 100644 --- a/reaction_board/web/static/views/home.js +++ b/reaction_board/web/static/views/home.js @@ -47,8 +47,8 @@ export async function renderHome(root) { api("/api/overview"), api("/api/videos", { sort: "published", within_days: 14, limit: 6 }), api("/api/videos", { sort: "heat", limit: 6 }), - // auto：服务端按 analytics.home_growth_days 解析；默认先试 7 天再降到 1 天 - api("/api/videos", { sort: "growth", growth_days: "auto", limit: 6 }), + // 不带天数：后端按 home_growth_days（默认 auto：有 7 日则 7，否则 1）决定，响应带 days + api("/api/videos", { sort: "growth", limit: 6 }), ]); const fans = overview.fans; @@ -60,7 +60,8 @@ export async function renderHome(root) { const pending = overview.counts?.candidates_pending || 0; // 候选提示仅对已登录（或未启用鉴权）用户可见，访客不展示运行期信息。 const showCandidates = pending > 0 && canAccessProtected(); - const growthDays = Number(growth.growth_days) || 1; + // 以后端返回的 days（与 growth_days 同步）为准，避免前端再探查 + const growthDays = Number(growth.days ?? growth.growth_days) || 1; const growthLabel = `近 ${growthDays} 日`; const growthEmptyTitle = growth.reason || `近 ${growthDays} 日增长暂不可用`; diff --git a/tests/test_web_queries.py b/tests/test_web_queries.py index a5e1ca1..f157eb3 100644 --- a/tests/test_web_queries.py +++ b/tests/test_web_queries.py @@ -106,6 +106,8 @@ def test_growth_and_cold_start(): try: growth = q.list_videos(sort="growth", growth_days=2) assert growth["available"] is True + assert growth["days"] == 2 + assert growth["growth_days"] == 2 assert {item["aid"] for item in growth["items"]} == {101, 102} first = next(item for item in growth["items"] if item["aid"] == 101) assert first["growth"]["available"] is True @@ -570,15 +572,19 @@ def test_growth_auto_falls_back_to_one_day(): with tempfile.TemporaryDirectory() as tmp: db, q = setup(tmp) try: - auto = q.list_videos(sort="growth", growth_days="auto", page_size=6) + # 省略 growth_days 与显式 auto 等价：后端决定窗口并回传 days + auto = q.list_videos(sort="growth", page_size=6) assert auto["available"] is True assert auto["growth_days"] == 1 + assert auto["days"] == 1 assert {item["aid"] for item in auto["items"]} == {101, 102} first = next(item for item in auto["items"] if item["aid"] == 101) assert first["growth"]["delta"] == 100 # 相邻日 forced7 = q.list_videos(sort="growth", growth_days=7) assert forced7["available"] is False + assert forced7["days"] == 7 + assert forced7["growth_days"] == 7 assert "至少 8 天快照" in forced7["reason"] # 配置钉死 7 天：auto 不再降级 @@ -589,8 +595,9 @@ def test_growth_auto_falls_back_to_one_day(): target = max(path.stat().st_mtime_ns, time.time_ns()) + 1_000_000 os.utime(path, ns=(target, target)) DASHBOARD_CACHE.clear() - pinned = q.list_videos(sort="growth", growth_days="auto") + pinned = q.list_videos(sort="growth") # omit → auto → 配置钉死 7 assert pinned["growth_days"] == 7 + assert pinned["days"] == 7 assert pinned["available"] is False finally: db.close() diff --git a/tests/test_web_server.py b/tests/test_web_server.py index 5d53fd2..cfb8872 100644 --- a/tests/test_web_server.py +++ b/tests/test_web_server.py @@ -108,8 +108,11 @@ def test_readonly_routes_and_static(): status, _, payload = running.request("/api/videos") assert status == 200 - items = json.loads(payload)["data"]["items"] + heat_payload = json.loads(payload)["data"] + items = heat_payload["items"] assert items and items[0]["aid"] == 102 + # 非增长排序：省略 growth_days 时 days/growth_days 回落为 7，不做 7→1 + assert heat_payload["days"] == heat_payload["growth_days"] == 7 status, _, payload = running.json("/api/videos?year=2026") assert status == 200 @@ -120,6 +123,13 @@ def test_readonly_routes_and_static(): assert status == 200 assert payload["data"]["year"] == 2026 + # 首页形态：省略 growth_days → 后端 auto，响应带 days/growth_days + status, _, payload = running.json("/api/videos?sort=growth&limit=6") + assert status == 200 + growth = payload["data"] + assert growth["days"] == growth["growth_days"] == 1 + assert growth["available"] is True + status, _, payload = running.request("/api/ups") assert status == 200 assert json.loads(payload)["data"]["sort"] == "efficiency" -- 2.54.0