feat: home growth auto(7→1) and hourly recent-video snapshot

- Add analytics.home_growth_days (auto|1|7); API growth_days=auto tries 7 then 1
- Homepage shows active window and growth-board link; clearer empty reason
- New snapshot_recent scheduler task (schedule.recent_snapshot_minutes/days)
- run_recent_videos refreshes new aids only; never touches completed/backup
- Full 00:01 job keeps ok=1 skip (covers hourly refresh within 1h); mutex with recent
This commit is contained in:
bot-ci committed 2026-11-02 10:00:02 +08:00
1 parent 04716aec1b
commit 9163f47b76
15 files changed
+496 -22

No files matched your search

+56
View File
@@ -163,6 +163,13 @@ def _clear_task_running(repo: Repository, name: str) -> None:
pass
def _is_task_running(repo: Repository, name: str) -> bool:
try:
return bool(repo.kv_get(_task_running_key(name)))
except Exception: # noqa: BLE001
return False
def _record_task(
repo: Repository,
name: str,
@@ -547,6 +554,16 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
def run_snapshot():
started = int(time.time())
if _is_task_running(repo, "snapshot_recent"):
logger.info(
"snapshot.skip", reason="snapshot_recent_running",
)
_record_task(
repo, "snapshot", ok=True, started_at=started,
finished_at=int(time.time()),
payload={"skipped": True, "reason": "snapshot_recent_running"},
)
return None
_mark_task_running(repo, "snapshot", started_at=started)
try:
summary = snapshot_service.run_if_needed()
@@ -567,6 +584,39 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
logger.info("snapshot.summary", **payload)
return summary
def run_snapshot_recent():
started = int(time.time())
if _is_task_running(repo, "snapshot"):
logger.info(
"snapshot.recent_skip", reason="snapshot_running",
)
_record_task(
repo, "snapshot_recent", ok=True, started_at=started,
finished_at=int(time.time()),
payload={"skipped": True, "reason": "snapshot_running"},
)
return None
_mark_task_running(repo, "snapshot_recent", started_at=started)
try:
summary = snapshot_service.run_recent_videos()
except Exception as exc: # noqa: BLE001
_record_task(
repo, "snapshot_recent", ok=False, started_at=started,
finished_at=int(time.time()), error=str(exc),
)
raise
payload = summary.as_dict()
ok = summary.failed == 0
error = ";".join(summary.errors) if summary.errors else None
_record_task(
repo, "snapshot_recent", ok=ok, started_at=summary.started_at,
finished_at=summary.finished_at or int(time.time()),
payload=payload, error=error,
)
last["snapshot_recent"] = summary
logger.info("snapshot.recent_summary", **payload)
return summary
def run_fans():
started = int(time.time())
_mark_task_running(repo, "fans", started_at=started)
@@ -601,6 +651,7 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
snapshot_at = f"{hh:02d}:{mm:02d}"
recent_minutes = int(store.get().schedule.recent_snapshot_minutes)
scheduler = Scheduler(
[
ScheduledTask(
@@ -617,6 +668,11 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
next_run=0.0,
next_run_fn=lambda started: next_daily_run(started, snapshot_at),
),
ScheduledTask(
"snapshot_recent",
float(recent_minutes * 60),
run_snapshot_recent,
),
ScheduledTask(
"discovery",
float(cfg.schedule.discover_minutes * 60),
+24
View File
@@ -49,6 +49,7 @@ DEFAULT_ANALYTICS: dict[str, Any] = {
"min_view_for_ranking": 2000,
"video_shrink_minutes": 60,
"shrink_minutes": None,
"home_growth_days": "auto",
}
DEFAULT_CONFIG: dict[str, Any] = {
@@ -84,6 +85,8 @@ DEFAULT_CONFIG: dict[str, Any] = {
"discover_minutes": 60,
"snapshot_at": "00:01",
"candidate_ttl_days": 7,
"recent_snapshot_minutes": 60,
"recent_video_days": 2,
},
"discover": {
"keywords": ["咻咻满"],
@@ -177,6 +180,8 @@ class ScheduleConfig:
discover_minutes: int
snapshot_at: str
candidate_ttl_days: int
recent_snapshot_minutes: int
recent_video_days: int
@dataclass(frozen=True)
@@ -200,6 +205,7 @@ class AnalyticsConfig:
min_view_for_ranking: int
video_shrink_minutes: int
shrink_minutes: int | None
home_growth_days: str | int
@dataclass(frozen=True)
@@ -456,6 +462,16 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
"schedule.candidate_ttl_days",
minimum=1,
),
recent_snapshot_minutes=_as_int(
sched_raw.get("recent_snapshot_minutes", 60),
"schedule.recent_snapshot_minutes",
minimum=1,
),
recent_video_days=_as_int(
sched_raw.get("recent_video_days", 2),
"schedule.recent_video_days",
minimum=1,
),
)
notify_raw = data["notify"]
@@ -533,6 +549,13 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
if not isinstance(analytics_raw, Mapping):
raise ConfigError("analytics 必须是对象")
shrink_raw = analytics_raw.get("shrink_minutes")
home_growth_raw = analytics_raw.get("home_growth_days", "auto")
if home_growth_raw == "auto":
home_growth_days: str | int = "auto"
elif home_growth_raw in (1, 7):
home_growth_days = int(home_growth_raw)
else:
raise ConfigError('analytics.home_growth_days 必须是 "auto"、1 或 7')
analytics = AnalyticsConfig(
min_duration_seconds=_as_int(
analytics_raw.get("min_duration_seconds", 600),
@@ -559,6 +582,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
shrink_raw, "analytics.shrink_minutes", minimum=1
)
),
home_growth_days=home_growth_days,
)
return AppConfig(
+13
View File
@@ -637,6 +637,19 @@ class Repository:
return [int(row["aid"]) for row in rows]
return self.db.read(run)
def video_aids_recent_for_snapshot(self, *, since_pubdate: int) -> list[int]:
"""近 N 天新视频:按 pubdate 墙钟窗口(since_pubdate = now - days*86400)。"""
def run(conn: sqlite3.Connection) -> list[int]:
rows = conn.execute(
"SELECT aid FROM video "
"WHERE status IN ('active', 'unfaved') "
"AND pubdate IS NOT NULL AND pubdate >= ? "
"ORDER BY aid",
(int(since_pubdate),),
).fetchall()
return [int(row["aid"]) for row in rows]
return self.db.read(run)
def pending_snapshot_aids(self, stat_date: str) -> list[int]:
"""当天仍缺成功快照的追踪视频;用于新入库/恢复后的同日补采。"""
def run(conn: sqlite3.Connection) -> list[int]:
+96 -1
View File
@@ -4,7 +4,8 @@
规则(DEVELOPMENT.md §7.4):
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``;
* 成功写入七项指标;``ok=1`` 的同日快照直接跳过(断点续跑);
* 成功写入七项指标;``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
* ``run_recent_videos`` 只刷新近 N 天新片的当日行,不改完成日 / 备份;
* 视频不可访问时标记 ``removed``;
* 其他错误最多重试 3 次,仍失败写 ``ok=0`` 占位,不写假零值;
* 对 ``track_fans`` 逐个请求 ``relation/stat``(独立任务,不影响视频完成日);
@@ -198,6 +199,8 @@ class SnapshotService:
for index, aid in enumerate(videos, 1):
existing = self.repo.get_snapshot(stat_date, aid)
if existing is not None and int(existing.get("ok") or 0) == 1 and not force:
# 断点续跑:同日已成功则跳过。小时任务刷新过的行也是 ok=1,
# 自然覆盖「1 小时内 captured_at 已更新则不再次更新」的要求。
summary.skipped += 1
continue
@@ -269,6 +272,98 @@ class SnapshotService:
)
return summary
def run_recent_videos(
self,
*,
date: str | None = None,
recent_days: int | None = None,
) -> SnapshotSummary:
"""刷新近 N 天新视频的当日快照;不碰完成日标记与日备份。
「近 N 天」按 pubdate 墙钟窗口:``now - recent_video_days * 86400``,
不是上海日历日差。允许覆盖同日已有 ``ok=1`` 行以便小时级刷新。
"""
cfg = self.config_store.get()
client = self._client()
now = self._now()
stat_date = date or shanghai_date(now)
if not is_date(stat_date):
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
days = (
int(cfg.schedule.recent_video_days)
if recent_days is None else int(recent_days)
)
if days < 1:
raise ValueError("recent_days 必须 >= 1")
# 墙钟窗口,而非按上海日历截断到日界。
since_pubdate = now - days * 86400
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
self._log(
"info", "snapshot.recent_start",
stat_date=stat_date, recent_days=days, since_pubdate=since_pubdate,
)
videos = self.repo.video_aids_recent_for_snapshot(since_pubdate=since_pubdate)
summary.total = len(videos)
for index, aid in enumerate(videos, 1):
stat, error, unavailable = self._fetch_view(client, aid)
if unavailable:
summary.removed += 1
self.repo.set_video_status(aid, "removed", now=now)
self._log(
"warning", "snapshot.recent_removed",
stat_date=stat_date, aid=aid, error=error,
)
elif stat is not None:
self.repo.upsert_snapshot(
stat_date,
aid,
metrics=stat.metrics(),
ok=True,
err=None,
captured_at=self._now(),
)
if getattr(stat, "up_face", None):
self.repo.apply_video_up_face(aid, stat.up_face)
summary.success += 1
else:
summary.failed += 1
self.repo.upsert_snapshot(
stat_date,
aid,
metrics=None,
ok=False,
err=error,
captured_at=self._now(),
)
summary.errors.append(f"aid {aid}: {error}")
self._log(
"error", "snapshot.recent_failed",
stat_date=stat_date, aid=aid, error=error,
)
if self.progress_every > 0 and (
index % self.progress_every == 0 or index == summary.total
):
self._log(
"info", "snapshot.recent_progress",
stat_date=stat_date, done=index, total=summary.total,
success=summary.success, failed=summary.failed,
removed=summary.removed,
)
# 刻意不写 snapshot:completed / last_completed / 日备份:完成日只归全量任务。
summary.completed = summary.failed == 0
if summary.success or summary.removed or summary.failed:
self.repo.bump_cache_version("videos")
summary.finished_at = self._now()
self._log(
"info" if summary.completed else "warning",
"snapshot.recent_done",
**summary.as_dict(),
)
return summary
def run_fans_if_needed(
self, *, date: str | None = None, force: bool = False
) -> SnapshotSummary:
+36 -5
View File
@@ -699,7 +699,7 @@ class DashboardQueries:
sort: str = "heat",
within_days: int | None = None,
year: int | None = None,
growth_days: int = 7,
growth_days: int | str = 7,
folder_id: int | None = None,
q: str | None = None,
page: int = 1,
@@ -734,7 +734,7 @@ class DashboardQueries:
sort: str = "heat",
within_days: int | None = None,
year: int | None = None,
growth_days: int = 7,
growth_days: int | str = 7,
folder_id: int | None = None,
q: str | None = None,
page: int = 1,
@@ -742,6 +742,25 @@ class DashboardQueries:
) -> dict[str, Any]:
if sort not in VIDEO_SORTS:
raise ValueError(f"不支持的排序: {sort}")
if growth_days == "auto":
preferred = self.config_store.get().analytics.home_growth_days
if preferred in (1, 7):
growth_days = int(preferred)
else:
trial = self._list_videos_uncached(
sort=sort,
within_days=within_days,
year=year,
growth_days=7,
folder_id=folder_id,
q=q,
page=page,
page_size=page_size,
)
if trial.get("available") and int(trial.get("total") or 0) > 0:
return trial
growth_days = 1
growth_days = int(growth_days)
if growth_days < 1 or growth_days > 365:
raise ValueError("growth_days 必须在 1~365 之间")
page = max(1, int(page))
@@ -900,7 +919,9 @@ class DashboardQueries:
reason = None
if sort in {"growth", "growth_rate"} and not growth_available:
available = False
reason = "需要至少两天的快照才能计算增长"
reason = (
f"近 {growth_days} 日增长需要至少 {growth_days + 1} 天快照"
)
duration_filter = None
if sort in duration_sorts:
duration_filter = {
@@ -1571,6 +1592,7 @@ class DashboardQueries:
"min_view_for_ranking": cfg.analytics.min_view_for_ranking,
"video_shrink_minutes": cfg.analytics.video_shrink_minutes,
"shrink_minutes": cfg.analytics.shrink_minutes,
"home_growth_days": cfg.analytics.home_growth_days,
},
"heat_weights": {
metric: float(value)
@@ -1747,23 +1769,30 @@ class DashboardQueries:
running = self._task_running(conn)
now = self._now()
discover_interval = int(cfg.schedule.discover_minutes) * 60
recent_interval = int(cfg.schedule.recent_snapshot_minutes) * 60
next_runs = {
"fans": int(next_daily_run(now, cfg.schedule.snapshot_at)),
"snapshot": int(next_daily_run(now, cfg.schedule.snapshot_at)),
"discover": int(now + discover_interval),
"snapshot_recent": int(now + recent_interval),
}
for name, key in (
("fans", "fans"),
("snapshot", "snapshot"),
("discover", "discovery"),
("snapshot_recent", "snapshot_recent"),
):
record = tasks.get(key)
if not (isinstance(record, dict) and record.get("finished_at")):
continue
finished_at = int(record["finished_at"])
if name == "discover":
if name in {"discover", "snapshot_recent"}:
# 只按上次完成推下一档;进行中由 running 展示,不按墙钟抢跑。
next_runs[name] = finished_at + discover_interval
interval = (
recent_interval if name == "snapshot_recent"
else discover_interval
)
next_runs[name] = finished_at + interval
else:
next_runs[name] = int(
next_daily_run(finished_at, cfg.schedule.snapshot_at)
@@ -1805,6 +1834,8 @@ class DashboardQueries:
"discover_minutes": cfg.schedule.discover_minutes,
"snapshot_at": cfg.schedule.snapshot_at,
"candidate_ttl_days": cfg.schedule.candidate_ttl_days,
"recent_snapshot_minutes": cfg.schedule.recent_snapshot_minutes,
"recent_video_days": cfg.schedule.recent_video_days,
},
"telegram_enabled": bool(cfg.telegram.enabled),
}
+24 -2
View File
@@ -373,7 +373,7 @@ border-color:#3a2c22;color:#f3e9dc}button{background:#e9b97c;color:#2a1a10}}
sort=_param(query, "sort", "heat"),
within_days=_optional_int(query, "within_days"),
year=_year_param(query),
growth_days=_int_param(query, "growth_days", 7, 1, 365),
growth_days=_growth_days_param(query),
folder_id=_optional_int(query, "folder"),
q=_optional_str(query, "q"),
page=_int_param(query, "page", 1, 1, 1_000_000),
@@ -422,7 +422,7 @@ border-color:#3a2c22;color:#f3e9dc}button{background:#e9b97c;color:#2a1a10}}
sort=_param(query, "sort", "heat"),
within_days=_optional_int(query, "within_days"),
year=_year_param(query),
growth_days=_int_param(query, "growth_days", 7, 1, 365),
growth_days=_growth_days_param(query),
folder_id=_optional_int(query, "folder"),
q=_optional_str(query, "q"),
page=_int_param(query, "page", 1, 1, 1_000_000),
@@ -649,6 +649,28 @@ def _year_param(query: Mapping[str, list[str]]) -> int | None:
return value
def _growth_days_param(query: Mapping[str, list[str]]) -> int | str:
"""榜单传 1~365;首页可传 auto(由查询层按 home_growth_days 解析)。"""
raw = _optional_str(query, "growth_days")
if raw is None:
return 7
if raw == "auto":
return "auto"
try:
value = int(raw)
except ValueError as exc:
raise ApiError(
HTTPStatus.BAD_REQUEST, "bad_request",
"growth_days 必须是整数或 auto",
) from exc
if value < 1 or value > 365:
raise ApiError(
HTTPStatus.BAD_REQUEST, "bad_request",
"growth_days 必须在 1~365 之间",
)
return value
def _int_param(
query: Mapping[str, list[str]],
name: str,
+2 -1
View File
@@ -510,6 +510,7 @@ a.live-chip:hover { border-color: color-mix(in srgb, var(--accent) 55%, var(--bo
.status-grid strong { display: block; margin-top: 2px; font-size: 12.5px; font-weight: 600; }
/* 预计下次运行等 3 项:PC 与移动端都保持一行 */
.status-grid.status-row-3 { grid-template-columns: repeat(3, minmax(0, 1fr)); }
.status-grid.status-row-4 { grid-template-columns: repeat(2, minmax(0, 1fr)); }
.segmented {
display: inline-flex;
@@ -993,4 +994,4 @@ a.live-chip:hover { border-color: color-mix(in srgb, var(--accent) 55%, var(--bo
.about-page .panel h2 { font-size: 15px; margin-bottom: 8px; }
.about-list { margin: 0; padding-left: 18px; display: grid; gap: 6px; font-size: 13px; line-height: 1.55; }
.about-list code { padding: 1px 4px; border-radius: 4px; background: var(--accent-soft); font-size: 12px; }
.about-list.about-weights { list-style: none; padding-left: 0; grid-template-columns: repeat(2, minmax(0, 1fr)); }
.about-list.about-weights { list-style: none; padding-left: 0; grid-template-columns: repeat(2, minmax(0, 1fr)); }
+12 -4
View File
@@ -47,7 +47,8 @@ export async function renderHome(root) {
api("/api/overview"),
api("/api/videos", { sort: "published", within_days: 14, limit: 6 }),
api("/api/videos", { sort: "heat", limit: 6 }),
api("/api/videos", { sort: "growth", growth_days: 7, limit: 6 }),
// auto:服务端按 analytics.home_growth_days 解析;默认先试 7 天再降到 1 天
api("/api/videos", { sort: "growth", growth_days: "auto", limit: 6 }),
]);
const fans = overview.fans;
@@ -59,6 +60,13 @@ export async function renderHome(root) {
const pending = overview.counts?.candidates_pending || 0;
// 候选提示仅对已登录(或未启用鉴权)用户可见,访客不展示运行期信息。
const showCandidates = pending > 0 && canAccessProtected();
const growthDays = Number(growth.growth_days) || 1;
const growthLabel = `近 ${growthDays} 日`;
const growthEmptyTitle = growth.reason
|| `近 ${growthDays} 日增长暂不可用`;
const growthEmptyHint = growthDays <= 1
? "当前快照不足,完成至少 2 天快照后即可显示播放增量。"
: `需要至少 ${growthDays + 1} 天快照才能计算近 ${growthDays} 日增长。`;
root.innerHTML = `<section class="page home-page">
<header class="page-head home-head">
@@ -101,12 +109,12 @@ export async function renderHome(root) {
<section class="panel">
<div class="section-head">
<h2>增长动态</h2>
<a class="text-link" href="/videos?sort=growth&growth_days=7">增长榜</a>
<h2>增长动态 · ${escapeHtml(growthLabel)}</h2>
<a class="text-link" href="/videos?sort=growth&growth_days=${growthDays}">增长榜</a>
</div>
${growth.available && growth.items.length
? `<div class="media-grid">${growth.items.map((item, index) => growthCard(item, index)).join("")}</div>`
: emptyState("需要至少 2 天快照", growth.reason || "当前只有 1 个数据点,明天开始显示播放增量。")}
: emptyState(growthEmptyTitle, growthEmptyHint)}
</section>
<section class="panel">
+4 -2
View File
@@ -38,6 +38,7 @@ export async function renderSystem(root) {
const health = system.snapshot_health || { total: 0, ok: 0, failed: 0 };
const counts = system.counts || {};
const next = system.next_runs || {};
const running = system.running || {};
const visitExtra = visitStats
? `<span class="section-note">近 ${visitStats.days} 天 · ${visitStats.pending ? "含未落库缓冲" : "已落库"}</span>`
@@ -92,12 +93,13 @@ export async function renderSystem(root) {
<section class="panel">
${sectionHeader("预计下次运行")}
<div class="status-grid status-row-3">
<div class="status-grid status-row-4">
<div><span>咻咻满粉丝数更新</span><strong>${running.fans ? "运行中" : escapeHtml(formatDateTime(next.fans))}</strong></div>
<div><span>视频数据快照 + UP 信息</span><strong>${running.snapshot ? "运行中" : escapeHtml(formatDateTime(next.snapshot))}</strong></div>
<div><span>新视频小时同步</span><strong>${running.snapshot_recent ? "运行中" : escapeHtml(formatDateTime(next.snapshot_recent))}</strong></div>
<div><span>新视频搜索</span><strong>${running.discover || running.discovery ? "运行中" : escapeHtml(formatDateTime(next.discover))}</strong></div>
</div>
<p class="section-note">进行中显示「运行中」;空闲时按该任务上次完成时间推算下次运行,不按墙钟抢跑。</p>
<p class="section-note">进行中显示「运行中」;空闲时按该任务上次完成时间推算下次运行,不按墙钟抢跑。新视频小时同步与全站快照互斥。</p>
</section>
<section class="panel">