feat: EOD 23:15 seal + midnight seed shell for video/fans

Move daily snapshot/fans to 23:15 and add 00:01 seed that clones
last_completed ok=1 rows into today without advancing completed.
Evening runs force through seed markers; growth previous is per-aid
adjacent. Home shows short-update clock + 全站数据 date, and 收录视频
counts.
This commit is contained in:
bot-ci committed 2026-11-02 12:38:55 +08:00
1 parent 374b138041
commit 8a9cb21408
17 files changed
+520 -39

No files matched your search

+42 -2
View File
@@ -566,7 +566,12 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
return None
_mark_task_running(repo, "snapshot", started_at=started)
try:
summary = snapshot_service.run_if_needed()
# 晚间全量:若当日是午夜 seed 壳,必须 force 打穿 ok=1 克隆行。
today = shanghai_date()
force_seed = repo.kv_get(
SnapshotService.seeded_key(today)
) == "1"
summary = snapshot_service.run_if_needed(force=force_seed)
except Exception as exc: # noqa: BLE001
_record_task(
repo, "snapshot", ok=False, started_at=started,
@@ -621,7 +626,11 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
started = int(time.time())
_mark_task_running(repo, "fans", started_at=started)
try:
summary = snapshot_service.run_fans_if_needed()
today = shanghai_date()
force_seed = repo.kv_get(
SnapshotService.fans_seeded_key(today)
) == "1"
summary = snapshot_service.run_fans_if_needed(force=force_seed)
except Exception as exc: # noqa: BLE001
_record_task(
repo, "fans", ok=False, started_at=started,
@@ -642,6 +651,27 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
logger.info("fans.summary", **payload)
return summary
def run_snapshot_seed():
started = int(time.time())
_mark_task_running(repo, "snapshot_seed", started_at=started)
try:
summary = snapshot_service.run_seed_from_last_completed()
except Exception as exc: # noqa: BLE001
_record_task(
repo, "snapshot_seed", ok=False, started_at=started,
finished_at=int(time.time()), error=str(exc),
)
raise
payload = summary.as_dict()
_record_task(
repo, "snapshot_seed", ok=True, started_at=summary.started_at,
finished_at=summary.finished_at or int(time.time()),
payload=payload,
)
last["snapshot_seed"] = summary
logger.info("snapshot.seed_summary", **payload)
return summary
def run_config_watch():
before = store.revision
store.get()
@@ -651,9 +681,18 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
snapshot_at = f"{hh:02d}:{mm:02d}"
sh, sm = parse_hhmm(cfg.schedule.seed_at)
seed_at = f"{sh:02d}:{sm:02d}"
recent_minutes = int(store.get().schedule.recent_snapshot_minutes)
scheduler = Scheduler(
[
ScheduledTask(
"snapshot_seed",
86400.0,
run_snapshot_seed,
next_run=0.0,
next_run_fn=lambda started: next_daily_run(started, seed_at),
),
ScheduledTask(
"snapshot",
86400.0,
@@ -707,6 +746,7 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
logger.info(
"scheduler.start",
snapshot_at=snapshot_at,
seed_at=seed_at,
)
scheduler.run_forever()
+8 -2
View File
@@ -83,7 +83,8 @@ DEFAULT_CONFIG: dict[str, Any] = {
"up_blacklist_seed": [],
"schedule": {
"discover_minutes": 60,
"snapshot_at": "00:01",
"snapshot_at": "23:15",
"seed_at": "00:01",
"candidate_ttl_days": 7,
"recent_snapshot_minutes": 60,
"recent_video_days": 2,
@@ -179,6 +180,7 @@ class FolderConfig:
class ScheduleConfig:
discover_minutes: int
snapshot_at: str
seed_at: str
candidate_ttl_days: int
recent_snapshot_minutes: int
recent_video_days: int
@@ -449,14 +451,18 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
sched_raw = data["schedule"]
if not isinstance(sched_raw, Mapping):
raise ConfigError("schedule 必须是对象")
snapshot_at = _as_str(sched_raw.get("snapshot_at", "00:01"), "schedule.snapshot_at")
snapshot_at = _as_str(sched_raw.get("snapshot_at", "23:15"), "schedule.snapshot_at")
if not _SNAPSHOT_AT_RE.match(snapshot_at):
raise ConfigError("schedule.snapshot_at 必须是 HH:MM(24 小时制)")
seed_at = _as_str(sched_raw.get("seed_at", "00:01"), "schedule.seed_at")
if not _SNAPSHOT_AT_RE.match(seed_at):
raise ConfigError("schedule.seed_at 必须是 HH:MM(24 小时制)")
schedule = ScheduleConfig(
discover_minutes=_as_int(
sched_raw.get("discover_minutes", 60), "schedule.discover_minutes", minimum=1
),
snapshot_at=snapshot_at,
seed_at=seed_at,
candidate_ttl_days=_as_int(
sched_raw.get("candidate_ttl_days", 7),
"schedule.candidate_ttl_days",
+78 -1
View File
@@ -10,7 +10,7 @@ from __future__ import annotations
import sqlite3
import time
from dataclasses import dataclass
from typing import Any, Iterable, Mapping
from typing import Any, Iterable, Mapping, Sequence
from .config import FolderConfig
from .database import Database
@@ -727,6 +727,83 @@ class Repository:
)
self.db.write(run)
def copy_ok_snapshots(
self,
source_date: str,
target_date: str,
*,
captured_at: int,
src: str = "seed",
) -> int:
"""将 source_date 的 ok=1 行复制到 target_date;已有 target 行则跳过。"""
def run(conn: sqlite3.Connection) -> int:
cur = conn.execute(
"""
INSERT INTO snapshot(
stat_date, aid, "view", "like", favorite, coin,
share, reply, danmaku, captured_at, ok, err, src
)
SELECT
?, aid, "view", "like", favorite, coin,
share, reply, danmaku, ?, 1, NULL, ?
FROM snapshot
WHERE stat_date = ? AND ok = 1
AND aid NOT IN (
SELECT aid FROM snapshot WHERE stat_date = ?
)
""",
(target_date, int(captured_at), src, source_date, target_date),
)
return int(cur.rowcount or 0)
return self.db.write(run)
def copy_ok_up_stats(
self,
source_date: str,
target_date: str,
*,
captured_at: int,
mids: Sequence[int] | None = None,
) -> int:
"""将 source_date 的 ok=1 粉丝行复制到 target_date;已有 target 行则跳过。"""
def run(conn: sqlite3.Connection) -> int:
if mids is not None:
mid_list = [int(m) for m in mids]
if not mid_list:
return 0
placeholders = ",".join("?" for _ in mid_list)
cur = conn.execute(
f"""
INSERT INTO up_stat(
stat_date, mid, follower, following, captured_at, ok
)
SELECT ?, mid, follower, following, ?, 1
FROM up_stat
WHERE stat_date = ? AND ok = 1 AND mid IN ({placeholders})
AND mid NOT IN (
SELECT mid FROM up_stat WHERE stat_date = ?
)
""",
(target_date, int(captured_at), source_date, *mid_list, target_date),
)
else:
cur = conn.execute(
"""
INSERT INTO up_stat(
stat_date, mid, follower, following, captured_at, ok
)
SELECT ?, mid, follower, following, ?, 1
FROM up_stat
WHERE stat_date = ? AND ok = 1
AND mid NOT IN (
SELECT mid FROM up_stat WHERE stat_date = ?
)
""",
(target_date, int(captured_at), source_date, target_date),
)
return int(cur.rowcount or 0)
return self.db.write(run)
def snapshot_counts(self, stat_date: str) -> dict[str, int]:
def run(conn: sqlite3.Connection) -> dict[str, int]:
rows = conn.execute(
+123 -2
View File
@@ -5,6 +5,8 @@
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``;
* 成功写入七项指标;``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
* 午夜 ``seed_at`` 用上一完成日 ok=1 行克隆出当日壳(``snapshot:seeded`` / ``fans:seeded``),不推进完成日;
* 晚间 ``snapshot_at`` 全量必须 force 打穿 seed,成功后才写完成日并清除 seeded;
* ``run_recent_videos`` 只刷新近 N 天新片的当日行,不改完成日 / 备份;
* 视频不可访问时标记 ``removed``;
* 其他错误最多重试 3 次,仍失败写 ``ok=0`` 占位,不写假零值;
@@ -129,18 +131,43 @@ class SnapshotService:
def backup_error_key(stat_date: str) -> str:
return f"snapshot:backup_error:{stat_date}"
@staticmethod
def seeded_key(stat_date: str) -> str:
return f"snapshot:seeded:{stat_date}"
@staticmethod
def fans_seeded_key(stat_date: str) -> str:
return f"fans:seeded:{stat_date}"
def run_if_needed(
self, *, date: str | None = None, force: bool = False
) -> SnapshotSummary:
"""视频快照(含 UP 头像补齐);粉丝采集请用 ``run_fans_if_needed``。"""
"""视频快照(含 UP 头像补齐);粉丝采集请用 ``run_fans_if_needed``。
当日若仅有午夜 seed 壳且未 ``force``,直接返回等待晚间落盘,
避免把「全员 skip」误标成 completed。
"""
now = self._now()
stat_date = date or shanghai_date(now)
if not is_date(stat_date):
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
seeded = self.repo.kv_get(self.seeded_key(stat_date)) == "1"
if force:
return self.run_videos(date=stat_date, force=True)
completed = self.repo.kv_get(self.completed_key(stat_date)) == "1"
if seeded and not completed:
self._log(
"info", "snapshot.skip",
stat_date=stat_date, reason="seeded_pending_seal",
)
return SnapshotSummary(
stat_date=stat_date,
started_at=now,
finished_at=now,
completed=False,
already_completed=False,
)
pending = self.repo.pending_snapshot_aids(stat_date)
if completed and not pending:
if self.repo.kv_get(self.backup_key(stat_date)) != "1":
@@ -256,6 +283,8 @@ class SnapshotService:
self.repo.kv_set("snapshot:last_attempt", stat_date)
if summary.completed:
self.repo.kv_set("snapshot:last_completed", stat_date)
# 真落盘后清掉午夜 seed 标记。
self.repo.kv_delete(self.seeded_key(stat_date))
elif self.repo.kv_get("snapshot:last_completed") == stat_date:
self.repo.kv_set("snapshot:last_completed", "")
@@ -378,7 +407,24 @@ class SnapshotService:
)
self._log("info", "fans.skip", stat_date=stat_date, reason="no_track_fans")
return summary
if not force and not self._pending_fan_mids(stat_date, cfg.track_fans):
seeded = self.repo.kv_get(self.fans_seeded_key(stat_date)) == "1"
if force or seeded:
# 晚间/显式 force,或存在 seed 壳时必须重采(seeded 时由调度传入 force)。
if seeded and not force:
self._log(
"info", "fans.skip",
stat_date=stat_date, reason="seeded_pending_seal",
)
return SnapshotSummary(
stat_date=stat_date,
started_at=now,
finished_at=now,
completed=False,
already_completed=False,
fans_skipped=len(cfg.track_fans),
)
return self.run_fans(date=stat_date, force=True)
if not self._pending_fan_mids(stat_date, cfg.track_fans):
self._log("info", "fans.skip", stat_date=stat_date, reason="already_completed")
return SnapshotSummary(
stat_date=stat_date,
@@ -456,6 +502,11 @@ class SnapshotService:
# 独立粉丝任务用 completed 表示粉丝段成败;合并进视频摘要时不改动视频完成标记。
if standalone:
summary.completed = fans_ok
if fans_ok:
self.repo.kv_delete(self.fans_seeded_key(stat_date))
elif fans_ok:
# CLI 合并跑:粉丝段成功也清 seed。
self.repo.kv_delete(self.fans_seeded_key(stat_date))
summary.finished_at = self._now()
self._log(
"info" if fans_ok else "warning",
@@ -468,6 +519,76 @@ class SnapshotService:
)
return summary
def run_seed_from_last_completed(
self, *, date: str | None = None
) -> SnapshotSummary:
"""用上一完成日的 ok=1 行克隆出当日壳(视频 + 粉丝)。
不写 ``snapshot:completed`` / ``snapshot:last_completed``;
仅打 ``snapshot:seeded`` / ``fans:seeded``,供白天展示与晚间 force 重采。
"""
cfg = self.config_store.get()
now = self._now()
stat_date = date or shanghai_date(now)
if not is_date(stat_date):
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
source = self.repo.kv_get("snapshot:last_completed")
if not source or source >= stat_date:
self._log(
"info", "snapshot.seed_skip",
stat_date=stat_date, reason="no_source", source=source,
)
summary.finished_at = self._now()
summary.completed = True
return summary
if self.repo.kv_get(self.completed_key(stat_date)) == "1":
self._log(
"info", "snapshot.seed_skip",
stat_date=stat_date, reason="already_completed",
)
summary.finished_at = self._now()
summary.completed = True
summary.already_completed = True
return summary
if self.repo.kv_get(self.seeded_key(stat_date)) == "1":
self._log(
"info", "snapshot.seed_skip",
stat_date=stat_date, reason="already_seeded", source=source,
)
summary.finished_at = self._now()
summary.completed = True
summary.already_completed = True
return summary
copied = self.repo.copy_ok_snapshots(
source, stat_date, captured_at=now, src="seed",
)
fans_copied = 0
if cfg.track_fans:
fans_copied = self.repo.copy_ok_up_stats(
source, stat_date, captured_at=now, mids=cfg.track_fans,
)
self.repo.kv_set(self.fans_seeded_key(stat_date), "1")
self.repo.kv_set(self.seeded_key(stat_date), "1")
summary.total = copied
summary.success = copied
summary.fans_ok = fans_copied
if copied or fans_copied:
self.repo.bump_cache_version("videos")
if fans_copied:
self.repo.bump_cache_version("fans")
summary.finished_at = self._now()
summary.completed = True # seed 任务本身成功;不等于快照落盘完成
self._log(
"info", "snapshot.seed_done",
stat_date=stat_date, source=source,
copied=copied, fans_copied=fans_copied,
)
return summary
def _pending_fan_mids(self, stat_date: str, mids: tuple[int, ...] | list[int]) -> list[int]:
pending: list[int] = []
for mid in mids:
+65 -11
View File
@@ -284,6 +284,37 @@ class DashboardQueries:
result[int(data["aid"])] = data
return result
@classmethod
def _snapshot_previous_relative(
cls,
conn: sqlite3.Connection,
current: Mapping[int, Mapping[str, Any]],
*,
days_back: int,
) -> dict[int, dict[str, Any]]:
"""相对各片当前快照日取 previous:days_back=1 为相邻成功日,否则 <= current-N。"""
if not current or days_back < 1:
return {}
by_date: dict[str, list[int]] = {}
for aid, snap in current.items():
day = snap.get("stat_date")
if not day:
continue
by_date.setdefault(str(day), []).append(int(aid))
result: dict[int, dict[str, Any]] = {}
for day, aids in by_date.items():
if days_back <= 1:
prev = cls._snapshot_map(
conn, aids, before=day, before_inclusive=False,
)
else:
baseline = _date_minus(day, days_back)
prev = cls._snapshot_map(
conn, aids, before=baseline, before_inclusive=True,
)
result.update(prev)
return result
@staticmethod
def _videos(
conn: sqlite3.Connection,
@@ -443,7 +474,8 @@ class DashboardQueries:
videos = self._videos(conn)
aids = [int(video["aid"]) for video in videos]
snapshots = self._snapshot_map(conn, aids)
previous = self._snapshot_map(conn, aids, before=latest or self._today())
# 播放增量:按片相对当前点的相邻成功日,避免 seed/recent 拉到 T-2。
previous = self._snapshot_previous_relative(conn, snapshots, days_back=1)
totals = {metric: 0 for metric in METRICS}
for video in videos:
metrics = _metric_map(snapshots.get(int(video["aid"])))
@@ -521,6 +553,7 @@ class DashboardQueries:
reverse=True,
)
tasks = self._task_records(conn)
updated_at = self._latest_short_update_at(tasks)
auth = {
"ok": self._kv(conn, "auth:ok"),
"message": self._kv(conn, "auth:message"),
@@ -535,6 +568,7 @@ class DashboardQueries:
"generated_at": int(self._now()),
"today": self._today(),
"data_date": latest,
"updated_at": updated_at,
"snapshot_days": snapshot_days,
"first_seen_date": first_seen_date,
"is_first_day": is_first_day,
@@ -542,6 +576,7 @@ class DashboardQueries:
"added_today": added_today,
"counts": {
**by_status,
"total": sum(by_status.values()),
"active_ups": int(conn.execute(
"SELECT COUNT(DISTINCT up_mid) FROM video "
"WHERE status IN ('active','unfaved') AND up_mid IS NOT NULL"
@@ -571,6 +606,22 @@ class DashboardQueries:
row = conn.execute("SELECT v FROM kv WHERE k = ?", (key,)).fetchone()
return None if row is None else str(row["v"])
@staticmethod
def _latest_short_update_at(tasks: Mapping[str, Any]) -> int | None:
"""首页「更新」时刻:优先短更新任务,其次全日更 / 粉丝。"""
for name in ("snapshot_recent", "snapshot", "fans"):
raw = tasks.get(name)
if not isinstance(raw, Mapping):
continue
finished = raw.get("finished_at")
if finished is None:
continue
try:
return int(finished)
except (TypeError, ValueError):
continue
return None
def _task_records(self, conn: sqlite3.Connection) -> dict[str, Any]:
rows = conn.execute(
"SELECT k, v FROM kv WHERE k LIKE 'task:last:%' ORDER BY k"
@@ -802,8 +853,10 @@ class DashboardQueries:
previous: dict[int, dict[str, Any]] = {}
if sort in {"growth", "growth_rate", "interaction"}:
baseline_date = _date_minus(latest, growth_days)
previous = self._snapshot_map(conn, aids, before=baseline_date, before_inclusive=True)
# 相对各片当前快照日回退 growth_days,避免 seed/recent 相对 last_completed 拉长窗口。
previous = self._snapshot_previous_relative(
conn, snapshots, days_back=growth_days,
)
elif sort in {"heat", "view", "like", "coin", "favorite", "share", "reply", "danmaku"}:
previous = {}
@@ -1780,14 +1833,16 @@ class DashboardQueries:
next_runs = {
"fans": int(next_daily_run(now, cfg.schedule.snapshot_at)),
"snapshot": int(next_daily_run(now, cfg.schedule.snapshot_at)),
"snapshot_seed": int(next_daily_run(now, cfg.schedule.seed_at)),
"discover": int(now + discover_interval),
"snapshot_recent": int(now + recent_interval),
}
for name, key in (
("fans", "fans"),
("snapshot", "snapshot"),
("discover", "discovery"),
("snapshot_recent", "snapshot_recent"),
for name, key, at in (
("fans", "fans", cfg.schedule.snapshot_at),
("snapshot", "snapshot", cfg.schedule.snapshot_at),
("snapshot_seed", "snapshot_seed", cfg.schedule.seed_at),
("discover", "discovery", None),
("snapshot_recent", "snapshot_recent", None),
):
record = tasks.get(key)
if not (isinstance(record, dict) and record.get("finished_at")):
@@ -1801,9 +1856,7 @@ class DashboardQueries:
)
next_runs[name] = finished_at + interval
else:
next_runs[name] = int(
next_daily_run(finished_at, cfg.schedule.snapshot_at)
)
next_runs[name] = int(next_daily_run(finished_at, at))
errors = [
{"task": name, **record}
for name, record in tasks.items()
@@ -1840,6 +1893,7 @@ class DashboardQueries:
"schedule": {
"discover_minutes": cfg.schedule.discover_minutes,
"snapshot_at": cfg.schedule.snapshot_at,
"seed_at": cfg.schedule.seed_at,
"candidate_ttl_days": cfg.schedule.candidate_ttl_days,
"recent_snapshot_minutes": cfg.schedule.recent_snapshot_minutes,
"recent_video_days": cfg.schedule.recent_video_days,
+16
View File
@@ -55,6 +55,22 @@ export function formatDate(timestamp) {
}).format(date);
}
export function formatClock(timestamp) {
if (!timestamp) return "—";
const date = new Date(Number(timestamp) * 1000);
if (Number.isNaN(date.getTime())) return "—";
return new Intl.DateTimeFormat("zh-CN", {
hour: "2-digit", minute: "2-digit", hour12: false,
timeZone: "Asia/Shanghai",
}).format(date);
}
export function formatDataDate(value) {
if (!value) return "—";
return String(value).replace(/-/g, "/");
}
export function formatDateTime(timestamp) {
if (!timestamp) return "—";
const date = new Date(Number(timestamp) * 1000);
+11 -4
View File
@@ -1,8 +1,8 @@
import { api } from "../api.js";
import { emptyState, mediaCard } from "../components.js";
import {
escapeAttr, escapeHtml, formatCount, formatDelta,
formatPercent, formatScore,
escapeAttr, escapeHtml, formatClock, formatCount, formatDataDate,
formatDelta, formatPercent, formatScore,
} from "../format.js";
import { canAccessProtected } from "../session.js";
@@ -55,6 +55,11 @@ export async function renderHome(root) {
? formatCount(overview.initial_import_count)
: formatCount(overview.added_today);
const activeCount = overview.counts?.active || 0;
const totalCount = overview.counts?.total
?? ((overview.counts?.active || 0)
+ (overview.counts?.unfaved || 0)
+ (overview.counts?.removed || 0)
+ (overview.counts?.dropped || 0));
const pending = overview.counts?.candidates_pending || 0;
// 候选提示仅对已登录(或未启用鉴权)用户可见,访客不展示运行期信息。
const showCandidates = pending > 0 && canAccessProtected();
@@ -66,12 +71,14 @@ export async function renderHome(root) {
const growthEmptyHint = growthDays <= 1
? "当前快照不足,完成至少 2 天快照后即可显示播放增量。"
: `需要至少 ${growthDays + 1} 天快照才能计算近 ${growthDays} 日增长。`;
const updatedLabel = formatClock(overview.updated_at);
const dataDateLabel = formatDataDate(overview.data_date);
root.innerHTML = `<section class="page home-page">
<header class="page-head home-head">
<div>
<h1>最新动态</h1>
<p class="page-sub">收录视频 ${formatCount(activeCount)}</p>
<p class="page-sub">更新:${escapeHtml(updatedLabel)} / 全站数据:${escapeHtml(dataDateLabel)}</p>
</div>
<button class="icon-button" id="refresh-home" type="button" title="刷新">刷新</button>
</header>
@@ -86,7 +93,7 @@ export async function renderHome(root) {
${statChip("7 日粉丝变化", fans?.delta_7d === null || fans?.delta_7d === undefined ? "—" : formatDelta(fans.delta_7d), {
href: "/fans",
})}
${statChip("数据日期", escapeHtml(overview.data_date || "—"))}
${statChip("收录视频", `${formatCount(activeCount)} / ${formatCount(totalCount)}`)}
</div>
${showCandidates ? `<a class="notice-row" href="/candidates">
@@ -94,6 +94,7 @@ export async function renderSystem(root) {
<section class="panel">
${sectionHeader("预计下次运行")}
<div class="status-grid status-row-4">
<div><span>午夜落盘克隆</span><strong>${running.snapshot_seed ? "运行中" : escapeHtml(formatDateTime(next.snapshot_seed))}</strong></div>
<div><span>咻咻满粉丝数更新</span><strong>${running.fans ? "运行中" : escapeHtml(formatDateTime(next.fans))}</strong></div>
<div><span>视频数据快照 + UP 信息</span><strong>${running.snapshot ? "运行中" : escapeHtml(formatDateTime(next.snapshot))}</strong></div>
<div><span>新视频小时同步</span><strong>${running.snapshot_recent ? "运行中" : escapeHtml(formatDateTime(next.snapshot_recent))}</strong></div>