feat: EOD 23:15 seal + midnight seed shell for video/fans

Move daily snapshot/fans to 23:15 and add 00:01 seed that clones
last_completed ok=1 rows into today without advancing completed.
Evening runs force through seed markers; growth previous is per-aid
adjacent. Home shows short-update clock + 全站数据 date, and 收录视频
counts.
This commit is contained in:
bot-ci committed 2026-11-02 12:38:55 +08:00
1 parent 374b138041
commit 8a9cb21408
17 files changed
+520 -39

No files matched your search

+123 -2
View File
@@ -5,6 +5,8 @@
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``;
* 成功写入七项指标;``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
* 午夜 ``seed_at`` 用上一完成日 ok=1 行克隆出当日壳(``snapshot:seeded`` / ``fans:seeded``),不推进完成日;
* 晚间 ``snapshot_at`` 全量必须 force 打穿 seed,成功后才写完成日并清除 seeded;
* ``run_recent_videos`` 只刷新近 N 天新片的当日行,不改完成日 / 备份;
* 视频不可访问时标记 ``removed``;
* 其他错误最多重试 3 次,仍失败写 ``ok=0`` 占位,不写假零值;
@@ -129,18 +131,43 @@ class SnapshotService:
def backup_error_key(stat_date: str) -> str:
return f"snapshot:backup_error:{stat_date}"
@staticmethod
def seeded_key(stat_date: str) -> str:
return f"snapshot:seeded:{stat_date}"
@staticmethod
def fans_seeded_key(stat_date: str) -> str:
return f"fans:seeded:{stat_date}"
def run_if_needed(
self, *, date: str | None = None, force: bool = False
) -> SnapshotSummary:
"""视频快照(含 UP 头像补齐);粉丝采集请用 ``run_fans_if_needed``。"""
"""视频快照(含 UP 头像补齐);粉丝采集请用 ``run_fans_if_needed``。
当日若仅有午夜 seed 壳且未 ``force``,直接返回等待晚间落盘,
避免把「全员 skip」误标成 completed。
"""
now = self._now()
stat_date = date or shanghai_date(now)
if not is_date(stat_date):
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
seeded = self.repo.kv_get(self.seeded_key(stat_date)) == "1"
if force:
return self.run_videos(date=stat_date, force=True)
completed = self.repo.kv_get(self.completed_key(stat_date)) == "1"
if seeded and not completed:
self._log(
"info", "snapshot.skip",
stat_date=stat_date, reason="seeded_pending_seal",
)
return SnapshotSummary(
stat_date=stat_date,
started_at=now,
finished_at=now,
completed=False,
already_completed=False,
)
pending = self.repo.pending_snapshot_aids(stat_date)
if completed and not pending:
if self.repo.kv_get(self.backup_key(stat_date)) != "1":
@@ -256,6 +283,8 @@ class SnapshotService:
self.repo.kv_set("snapshot:last_attempt", stat_date)
if summary.completed:
self.repo.kv_set("snapshot:last_completed", stat_date)
# 真落盘后清掉午夜 seed 标记。
self.repo.kv_delete(self.seeded_key(stat_date))
elif self.repo.kv_get("snapshot:last_completed") == stat_date:
self.repo.kv_set("snapshot:last_completed", "")
@@ -378,7 +407,24 @@ class SnapshotService:
)
self._log("info", "fans.skip", stat_date=stat_date, reason="no_track_fans")
return summary
if not force and not self._pending_fan_mids(stat_date, cfg.track_fans):
seeded = self.repo.kv_get(self.fans_seeded_key(stat_date)) == "1"
if force or seeded:
# 晚间/显式 force,或存在 seed 壳时必须重采(seeded 时由调度传入 force)。
if seeded and not force:
self._log(
"info", "fans.skip",
stat_date=stat_date, reason="seeded_pending_seal",
)
return SnapshotSummary(
stat_date=stat_date,
started_at=now,
finished_at=now,
completed=False,
already_completed=False,
fans_skipped=len(cfg.track_fans),
)
return self.run_fans(date=stat_date, force=True)
if not self._pending_fan_mids(stat_date, cfg.track_fans):
self._log("info", "fans.skip", stat_date=stat_date, reason="already_completed")
return SnapshotSummary(
stat_date=stat_date,
@@ -456,6 +502,11 @@ class SnapshotService:
# 独立粉丝任务用 completed 表示粉丝段成败;合并进视频摘要时不改动视频完成标记。
if standalone:
summary.completed = fans_ok
if fans_ok:
self.repo.kv_delete(self.fans_seeded_key(stat_date))
elif fans_ok:
# CLI 合并跑:粉丝段成功也清 seed。
self.repo.kv_delete(self.fans_seeded_key(stat_date))
summary.finished_at = self._now()
self._log(
"info" if fans_ok else "warning",
@@ -468,6 +519,76 @@ class SnapshotService:
)
return summary
def run_seed_from_last_completed(
self, *, date: str | None = None
) -> SnapshotSummary:
"""用上一完成日的 ok=1 行克隆出当日壳(视频 + 粉丝)。
不写 ``snapshot:completed`` / ``snapshot:last_completed``;
仅打 ``snapshot:seeded`` / ``fans:seeded``,供白天展示与晚间 force 重采。
"""
cfg = self.config_store.get()
now = self._now()
stat_date = date or shanghai_date(now)
if not is_date(stat_date):
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
source = self.repo.kv_get("snapshot:last_completed")
if not source or source >= stat_date:
self._log(
"info", "snapshot.seed_skip",
stat_date=stat_date, reason="no_source", source=source,
)
summary.finished_at = self._now()
summary.completed = True
return summary
if self.repo.kv_get(self.completed_key(stat_date)) == "1":
self._log(
"info", "snapshot.seed_skip",
stat_date=stat_date, reason="already_completed",
)
summary.finished_at = self._now()
summary.completed = True
summary.already_completed = True
return summary
if self.repo.kv_get(self.seeded_key(stat_date)) == "1":
self._log(
"info", "snapshot.seed_skip",
stat_date=stat_date, reason="already_seeded", source=source,
)
summary.finished_at = self._now()
summary.completed = True
summary.already_completed = True
return summary
copied = self.repo.copy_ok_snapshots(
source, stat_date, captured_at=now, src="seed",
)
fans_copied = 0
if cfg.track_fans:
fans_copied = self.repo.copy_ok_up_stats(
source, stat_date, captured_at=now, mids=cfg.track_fans,
)
self.repo.kv_set(self.fans_seeded_key(stat_date), "1")
self.repo.kv_set(self.seeded_key(stat_date), "1")
summary.total = copied
summary.success = copied
summary.fans_ok = fans_copied
if copied or fans_copied:
self.repo.bump_cache_version("videos")
if fans_copied:
self.repo.bump_cache_version("fans")
summary.finished_at = self._now()
summary.completed = True # seed 任务本身成功;不等于快照落盘完成
self._log(
"info", "snapshot.seed_done",
stat_date=stat_date, source=source,
copied=copied, fans_copied=fans_copied,
)
return summary
def _pending_fan_mids(self, stat_date: str, mids: tuple[int, ...] | list[int]) -> list[int]:
pending: list[int] = []
for mid in mids: