feat(scheduler): persist task state/progress/due_at across restart (#8)

Replace process-start next_run=0 guesses with kv task:state records
(status, due_at, stat_date, progress). Startup recovers stale running,
honors future due_at for delayed retries, and resumes the same
stat_date. Seed only enqueues evening seal; seal force keeps non-seed
ok=1 as progress (keep_scraped). Keeps !7 fail-logging and cleanup retry.
This commit is contained in:
bot-ci committed 2026-11-03 01:50:00 +08:00
1 parent 51ffa44577
commit f408da93c2
7 files changed
+1793 -902

No files matched your search

+32 -8
View File
@@ -155,7 +155,10 @@ class SnapshotService:
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
seeded = self.repo.kv_get(self.seeded_key(stat_date)) == "1"
if force:
return self.run_videos(date=stat_date, force=True)
# 晚间 seal / 续跑:打穿 seed,保留已真实采集的 ok=1 作为进度。
return self.run_videos(
date=stat_date, force=True, keep_scraped=True,
)
completed = self.repo.kv_get(self.completed_key(stat_date)) == "1"
if seeded and not completed:
@@ -169,6 +172,7 @@ class SnapshotService:
finished_at=now,
completed=False,
already_completed=False,
reason="seeded_pending_seal",
)
pending = self.repo.pending_snapshot_aids(stat_date)
if completed and not pending:
@@ -205,8 +209,19 @@ class SnapshotService:
summary = self.run_videos(date=date, force=force)
return self.run_fans(date=summary.stat_date, force=force, summary=summary)
def run_videos(self, *, date: str | None = None, force: bool = False) -> SnapshotSummary:
"""采集视频七项指标并在全部成功后标记完成日 / 备份。"""
def run_videos(
self,
*,
date: str | None = None,
force: bool = False,
keep_scraped: bool = False,
) -> SnapshotSummary:
"""采集视频七项指标并在全部成功后标记完成日 / 备份。
``force``:忽略完成标记,重跑。
``keep_scraped``:force 时仍跳过非 seed 的 ok=1(seal 断点续跑);
CLI ``--force`` 不传此标志,会重采全部成功行。
"""
cfg = self.config_store.get()
client = self._client()
now = self._now()
@@ -227,11 +242,19 @@ class SnapshotService:
summary.total = len(videos)
for index, aid in enumerate(videos, 1):
existing = self.repo.get_snapshot(stat_date, aid)
if existing is not None and int(existing.get("ok") or 0) == 1 and not force:
# 断点续跑:同日已成功则跳过。小时任务刷新过的行也是 ok=1,
# 自然覆盖「1 小时内 captured_at 已更新则不再次更新」的要求。
summary.skipped += 1
continue
if existing is not None and int(existing.get("ok") or 0) == 1:
src = str(existing.get("src") or "")
if keep_scraped and src == "seed":
# seal 续跑:必须打穿克隆壳
pass
elif keep_scraped and src != "seed":
summary.skipped += 1
continue
elif not force:
# 普通续跑:同日已成功则跳过(含小时任务刚刷新的)
summary.skipped += 1
continue
# force 且非 keep_scraped:CLI 全量重采,不 skip
stat, error, unavailable = self._fetch_view(client, aid)
if unavailable:
@@ -425,6 +448,7 @@ class SnapshotService:
completed=False,
already_completed=False,
fans_skipped=len(cfg.track_fans),
reason="seeded_pending_seal",
)
return self.run_fans(date=stat_date, force=True)
if not self._pending_fan_mids(stat_date, cfg.track_fans):