feat(scheduler): persist task state/progress/due_at across restart (#8)
Replace process-start next_run=0 guesses with kv task:state records (status, due_at, stat_date, progress). Startup recovers stale running, honors future due_at for delayed retries, and resumes the same stat_date. Seed only enqueues evening seal; seal force keeps non-seed ok=1 as progress (keep_scraped). Keeps !7 fail-logging and cleanup retry.
This commit is contained in:
1 parent
51ffa44577
commit
f408da93c2
7 files changed
+1793
-902
No files matched your search
@@ -155,7 +155,10 @@ class SnapshotService:
|
||||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||||
seeded = self.repo.kv_get(self.seeded_key(stat_date)) == "1"
|
||||
if force:
|
||||
return self.run_videos(date=stat_date, force=True)
|
||||
# 晚间 seal / 续跑:打穿 seed,保留已真实采集的 ok=1 作为进度。
|
||||
return self.run_videos(
|
||||
date=stat_date, force=True, keep_scraped=True,
|
||||
)
|
||||
|
||||
completed = self.repo.kv_get(self.completed_key(stat_date)) == "1"
|
||||
if seeded and not completed:
|
||||
@@ -169,6 +172,7 @@ class SnapshotService:
|
||||
finished_at=now,
|
||||
completed=False,
|
||||
already_completed=False,
|
||||
reason="seeded_pending_seal",
|
||||
)
|
||||
pending = self.repo.pending_snapshot_aids(stat_date)
|
||||
if completed and not pending:
|
||||
@@ -205,8 +209,19 @@ class SnapshotService:
|
||||
summary = self.run_videos(date=date, force=force)
|
||||
return self.run_fans(date=summary.stat_date, force=force, summary=summary)
|
||||
|
||||
def run_videos(self, *, date: str | None = None, force: bool = False) -> SnapshotSummary:
|
||||
"""采集视频七项指标并在全部成功后标记完成日 / 备份。"""
|
||||
def run_videos(
|
||||
self,
|
||||
*,
|
||||
date: str | None = None,
|
||||
force: bool = False,
|
||||
keep_scraped: bool = False,
|
||||
) -> SnapshotSummary:
|
||||
"""采集视频七项指标并在全部成功后标记完成日 / 备份。
|
||||
|
||||
``force``:忽略完成标记,重跑。
|
||||
``keep_scraped``:force 时仍跳过非 seed 的 ok=1(seal 断点续跑);
|
||||
CLI ``--force`` 不传此标志,会重采全部成功行。
|
||||
"""
|
||||
cfg = self.config_store.get()
|
||||
client = self._client()
|
||||
now = self._now()
|
||||
@@ -227,11 +242,19 @@ class SnapshotService:
|
||||
summary.total = len(videos)
|
||||
for index, aid in enumerate(videos, 1):
|
||||
existing = self.repo.get_snapshot(stat_date, aid)
|
||||
if existing is not None and int(existing.get("ok") or 0) == 1 and not force:
|
||||
# 断点续跑:同日已成功则跳过。小时任务刷新过的行也是 ok=1,
|
||||
# 自然覆盖「1 小时内 captured_at 已更新则不再次更新」的要求。
|
||||
summary.skipped += 1
|
||||
continue
|
||||
if existing is not None and int(existing.get("ok") or 0) == 1:
|
||||
src = str(existing.get("src") or "")
|
||||
if keep_scraped and src == "seed":
|
||||
# seal 续跑:必须打穿克隆壳
|
||||
pass
|
||||
elif keep_scraped and src != "seed":
|
||||
summary.skipped += 1
|
||||
continue
|
||||
elif not force:
|
||||
# 普通续跑:同日已成功则跳过(含小时任务刚刷新的)
|
||||
summary.skipped += 1
|
||||
continue
|
||||
# force 且非 keep_scraped:CLI 全量重采,不 skip
|
||||
|
||||
stat, error, unavailable = self._fetch_view(client, aid)
|
||||
if unavailable:
|
||||
@@ -425,6 +448,7 @@ class SnapshotService:
|
||||
completed=False,
|
||||
already_completed=False,
|
||||
fans_skipped=len(cfg.track_fans),
|
||||
reason="seeded_pending_seal",
|
||||
)
|
||||
return self.run_fans(date=stat_date, force=True)
|
||||
if not self._pending_fan_mids(stat_date, cfg.track_fans):
|
||||
|
||||
Reference in new issue
Block a user