fix(scheduler): daily slot off-plan + cross-midnight resume (#8)

- Cross-midnight: unfinished yesterday seals ASAP with yesterday's stat_date;
  force seal ignores today's snapshot_at gate when stat_date < today.
- Off-plan already_completed: compare completion started_at (not finished_at)
  to today's planned slot; earlier → still run at slot for all daily tasks.
- Cold start: bootstrap task:state from task:last / seed markers.
- ensure_pending: do not glue new due onto a different day's unfinished state.
- run_if_needed(force): keep_scraped True for seal, False for CLI --force.
- Scheduler due_reader: pick up disk due_at each loop (seed→evening seal).
This commit is contained in:
bot-ci committed 2026-11-03 02:35:13 +08:00
1 parent dcf37c4a1d
commit f6bef59e07
7 files changed
+630 -24

No files matched your search

+125 -13
View File
@@ -377,14 +377,19 @@ def _initial_daily_next_run(
) -> float:
"""进程启动时日更任务的下次运行:优先 ``task:state.due_at``。
不得返回 ``0`` / 用「无状态」以外的过去时间去误伤——无状态时对齐墙钟。
失败且 ``due_at`` 已过:返回该 ``due_at``(调度器会立刻补跑),不退回次日墙钟。
无状态时从 ``task:last`` / seed 标记冷启动回填;不得返回 ``0`` 误伤。
失败且 ``due_at`` 已过:立刻补跑,不退回次日墙钟。
"""
del retry_seconds # 间隔已落在 task:state.due_at
current = time.time() if now is None else float(now)
wall = next_daily_run(current, at)
retry = float(retry_seconds) if retry_seconds is not None else _TASK_RETRY_SECONDS
return task_state_mod.startup_next_run(
repo, name, wall_due_at=wall, now=current,
repo,
name,
wall_due_at=wall,
now=current,
slot_hhmm=at,
retry_seconds=retry,
)
@@ -396,6 +401,67 @@ def _reached_daily_slot(at: str, now: float | None = None) -> bool:
return current >= slot
def _today_slot_ts(at: str, now: float | None = None) -> float:
"""今日 ``at`` 墙钟点的时间戳(不论是否已过)。"""
current = shanghai_datetime(now)
hour, minute = parse_hhmm(at)
return current.replace(hour=hour, minute=minute, second=0, microsecond=0).timestamp()
def _last_success_started_at(repo: Repository, name: str) -> float | None:
"""最近一次成功完成记录的 ``started_at``(不用 finished_at)。"""
try:
raw = repo.kv_get(f"task:last:{name}")
except Exception: # noqa: BLE001
return None
if not raw:
return None
try:
data = json.loads(raw)
except (TypeError, ValueError, json.JSONDecodeError):
return None
if not isinstance(data, dict):
return None
if not data.get("ok") or data.get("skipped"):
return None
try:
return float(data["started_at"])
except (KeyError, TypeError, ValueError):
return None
def _off_plan_completion(
repo: Repository,
name: str,
at: str,
now: float | None = None,
) -> bool:
"""成功完成的 started_at 早于当日计划点 → 非计划内,到点仍应再跑。
产品规则:只用完成记录的 **started_at** 与今日 ``at`` 比较,禁止 finished_at。
"""
started = _last_success_started_at(repo, name)
if started is None:
return False
return float(started) < _today_slot_ts(at, now)
def _should_force_daily_seal(
*,
seeded: bool,
stat_date: str,
today: str,
slot_hhmm: str,
now: float,
) -> bool:
"""seed 壳是否 force 打穿:跨天未完成立即 force;同日须已到计划点。"""
if not seeded:
return False
if stat_date < today:
return True
return _reached_daily_slot(slot_hhmm, now)
def _evening_due_at(now: float, snapshot_at: str) -> float:
"""尚未到当日 snapshot_at 则挂今晚,否则下一档墙钟。"""
hour, minute = parse_hhmm(snapshot_at)
@@ -535,7 +601,11 @@ def cmd_snapshot(args, logger: JsonLogger) -> int:
stat_date=shanghai_date(float(started)),
)
try:
video_summary = service.run_if_needed(force=args.force)
# CLI --force:全量重采(keep_scraped=False);非 force 走常规门闩
video_summary = service.run_if_needed(
force=args.force,
keep_scraped=False if args.force else None,
)
except Exception as exc: # noqa: BLE001
finished = int(time.time())
_record_task(
@@ -1014,16 +1084,33 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
repo, "snapshot", started_at=started, stat_date=stat_date,
)
try:
# seed 壳 ≠ 白天强采:仅到达当日 snapshot_at(或显式续跑已到期的
# seal pending)才 force。due_at 门闩负责「会不会跑」;这里是
# force 的第二道闸。
# seed 壳 force:跨天未完成(stat_date < today)立刻打穿;
# 同日仅到达 snapshot_at 后才 force。due_at 管「会不会跑」。
seeded = repo.kv_get(SnapshotService.seeded_key(stat_date)) == "1"
force_seed = seeded and _reached_daily_slot(
snapshot_at, float(started),
force_seed = _should_force_daily_seal(
seeded=seeded,
stat_date=stat_date,
today=today,
slot_hhmm=snapshot_at,
now=float(started),
)
summary = snapshot_service.run_if_needed(
date=stat_date, force=force_seed,
date=stat_date,
force=force_seed,
keep_scraped=True if force_seed else None,
)
# 日更规则:already_completed 但完成 started_at 早于今日计划点
# → 非计划内,到点仍按计划再跑(全量,不用 finished_at)。
if (
summary.already_completed
and _reached_daily_slot(snapshot_at, float(started))
and _off_plan_completion(
repo, "snapshot", snapshot_at, float(started),
)
):
summary = snapshot_service.run_if_needed(
date=stat_date, force=True, keep_scraped=False,
)
except Exception as exc: # noqa: BLE001
finished = int(time.time())
next_run = _retry_soon(float(finished))
@@ -1138,12 +1225,26 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
seeded = (
repo.kv_get(SnapshotService.fans_seeded_key(stat_date)) == "1"
)
force_seed = seeded and _reached_daily_slot(
snapshot_at, float(started),
force_seed = _should_force_daily_seal(
seeded=seeded,
stat_date=stat_date,
today=today,
slot_hhmm=snapshot_at,
now=float(started),
)
summary = snapshot_service.run_fans_if_needed(
date=stat_date, force=force_seed,
)
if (
summary.already_completed
and _reached_daily_slot(snapshot_at, float(started))
and _off_plan_completion(
repo, "fans", snapshot_at, float(started),
)
):
summary = snapshot_service.run_fans_if_needed(
date=stat_date, force=True,
)
except Exception as exc: # noqa: BLE001
finished = int(time.time())
next_run = _retry_soon(float(finished))
@@ -1287,6 +1388,16 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
cleanup_next = task_state_mod.startup_next_run(
repo, "candidate_cleanup", wall_due_at=now_ts, now=now_ts,
)
def _due_reader(task_name: str) -> float | None:
state = task_state_mod.load_state(repo, task_name)
if state is None:
return None
try:
due = float(state.due_at or 0)
except (TypeError, ValueError):
return None
return due if due > 0 else None
scheduler = Scheduler(
[
ScheduledTask(
@@ -1328,6 +1439,7 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
ScheduledTask("config_watch", 60.0, run_config_watch),
],
logger=logger,
due_reader=_due_reader,
)
if serve_web and not args.once: