fix(scheduler): daily slot off-plan + cross-midnight resume (#8)
- Cross-midnight: unfinished yesterday seals ASAP with yesterday's stat_date; force seal ignores today's snapshot_at gate when stat_date < today. - Off-plan already_completed: compare completion started_at (not finished_at) to today's planned slot; earlier → still run at slot for all daily tasks. - Cold start: bootstrap task:state from task:last / seed markers. - ensure_pending: do not glue new due onto a different day's unfinished state. - run_if_needed(force): keep_scraped True for seal, False for CLI --force. - Scheduler due_reader: pick up disk due_at each loop (seed→evening seal).
This commit is contained in:
1 parent
dcf37c4a1d
commit
f6bef59e07
7 files changed
+630
-24
No files matched your search
+125
-13
@@ -377,14 +377,19 @@ def _initial_daily_next_run(
|
||||
) -> float:
|
||||
"""进程启动时日更任务的下次运行:优先 ``task:state.due_at``。
|
||||
|
||||
不得返回 ``0`` / 用「无状态」以外的过去时间去误伤——无状态时对齐墙钟。
|
||||
失败且 ``due_at`` 已过:返回该 ``due_at``(调度器会立刻补跑),不退回次日墙钟。
|
||||
无状态时从 ``task:last`` / seed 标记冷启动回填;不得返回 ``0`` 误伤。
|
||||
失败且 ``due_at`` 已过:立刻补跑,不退回次日墙钟。
|
||||
"""
|
||||
del retry_seconds # 间隔已落在 task:state.due_at
|
||||
current = time.time() if now is None else float(now)
|
||||
wall = next_daily_run(current, at)
|
||||
retry = float(retry_seconds) if retry_seconds is not None else _TASK_RETRY_SECONDS
|
||||
return task_state_mod.startup_next_run(
|
||||
repo, name, wall_due_at=wall, now=current,
|
||||
repo,
|
||||
name,
|
||||
wall_due_at=wall,
|
||||
now=current,
|
||||
slot_hhmm=at,
|
||||
retry_seconds=retry,
|
||||
)
|
||||
|
||||
|
||||
@@ -396,6 +401,67 @@ def _reached_daily_slot(at: str, now: float | None = None) -> bool:
|
||||
return current >= slot
|
||||
|
||||
|
||||
def _today_slot_ts(at: str, now: float | None = None) -> float:
|
||||
"""今日 ``at`` 墙钟点的时间戳(不论是否已过)。"""
|
||||
current = shanghai_datetime(now)
|
||||
hour, minute = parse_hhmm(at)
|
||||
return current.replace(hour=hour, minute=minute, second=0, microsecond=0).timestamp()
|
||||
|
||||
|
||||
def _last_success_started_at(repo: Repository, name: str) -> float | None:
|
||||
"""最近一次成功完成记录的 ``started_at``(不用 finished_at)。"""
|
||||
try:
|
||||
raw = repo.kv_get(f"task:last:{name}")
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
if not data.get("ok") or data.get("skipped"):
|
||||
return None
|
||||
try:
|
||||
return float(data["started_at"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _off_plan_completion(
|
||||
repo: Repository,
|
||||
name: str,
|
||||
at: str,
|
||||
now: float | None = None,
|
||||
) -> bool:
|
||||
"""成功完成的 started_at 早于当日计划点 → 非计划内,到点仍应再跑。
|
||||
|
||||
产品规则:只用完成记录的 **started_at** 与今日 ``at`` 比较,禁止 finished_at。
|
||||
"""
|
||||
started = _last_success_started_at(repo, name)
|
||||
if started is None:
|
||||
return False
|
||||
return float(started) < _today_slot_ts(at, now)
|
||||
|
||||
|
||||
def _should_force_daily_seal(
|
||||
*,
|
||||
seeded: bool,
|
||||
stat_date: str,
|
||||
today: str,
|
||||
slot_hhmm: str,
|
||||
now: float,
|
||||
) -> bool:
|
||||
"""seed 壳是否 force 打穿:跨天未完成立即 force;同日须已到计划点。"""
|
||||
if not seeded:
|
||||
return False
|
||||
if stat_date < today:
|
||||
return True
|
||||
return _reached_daily_slot(slot_hhmm, now)
|
||||
|
||||
|
||||
def _evening_due_at(now: float, snapshot_at: str) -> float:
|
||||
"""尚未到当日 snapshot_at 则挂今晚,否则下一档墙钟。"""
|
||||
hour, minute = parse_hhmm(snapshot_at)
|
||||
@@ -535,7 +601,11 @@ def cmd_snapshot(args, logger: JsonLogger) -> int:
|
||||
stat_date=shanghai_date(float(started)),
|
||||
)
|
||||
try:
|
||||
video_summary = service.run_if_needed(force=args.force)
|
||||
# CLI --force:全量重采(keep_scraped=False);非 force 走常规门闩
|
||||
video_summary = service.run_if_needed(
|
||||
force=args.force,
|
||||
keep_scraped=False if args.force else None,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
_record_task(
|
||||
@@ -1014,16 +1084,33 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
repo, "snapshot", started_at=started, stat_date=stat_date,
|
||||
)
|
||||
try:
|
||||
# seed 壳 ≠ 白天强采:仅到达当日 snapshot_at(或显式续跑已到期的
|
||||
# seal pending)才 force。due_at 门闩负责「会不会跑」;这里是
|
||||
# force 的第二道闸。
|
||||
# seed 壳 force:跨天未完成(stat_date < today)立刻打穿;
|
||||
# 同日仅到达 snapshot_at 后才 force。due_at 管「会不会跑」。
|
||||
seeded = repo.kv_get(SnapshotService.seeded_key(stat_date)) == "1"
|
||||
force_seed = seeded and _reached_daily_slot(
|
||||
snapshot_at, float(started),
|
||||
force_seed = _should_force_daily_seal(
|
||||
seeded=seeded,
|
||||
stat_date=stat_date,
|
||||
today=today,
|
||||
slot_hhmm=snapshot_at,
|
||||
now=float(started),
|
||||
)
|
||||
summary = snapshot_service.run_if_needed(
|
||||
date=stat_date, force=force_seed,
|
||||
date=stat_date,
|
||||
force=force_seed,
|
||||
keep_scraped=True if force_seed else None,
|
||||
)
|
||||
# 日更规则:already_completed 但完成 started_at 早于今日计划点
|
||||
# → 非计划内,到点仍按计划再跑(全量,不用 finished_at)。
|
||||
if (
|
||||
summary.already_completed
|
||||
and _reached_daily_slot(snapshot_at, float(started))
|
||||
and _off_plan_completion(
|
||||
repo, "snapshot", snapshot_at, float(started),
|
||||
)
|
||||
):
|
||||
summary = snapshot_service.run_if_needed(
|
||||
date=stat_date, force=True, keep_scraped=False,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
next_run = _retry_soon(float(finished))
|
||||
@@ -1138,12 +1225,26 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
seeded = (
|
||||
repo.kv_get(SnapshotService.fans_seeded_key(stat_date)) == "1"
|
||||
)
|
||||
force_seed = seeded and _reached_daily_slot(
|
||||
snapshot_at, float(started),
|
||||
force_seed = _should_force_daily_seal(
|
||||
seeded=seeded,
|
||||
stat_date=stat_date,
|
||||
today=today,
|
||||
slot_hhmm=snapshot_at,
|
||||
now=float(started),
|
||||
)
|
||||
summary = snapshot_service.run_fans_if_needed(
|
||||
date=stat_date, force=force_seed,
|
||||
)
|
||||
if (
|
||||
summary.already_completed
|
||||
and _reached_daily_slot(snapshot_at, float(started))
|
||||
and _off_plan_completion(
|
||||
repo, "fans", snapshot_at, float(started),
|
||||
)
|
||||
):
|
||||
summary = snapshot_service.run_fans_if_needed(
|
||||
date=stat_date, force=True,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
next_run = _retry_soon(float(finished))
|
||||
@@ -1287,6 +1388,16 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
cleanup_next = task_state_mod.startup_next_run(
|
||||
repo, "candidate_cleanup", wall_due_at=now_ts, now=now_ts,
|
||||
)
|
||||
def _due_reader(task_name: str) -> float | None:
|
||||
state = task_state_mod.load_state(repo, task_name)
|
||||
if state is None:
|
||||
return None
|
||||
try:
|
||||
due = float(state.due_at or 0)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return due if due > 0 else None
|
||||
|
||||
scheduler = Scheduler(
|
||||
[
|
||||
ScheduledTask(
|
||||
@@ -1328,6 +1439,7 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
ScheduledTask("config_watch", 60.0, run_config_watch),
|
||||
],
|
||||
logger=logger,
|
||||
due_reader=_due_reader,
|
||||
)
|
||||
|
||||
if serve_web and not args.once:
|
||||
|
||||
Reference in new issue
Block a user