feat(scheduler): persist task state/progress/due_at across restart (#8)
Replace process-start next_run=0 guesses with kv task:state records (status, due_at, stat_date, progress). Startup recovers stale running, honors future due_at for delayed retries, and resumes the same stat_date. Seed only enqueues evening seal; seal force keeps non-seed ok=1 as progress (keep_scraped). Keeps !7 fail-logging and cleanup retry.
This commit is contained in:
bot-ci committed
1 parent
51ffa44577
commit
f408da93c2
7 files changed
+1793
-902
No files matched your search
@@ -0,0 +1,291 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""调度任务状态持久化（跨进程重启 / 容器重建）。
|
||||
|
||||
每条任务一条 kv 记录 ``task:state:{name}``，字段见 :class:`TaskState`。
|
||||
调度器只认落盘的 ``due_at`` / ``status`` / ``stat_date`` / ``progress``，
|
||||
不再靠启动时把 ``next_run=0`` 猜「要不要立刻跑」。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Any, Mapping
|
||||
|
||||
from .repository import Repository
|
||||
|
||||
STATUS_PENDING = "pending"
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_DONE = "done"
|
||||
STATUS_FAILED = "failed"
|
||||
STATUS_DEFERRED = "deferred"
|
||||
|
||||
_ACTIVE = frozenset({STATUS_PENDING, STATUS_RUNNING, STATUS_FAILED, STATUS_DEFERRED})
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskState:
|
||||
name: str
|
||||
status: str
|
||||
due_at: float
|
||||
stat_date: str | None = None
|
||||
progress: dict[str, Any] | None = None
|
||||
updated_at: float = 0.0
|
||||
error: str | None = None
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
payload = asdict(self)
|
||||
# JSON 友好：去掉 None progress 也可保留
|
||||
return payload
|
||||
|
||||
|
||||
def state_key(name: str) -> str:
|
||||
return f"task:state:{name}"
|
||||
|
||||
|
||||
def load_state(repo: Repository, name: str) -> TaskState | None:
|
||||
raw = repo.kv_get(state_key(name))
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
return None
|
||||
if not isinstance(data, dict) or not data.get("name"):
|
||||
return None
|
||||
try:
|
||||
due_at = float(data.get("due_at") or 0)
|
||||
updated_at = float(data.get("updated_at") or 0)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
progress = data.get("progress")
|
||||
if progress is not None and not isinstance(progress, dict):
|
||||
progress = None
|
||||
return TaskState(
|
||||
name=str(data.get("name") or name),
|
||||
status=str(data.get("status") or STATUS_PENDING),
|
||||
due_at=due_at,
|
||||
stat_date=(str(data["stat_date"]) if data.get("stat_date") else None),
|
||||
progress=progress,
|
||||
updated_at=updated_at,
|
||||
error=(str(data["error"]) if data.get("error") else None),
|
||||
)
|
||||
|
||||
|
||||
def save_state(repo: Repository, state: TaskState) -> None:
|
||||
repo.kv_set(state_key(state.name), json.dumps(state.as_dict(), ensure_ascii=False))
|
||||
|
||||
|
||||
def _now(now: float | None) -> float:
|
||||
return time.time() if now is None else float(now)
|
||||
|
||||
|
||||
def recover_stale_running(
|
||||
repo: Repository,
|
||||
name: str,
|
||||
*,
|
||||
now: float | None = None,
|
||||
) -> TaskState | None:
|
||||
"""进程已死但仍为 ``running``：改为 ``pending``，保留 progress / stat_date。
|
||||
|
||||
``due_at`` 取 ``min(原 due_at, now)``，使过期/中断任务可立即续跑，
|
||||
未到期的（极少见）仍尊重原 due_at。
|
||||
"""
|
||||
current = _now(now)
|
||||
state = load_state(repo, name)
|
||||
if state is None or state.status != STATUS_RUNNING:
|
||||
return state
|
||||
state.status = STATUS_PENDING
|
||||
state.due_at = min(float(state.due_at or current), current)
|
||||
state.updated_at = current
|
||||
save_state(repo, state)
|
||||
return state
|
||||
|
||||
|
||||
def mark_running(
|
||||
repo: Repository,
|
||||
name: str,
|
||||
*,
|
||||
stat_date: str | None,
|
||||
now: float | None = None,
|
||||
progress: Mapping[str, Any] | None = None,
|
||||
due_at: float | None = None,
|
||||
) -> TaskState:
|
||||
current = _now(now)
|
||||
prev = load_state(repo, name)
|
||||
state = TaskState(
|
||||
name=name,
|
||||
status=STATUS_RUNNING,
|
||||
due_at=float(due_at if due_at is not None else (prev.due_at if prev else current)),
|
||||
stat_date=stat_date if stat_date is not None else (prev.stat_date if prev else None),
|
||||
progress=dict(progress) if progress is not None else (prev.progress if prev else None),
|
||||
updated_at=current,
|
||||
error=None,
|
||||
)
|
||||
save_state(repo, state)
|
||||
return state
|
||||
|
||||
|
||||
def mark_done(
|
||||
repo: Repository,
|
||||
name: str,
|
||||
*,
|
||||
due_at: float,
|
||||
now: float | None = None,
|
||||
stat_date: str | None = None,
|
||||
progress: Mapping[str, Any] | None = None,
|
||||
) -> TaskState:
|
||||
"""成功：记 done，并写入下一次 ``due_at``（下一条墙钟/间隔）。"""
|
||||
current = _now(now)
|
||||
state = TaskState(
|
||||
name=name,
|
||||
status=STATUS_DONE,
|
||||
due_at=float(due_at),
|
||||
stat_date=stat_date,
|
||||
progress=dict(progress) if progress is not None else None,
|
||||
updated_at=current,
|
||||
error=None,
|
||||
)
|
||||
save_state(repo, state)
|
||||
return state
|
||||
|
||||
|
||||
def mark_failed(
|
||||
repo: Repository,
|
||||
name: str,
|
||||
*,
|
||||
due_at: float,
|
||||
now: float | None = None,
|
||||
stat_date: str | None = None,
|
||||
progress: Mapping[str, Any] | None = None,
|
||||
error: str | None = None,
|
||||
) -> TaskState:
|
||||
current = _now(now)
|
||||
prev = load_state(repo, name)
|
||||
state = TaskState(
|
||||
name=name,
|
||||
status=STATUS_FAILED,
|
||||
due_at=float(due_at),
|
||||
stat_date=stat_date if stat_date is not None else (prev.stat_date if prev else None),
|
||||
progress=(
|
||||
dict(progress) if progress is not None
|
||||
else (prev.progress if prev else None)
|
||||
),
|
||||
updated_at=current,
|
||||
error=error,
|
||||
)
|
||||
save_state(repo, state)
|
||||
return state
|
||||
|
||||
|
||||
def mark_deferred(
|
||||
repo: Repository,
|
||||
name: str,
|
||||
*,
|
||||
due_at: float,
|
||||
now: float | None = None,
|
||||
stat_date: str | None = None,
|
||||
progress: Mapping[str, Any] | None = None,
|
||||
error: str | None = None,
|
||||
) -> TaskState:
|
||||
"""主动延后（静默窗 / seed 避让等）：保留进度，写未来 due_at。"""
|
||||
current = _now(now)
|
||||
prev = load_state(repo, name)
|
||||
state = TaskState(
|
||||
name=name,
|
||||
status=STATUS_DEFERRED,
|
||||
due_at=float(due_at),
|
||||
stat_date=stat_date if stat_date is not None else (prev.stat_date if prev else None),
|
||||
progress=(
|
||||
dict(progress) if progress is not None
|
||||
else (prev.progress if prev else None)
|
||||
),
|
||||
updated_at=current,
|
||||
error=error,
|
||||
)
|
||||
save_state(repo, state)
|
||||
return state
|
||||
|
||||
|
||||
def ensure_pending(
|
||||
repo: Repository,
|
||||
name: str,
|
||||
*,
|
||||
due_at: float,
|
||||
stat_date: str | None,
|
||||
now: float | None = None,
|
||||
overwrite_done: bool = True,
|
||||
) -> TaskState:
|
||||
"""保证存在一条 pending（如 seed 成功后挂上当晚 seal）。
|
||||
|
||||
若已有未完成状态且 due_at 更早，保留更早的 due_at；
|
||||
``overwrite_done`` 时覆盖已完成记录。
|
||||
"""
|
||||
current = _now(now)
|
||||
prev = load_state(repo, name)
|
||||
if prev is not None and prev.status in _ACTIVE and prev.status != STATUS_RUNNING:
|
||||
# 未完成：取更早的 due_at，补上缺失的 stat_date
|
||||
due = min(float(prev.due_at or due_at), float(due_at))
|
||||
sd = prev.stat_date or stat_date
|
||||
if due == prev.due_at and sd == prev.stat_date and prev.status == STATUS_PENDING:
|
||||
return prev
|
||||
prev.due_at = due
|
||||
prev.stat_date = sd
|
||||
prev.status = STATUS_PENDING
|
||||
prev.updated_at = current
|
||||
save_state(repo, prev)
|
||||
return prev
|
||||
if prev is not None and prev.status == STATUS_RUNNING:
|
||||
return prev
|
||||
if prev is not None and prev.status == STATUS_DONE and not overwrite_done:
|
||||
return prev
|
||||
state = TaskState(
|
||||
name=name,
|
||||
status=STATUS_PENDING,
|
||||
due_at=float(due_at),
|
||||
stat_date=stat_date,
|
||||
progress=None,
|
||||
updated_at=current,
|
||||
error=None,
|
||||
)
|
||||
save_state(repo, state)
|
||||
return state
|
||||
|
||||
|
||||
def startup_next_run(
|
||||
repo: Repository,
|
||||
name: str,
|
||||
*,
|
||||
wall_due_at: float,
|
||||
now: float | None = None,
|
||||
) -> float:
|
||||
"""常驻启动时的 ``ScheduledTask.next_run``：只认落盘状态。
|
||||
|
||||
- 恢复残留 ``running`` → ``pending``
|
||||
- ``due_at > now`` → 原样返回（失败延时不会被提前）
|
||||
- ``due_at <= now`` 且未完成 → 立刻可跑（返回 ``due_at``，可为过去时间）
|
||||
- ``done`` 且 ``due_at > now`` → 等下一次
|
||||
- ``done`` 且 ``due_at <= now`` → 到期，返回 ``due_at``
|
||||
- 无状态 → 墙钟 ``wall_due_at``
|
||||
"""
|
||||
current = _now(now)
|
||||
state = recover_stale_running(repo, name, now=current)
|
||||
if state is None:
|
||||
return float(wall_due_at)
|
||||
due = float(state.due_at or 0)
|
||||
if due <= 0:
|
||||
return float(wall_due_at)
|
||||
# 无论 status：due_at 是唯一时间门闩
|
||||
if due > current:
|
||||
return due
|
||||
# 已到期：未完成要续跑；done 也表示下一轮已到期
|
||||
return due
|
||||
|
||||
|
||||
def resume_stat_date(repo: Repository, name: str, *, fallback: str) -> str:
|
||||
"""续跑时优先用状态里的归属日，避免跨日重启跑错天。"""
|
||||
state = load_state(repo, name)
|
||||
if state is not None and state.stat_date and state.status in _ACTIVE:
|
||||
return state.stat_date
|
||||
return fallback
|
||||
Reference in new issue
Block a user