bot-ci
f6bef59e07
- Cross-midnight: unfinished yesterday seals ASAP with yesterday's stat_date; force seal ignores today's snapshot_at gate when stat_date < today. - Off-plan already_completed: compare completion started_at (not finished_at) to today's planned slot; earlier → still run at slot for all daily tasks. - Cold start: bootstrap task:state from task:last / seed markers. - ensure_pending: do not glue new due onto a different day's unfinished state. - run_if_needed(force): keep_scraped True for seal, False for CLI --force. - Scheduler due_reader: pick up disk due_at each loop (seed→evening seal).
164 lines
5.3 KiB
Python
164 lines
5.3 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""单进程内置调度器。
|
||
|
||
P0 只注册收藏夹同步任务；P1/P3 会在同一调度器上追加快照、粉丝和发现任务。
|
||
调度只负责“什么时候运行”，具体任务保持幂等，并在自己的服务层处理重试。
|
||
|
||
任务 ``fn`` 可返回 :class:`TaskControl` 以覆盖 ``ok`` / ``next_run``
|
||
（用于静默窗延后、失败补跑等）；未返回时仍按 ``next_run_fn`` / ``interval`` 排期。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import time
|
||
from dataclasses import dataclass
|
||
from typing import Any, Callable, Sequence
|
||
|
||
|
||
@dataclass
|
||
class TaskControl:
|
||
"""任务可选返回值：覆盖成功标记与下次运行时间。"""
|
||
|
||
ok: bool = True
|
||
next_run: float | None = None
|
||
error: str | None = None
|
||
|
||
|
||
@dataclass
|
||
class ScheduledTask:
|
||
name: str
|
||
interval: float
|
||
fn: Callable[[], Any]
|
||
next_run: float = 0.0
|
||
next_run_fn: Callable[[float], float] | None = None
|
||
|
||
|
||
@dataclass
|
||
class TaskResult:
|
||
name: str
|
||
started_at: float
|
||
finished_at: float
|
||
ok: bool
|
||
error: str | None = None
|
||
|
||
def as_dict(self) -> dict[str, Any]:
|
||
return {
|
||
"name": self.name,
|
||
"started_at": int(self.started_at),
|
||
"finished_at": int(self.finished_at),
|
||
"ok": self.ok,
|
||
"error": self.error,
|
||
}
|
||
|
||
|
||
class Scheduler:
|
||
def __init__(
|
||
self,
|
||
tasks: Sequence[ScheduledTask],
|
||
*,
|
||
clock=time.time,
|
||
sleeper=time.sleep,
|
||
logger=None,
|
||
due_reader: Callable[[str], float | None] | None = None,
|
||
) -> None:
|
||
if not tasks:
|
||
raise ValueError("至少需要一个调度任务")
|
||
for task in tasks:
|
||
if task.interval <= 0:
|
||
raise ValueError(f"任务 {task.name} 的 interval 必须大于 0")
|
||
self.tasks = list(tasks)
|
||
self._clock = clock
|
||
self._sleeper = sleeper
|
||
self.logger = logger
|
||
# 每轮从落盘 due_at 刷新 next_run（seed 挂当晚 seal 等同进程可见）
|
||
self._due_reader = due_reader
|
||
|
||
def _log(self, level: str, event: str, **fields) -> None:
|
||
if self.logger is not None:
|
||
getattr(self.logger, level)(event, **fields)
|
||
|
||
def run_due(self, *, now: float | None = None) -> list[TaskResult]:
|
||
current = self._clock() if now is None else now
|
||
results: list[TaskResult] = []
|
||
for task in self.tasks:
|
||
if self._due_reader is not None:
|
||
disk_due = self._due_reader(task.name)
|
||
if disk_due is not None:
|
||
task.next_run = float(disk_due)
|
||
if task.next_run > current:
|
||
continue
|
||
started = current
|
||
error: str | None = None
|
||
control: TaskControl | None = None
|
||
try:
|
||
raw = task.fn()
|
||
except Exception as exc: # noqa: BLE001 单任务失败不终止调度
|
||
error = str(exc)
|
||
else:
|
||
if isinstance(raw, TaskControl):
|
||
control = raw
|
||
finished = self._clock()
|
||
if control is not None and control.next_run is not None:
|
||
task.next_run = float(control.next_run)
|
||
elif task.next_run_fn is not None:
|
||
task.next_run = task.next_run_fn(started)
|
||
else:
|
||
task.next_run = started + task.interval
|
||
ok = error is None
|
||
if control is not None:
|
||
ok = bool(control.ok) and error is None
|
||
if control.error and error is None:
|
||
error = control.error
|
||
# Skip/fail ≠ success：先定 ok/error，再记 scheduler 日志。
|
||
if ok:
|
||
self._log("info", "scheduler.task_ok", task=task.name)
|
||
elif control is not None:
|
||
self._log(
|
||
"warning",
|
||
"scheduler.task_fail",
|
||
task=task.name,
|
||
error=error,
|
||
)
|
||
else:
|
||
self._log(
|
||
"error",
|
||
"scheduler.task_error",
|
||
task=task.name,
|
||
error=error,
|
||
)
|
||
result = TaskResult(
|
||
name=task.name,
|
||
started_at=started,
|
||
finished_at=finished,
|
||
ok=ok,
|
||
error=error,
|
||
)
|
||
results.append(result)
|
||
return results
|
||
|
||
def run_forever(
|
||
self,
|
||
*,
|
||
stop_event=None,
|
||
poll_seconds: float = 30.0,
|
||
max_iterations: int | None = None,
|
||
) -> list[TaskResult]:
|
||
"""阻塞运行；``stop_event`` 或 ``max_iterations`` 用于可控退出/测试。"""
|
||
if poll_seconds <= 0:
|
||
raise ValueError("poll_seconds 必须大于 0")
|
||
all_results: list[TaskResult] = []
|
||
iterations = 0
|
||
while True:
|
||
if stop_event is not None and stop_event.is_set():
|
||
break
|
||
all_results.extend(self.run_due())
|
||
iterations += 1
|
||
if max_iterations is not None and iterations >= max_iterations:
|
||
break
|
||
if stop_event is not None:
|
||
if stop_event.wait(poll_seconds):
|
||
break
|
||
else:
|
||
self._sleeper(poll_seconds)
|
||
return all_results
|