fix(scheduler): daily slot off-plan + cross-midnight resume (#8)

- Cross-midnight: unfinished yesterday seals ASAP with yesterday's stat_date;
  force seal ignores today's snapshot_at gate when stat_date < today.
- Off-plan already_completed: compare completion started_at (not finished_at)
  to today's planned slot; earlier → still run at slot for all daily tasks.
- Cold start: bootstrap task:state from task:last / seed markers.
- ensure_pending: do not glue new due onto a different day's unfinished state.
- run_if_needed(force): keep_scraped True for seal, False for CLI --force.
- Scheduler due_reader: pick up disk due_at each loop (seed→evening seal).
This commit is contained in:
bot-ci committed 2026-11-03 02:35:13 +08:00
1 parent dcf37c4a1d
commit f6bef59e07
7 files changed
+630 -24

No files matched your search

+132
View File
@@ -20,6 +20,7 @@ from reaction_board.task_state import ( # noqa: E402
STATUS_FAILED,
STATUS_PENDING,
STATUS_RUNNING,
bootstrap_from_last,
ensure_pending,
load_state,
mark_deferred,
@@ -251,6 +252,133 @@ def test_fail_or_defer_preserves_progress():
db.close()
def test_bootstrap_from_last_fail_due_soon():
"""A: 仅有 task:last 失败、无 state → 回填 failed 且 due_at 可立刻补跑。"""
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 12, 0, tzinfo=SHANGHAI_TZ).timestamp()
wall = next_daily_run(now, "23:15")
repo.kv_set(
"task:last:snapshot",
json.dumps({
"name": "snapshot",
"ok": False,
"skipped": False,
"started_at": int(now - 600),
"finished_at": int(now - 500),
"error": "boom",
"summary": {"stat_date": "2026-11-03"},
}, ensure_ascii=False),
)
assert load_state(repo, "snapshot") is None
boot = bootstrap_from_last(
repo, "snapshot", wall_due_at=wall, now=now, retry_seconds=300.0,
)
assert boot is not None
assert boot.status == STATUS_FAILED
assert boot.stat_date == "2026-11-03"
assert boot.due_at <= now
nxt = startup_next_run(
repo, "snapshot", wall_due_at=wall, now=now, slot_hhmm="23:15",
)
assert nxt <= now
assert nxt != wall
finally:
db.close()
def test_ensure_pending_does_not_glue_yesterday_stat_date():
"""B: 昨天未完成保留;不得把今晚 due 粘到昨天的 stat_date 上。"""
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 0, 10, tzinfo=SHANGHAI_TZ).timestamp()
yday_due = now - 60
ensure_pending(
repo, "snapshot", due_at=yday_due,
stat_date="2026-11-02", now=now - 120,
)
tonight = datetime(2026, 11, 3, 23, 15, tzinfo=SHANGHAI_TZ).timestamp()
# seed 想挂今晚:不得冲掉昨天未完成
kept = ensure_pending(
repo, "snapshot", due_at=tonight,
stat_date="2026-11-03", now=now,
)
assert kept.stat_date == "2026-11-02"
assert kept.due_at == yday_due
assert resume_stat_date(
repo, "snapshot", fallback="2026-11-03",
) == "2026-11-02"
finally:
db.close()
def test_cross_midnight_mid_seal_resumes_yesterday():
"""规则1 / B: 23:15 崩溃 → 00:10 重启续跑昨天,due 立刻可跑。"""
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
crash = datetime(2026, 11, 2, 23, 40, tzinfo=SHANGHAI_TZ).timestamp()
mark_running(
repo, "snapshot", stat_date="2026-11-02", now=crash,
progress={"done": 3, "total": 10},
)
repo.kv_set("snapshot:seeded:2026-11-02", "1")
restart = datetime(2026, 11, 3, 0, 10, tzinfo=SHANGHAI_TZ).timestamp()
wall = next_daily_run(restart, "23:15")
nxt = startup_next_run(
repo, "snapshot", wall_due_at=wall, now=restart, slot_hhmm="23:15",
)
assert nxt <= restart
assert resume_stat_date(
repo, "snapshot", fallback="2026-11-03",
) == "2026-11-02"
state = load_state(repo, "snapshot")
assert state is not None
assert state.status == STATUS_PENDING
assert state.progress == {"done": 3, "total": 10}
finally:
db.close()
def test_scheduler_due_reader_picks_up_enqueued_seal():
"""D: 同进程 seed 写盘 due_at=now 后,下一轮 run_due 须跑 seal。"""
with tempfile.TemporaryDirectory() as tmp:
db, repo = _repo(tmp)
try:
now = datetime(2026, 11, 3, 23, 20, tzinfo=SHANGHAI_TZ).timestamp()
# snapshot 原 next_run 挂次日;盘上被 enqueue 成 now
tomorrow = next_daily_run(now, "23:15")
calls = []
def run_snap():
calls.append("snap")
return None
def due_reader(name: str):
st = load_state(repo, name)
return float(st.due_at) if st and st.due_at else None
clock = [now]
sched = Scheduler(
[ScheduledTask("snapshot", 86400.0, run_snap, next_run=tomorrow)],
clock=lambda: clock[0],
due_reader=due_reader,
)
assert sched.run_due() == []
assert calls == []
ensure_pending(
repo, "snapshot", due_at=now, stat_date="2026-11-03", now=now,
)
results = sched.run_due()
assert len(results) == 1
assert calls == ["snap"]
finally:
db.close()
def main_test():
test_restart_mid_run_resumes_same_stat_date()
test_seeded_before_snapshot_at_does_not_force_run()
@@ -259,6 +387,10 @@ def main_test():
test_done_writes_next_due_and_blocks_until_then()
test_no_state_falls_back_to_wall_clock()
test_fail_or_defer_preserves_progress()
test_bootstrap_from_last_fail_due_soon()
test_ensure_pending_does_not_glue_yesterday_stat_date()
test_cross_midnight_mid_seal_resumes_yesterday()
test_scheduler_due_reader_picks_up_enqueued_seal()
print("OK: task_state persistence / resume / due_at gates")