fix(scheduler): daily slot off-plan + cross-midnight resume (#8)
- Cross-midnight: unfinished yesterday seals ASAP with yesterday's stat_date; force seal ignores today's snapshot_at gate when stat_date < today. - Off-plan already_completed: compare completion started_at (not finished_at) to today's planned slot; earlier → still run at slot for all daily tasks. - Cold start: bootstrap task:state from task:last / seed markers. - ensure_pending: do not glue new due onto a different day's unfinished state. - run_if_needed(force): keep_scraped True for seal, False for CLI --force. - Scheduler due_reader: pick up disk due_at each loop (seed→evening seal).
This commit is contained in:
1 parent
dcf37c4a1d
commit
f6bef59e07
7 files changed
+630
-24
No files matched your search
@@ -20,6 +20,7 @@ from reaction_board.task_state import ( # noqa: E402
|
||||
STATUS_FAILED,
|
||||
STATUS_PENDING,
|
||||
STATUS_RUNNING,
|
||||
bootstrap_from_last,
|
||||
ensure_pending,
|
||||
load_state,
|
||||
mark_deferred,
|
||||
@@ -251,6 +252,133 @@ def test_fail_or_defer_preserves_progress():
|
||||
db.close()
|
||||
|
||||
|
||||
|
||||
def test_bootstrap_from_last_fail_due_soon():
|
||||
"""A: 仅有 task:last 失败、无 state → 回填 failed 且 due_at 可立刻补跑。"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, repo = _repo(tmp)
|
||||
try:
|
||||
now = datetime(2026, 11, 3, 12, 0, tzinfo=SHANGHAI_TZ).timestamp()
|
||||
wall = next_daily_run(now, "23:15")
|
||||
repo.kv_set(
|
||||
"task:last:snapshot",
|
||||
json.dumps({
|
||||
"name": "snapshot",
|
||||
"ok": False,
|
||||
"skipped": False,
|
||||
"started_at": int(now - 600),
|
||||
"finished_at": int(now - 500),
|
||||
"error": "boom",
|
||||
"summary": {"stat_date": "2026-11-03"},
|
||||
}, ensure_ascii=False),
|
||||
)
|
||||
assert load_state(repo, "snapshot") is None
|
||||
boot = bootstrap_from_last(
|
||||
repo, "snapshot", wall_due_at=wall, now=now, retry_seconds=300.0,
|
||||
)
|
||||
assert boot is not None
|
||||
assert boot.status == STATUS_FAILED
|
||||
assert boot.stat_date == "2026-11-03"
|
||||
assert boot.due_at <= now
|
||||
nxt = startup_next_run(
|
||||
repo, "snapshot", wall_due_at=wall, now=now, slot_hhmm="23:15",
|
||||
)
|
||||
assert nxt <= now
|
||||
assert nxt != wall
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_ensure_pending_does_not_glue_yesterday_stat_date():
|
||||
"""B: 昨天未完成保留;不得把今晚 due 粘到昨天的 stat_date 上。"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, repo = _repo(tmp)
|
||||
try:
|
||||
now = datetime(2026, 11, 3, 0, 10, tzinfo=SHANGHAI_TZ).timestamp()
|
||||
yday_due = now - 60
|
||||
ensure_pending(
|
||||
repo, "snapshot", due_at=yday_due,
|
||||
stat_date="2026-11-02", now=now - 120,
|
||||
)
|
||||
tonight = datetime(2026, 11, 3, 23, 15, tzinfo=SHANGHAI_TZ).timestamp()
|
||||
# seed 想挂今晚:不得冲掉昨天未完成
|
||||
kept = ensure_pending(
|
||||
repo, "snapshot", due_at=tonight,
|
||||
stat_date="2026-11-03", now=now,
|
||||
)
|
||||
assert kept.stat_date == "2026-11-02"
|
||||
assert kept.due_at == yday_due
|
||||
assert resume_stat_date(
|
||||
repo, "snapshot", fallback="2026-11-03",
|
||||
) == "2026-11-02"
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_cross_midnight_mid_seal_resumes_yesterday():
|
||||
"""规则1 / B: 23:15 崩溃 → 00:10 重启续跑昨天,due 立刻可跑。"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, repo = _repo(tmp)
|
||||
try:
|
||||
crash = datetime(2026, 11, 2, 23, 40, tzinfo=SHANGHAI_TZ).timestamp()
|
||||
mark_running(
|
||||
repo, "snapshot", stat_date="2026-11-02", now=crash,
|
||||
progress={"done": 3, "total": 10},
|
||||
)
|
||||
repo.kv_set("snapshot:seeded:2026-11-02", "1")
|
||||
restart = datetime(2026, 11, 3, 0, 10, tzinfo=SHANGHAI_TZ).timestamp()
|
||||
wall = next_daily_run(restart, "23:15")
|
||||
nxt = startup_next_run(
|
||||
repo, "snapshot", wall_due_at=wall, now=restart, slot_hhmm="23:15",
|
||||
)
|
||||
assert nxt <= restart
|
||||
assert resume_stat_date(
|
||||
repo, "snapshot", fallback="2026-11-03",
|
||||
) == "2026-11-02"
|
||||
state = load_state(repo, "snapshot")
|
||||
assert state is not None
|
||||
assert state.status == STATUS_PENDING
|
||||
assert state.progress == {"done": 3, "total": 10}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_scheduler_due_reader_picks_up_enqueued_seal():
|
||||
"""D: 同进程 seed 写盘 due_at=now 后,下一轮 run_due 须跑 seal。"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, repo = _repo(tmp)
|
||||
try:
|
||||
now = datetime(2026, 11, 3, 23, 20, tzinfo=SHANGHAI_TZ).timestamp()
|
||||
# snapshot 原 next_run 挂次日;盘上被 enqueue 成 now
|
||||
tomorrow = next_daily_run(now, "23:15")
|
||||
calls = []
|
||||
|
||||
def run_snap():
|
||||
calls.append("snap")
|
||||
return None
|
||||
|
||||
def due_reader(name: str):
|
||||
st = load_state(repo, name)
|
||||
return float(st.due_at) if st and st.due_at else None
|
||||
|
||||
clock = [now]
|
||||
sched = Scheduler(
|
||||
[ScheduledTask("snapshot", 86400.0, run_snap, next_run=tomorrow)],
|
||||
clock=lambda: clock[0],
|
||||
due_reader=due_reader,
|
||||
)
|
||||
assert sched.run_due() == []
|
||||
assert calls == []
|
||||
ensure_pending(
|
||||
repo, "snapshot", due_at=now, stat_date="2026-11-03", now=now,
|
||||
)
|
||||
results = sched.run_due()
|
||||
assert len(results) == 1
|
||||
assert calls == ["snap"]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def main_test():
|
||||
test_restart_mid_run_resumes_same_stat_date()
|
||||
test_seeded_before_snapshot_at_does_not_force_run()
|
||||
@@ -259,6 +387,10 @@ def main_test():
|
||||
test_done_writes_next_due_and_blocks_until_then()
|
||||
test_no_state_falls_back_to_wall_clock()
|
||||
test_fail_or_defer_preserves_progress()
|
||||
test_bootstrap_from_last_fail_due_soon()
|
||||
test_ensure_pending_does_not_glue_yesterday_stat_date()
|
||||
test_cross_midnight_mid_seal_resumes_yesterday()
|
||||
test_scheduler_due_reader_picks_up_enqueued_seal()
|
||||
print("OK: task_state persistence / resume / due_at gates")
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user