fix(scheduler): drop cross-task mutex, quiet window + retry
Incident: evening full snapshot could be recorded as ok with already_completed/elapsed=0 after mutual skip with hourly sync, then next_run jumped to the next day. - Full-site snapshot never skips because another task is running - 23:00-01:00 Asia/Shanghai: defer discovery / snapshot_recent / candidate_cleanup to after 01:00; fans never defers - Skip records ok=false + skipped=true (not success / not completed) - Skip/fail reschedule soon (~5m); daily success still next snapshot_at - TaskControl return value drives next_run; system UI shows skip fields
This commit is contained in:
bot-ci committed
1 parent
39378c65b2
commit
9969e65202
10 files changed
+411
-84
No files matched your search
@@ -610,12 +610,15 @@ def test_growth_auto_falls_back_to_one_day():
|
||||
|
||||
def test_snapshot_recent_next_run_follows_completion():
|
||||
import json
|
||||
from datetime import datetime
|
||||
from reaction_board.timeutil import SHANGHAI_TZ, interval_next_run
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, q = setup(tmp)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
finished = FIXED_NOW - 10
|
||||
# 午后完成：间隔后仍在静默窗外，应严格 finished+3600
|
||||
finished = int(datetime(2026, 10, 31, 15, 0, 0, tzinfo=SHANGHAI_TZ).timestamp())
|
||||
repo.kv_set(
|
||||
"task:last:snapshot_recent",
|
||||
json.dumps({
|
||||
@@ -627,6 +630,83 @@ def test_snapshot_recent_next_run_follows_completion():
|
||||
DASHBOARD_CACHE.clear()
|
||||
system = q.system()
|
||||
assert system["next_runs"]["snapshot_recent"] == finished + 3600
|
||||
assert system["next_runs"]["snapshot_recent"] == int(
|
||||
interval_next_run(float(finished), 3600)
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_snapshot_recent_next_run_defers_quiet_window_and_retry_on_skip():
|
||||
import json
|
||||
from datetime import datetime
|
||||
from reaction_board.timeutil import SHANGHAI_TZ, next_after_snapshot_quiet_window
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, q = setup(tmp)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
# 22:50 成功 → 下一档 23:50 落入静默窗 → 推到 01:00
|
||||
finished = int(datetime(2026, 10, 31, 22, 50, 0, tzinfo=SHANGHAI_TZ).timestamp())
|
||||
repo.kv_set(
|
||||
"task:last:snapshot_recent",
|
||||
json.dumps({
|
||||
"name": "snapshot_recent", "ok": True,
|
||||
"started_at": finished - 5, "finished_at": finished,
|
||||
"error": None, "summary": {},
|
||||
}),
|
||||
)
|
||||
DASHBOARD_CACHE.clear()
|
||||
system = q.system()
|
||||
expected = int(datetime(2026, 11, 1, 1, 0, 0, tzinfo=SHANGHAI_TZ).timestamp())
|
||||
assert system["next_runs"]["snapshot_recent"] == expected
|
||||
|
||||
# skip 记录：补跑到窗后 01:00，不得当成功推进
|
||||
quiet = int(datetime(2026, 10, 31, 23, 20, 0, tzinfo=SHANGHAI_TZ).timestamp())
|
||||
repo.kv_set(
|
||||
"task:last:snapshot_recent",
|
||||
json.dumps({
|
||||
"name": "snapshot_recent", "ok": False, "skipped": True,
|
||||
"started_at": quiet, "finished_at": quiet,
|
||||
"error": "snapshot_quiet_window",
|
||||
"summary": {"skipped": True, "reason": "snapshot_quiet_window"},
|
||||
}),
|
||||
)
|
||||
DASHBOARD_CACHE.clear()
|
||||
system = q.system()
|
||||
assert system["next_runs"]["snapshot_recent"] == int(
|
||||
next_after_snapshot_quiet_window(float(quiet))
|
||||
)
|
||||
assert not any(e["task"] == "snapshot_recent" for e in system["errors"])
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_snapshot_failed_next_run_retries_soon_not_next_day():
|
||||
import json
|
||||
from datetime import datetime
|
||||
from reaction_board.timeutil import SHANGHAI_TZ, next_daily_run
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, q = setup(tmp)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
finished = int(datetime(2026, 10, 31, 23, 15, 0, tzinfo=SHANGHAI_TZ).timestamp())
|
||||
repo.kv_set(
|
||||
"task:last:snapshot",
|
||||
json.dumps({
|
||||
"name": "snapshot", "ok": False,
|
||||
"started_at": finished - 1, "finished_at": finished,
|
||||
"error": "boom", "summary": {"completed": False},
|
||||
}),
|
||||
)
|
||||
DASHBOARD_CACHE.clear()
|
||||
system = q.system()
|
||||
assert system["next_runs"]["snapshot"] == finished + 300
|
||||
assert system["next_runs"]["snapshot"] != int(
|
||||
next_daily_run(finished, "23:15")
|
||||
)
|
||||
assert any(e["task"] == "snapshot" for e in system["errors"])
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -882,6 +962,8 @@ def main_test():
|
||||
test_system_tasks_sorted_by_started_at_desc()
|
||||
test_growth_auto_falls_back_to_one_day()
|
||||
test_snapshot_recent_next_run_follows_completion()
|
||||
test_snapshot_recent_next_run_defers_quiet_window_and_retry_on_skip()
|
||||
test_snapshot_failed_next_run_retries_soon_not_next_day()
|
||||
test_overview_adjacent_previous_with_seed_and_dual_fields()
|
||||
test_same_day_growth_baselines()
|
||||
test_growth_rate_sort_uses_prior()
|
||||
@@ -948,13 +1030,15 @@ def test_system_status_uses_completed_date_and_running_flags():
|
||||
|
||||
def test_system_next_runs_follow_last_completion_not_wall_clock():
|
||||
import json
|
||||
from datetime import datetime
|
||||
from reaction_board.timeutil import SHANGHAI_TZ
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db, q = setup(tmp)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
# 发现任务刚完成不久：下次 = finished + 60min，即使 now 已超过也不跳档
|
||||
finished = FIXED_NOW - 30
|
||||
# 午后完成：下次 = finished + 60min，不跳档、不误入静默窗推算
|
||||
finished = int(datetime(2026, 10, 31, 15, 0, 0, tzinfo=SHANGHAI_TZ).timestamp())
|
||||
repo.kv_set(
|
||||
"task:last:discovery",
|
||||
json.dumps({
|
||||
@@ -966,7 +1050,6 @@ def test_system_next_runs_follow_last_completion_not_wall_clock():
|
||||
DASHBOARD_CACHE.clear()
|
||||
system = q.system()
|
||||
assert system["next_runs"]["discover"] == finished + 3600
|
||||
# 仍小于/等于 now 也可以（表示本轮后的下一档已到期或未到），但绝不能跳到 +7200
|
||||
assert system["next_runs"]["discover"] != finished + 7200
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
Reference in new issue
Block a user