fix(scheduler): do not force full snapshot on post-seed restart

Schedule snapshot_seed/snapshot/fans to the next wall-clock seed_at /
snapshot_at on process start (optionally from last task record), instead
of next_run=0. Force-seal of seed shells only after today's snapshot_at
(or CLI --force), so a daytime restart cannot scrape the whole site.
This commit is contained in:
bot-ci committed 2026-11-03 01:09:50 +08:00
1 parent 1697573d95
commit 51ffa44577
3 files changed
+241 -10

No files matched your search

+75 -10
View File
@@ -48,6 +48,7 @@ from .timeutil import (
next_daily_run,
parse_hhmm,
shanghai_date,
shanghai_datetime,
)
ROOT = Path(__file__).resolve().parent.parent
@@ -297,6 +298,53 @@ def _defer_snapshot_seed(
)
def _initial_daily_next_run(
repo: Repository,
name: str,
at: str,
*,
now: float | None = None,
retry_seconds: float | None = None,
) -> float:
"""进程启动时日更任务的下次运行:墙钟下一档,或与系统页一致的上次记录推导。
不得返回 ``0`` / 过去时间来“立刻跑一遍”——午夜 seed 后重启否则会白天全量强采。
"""
current = time.time() if now is None else float(now)
wall = next_daily_run(current, at)
raw = repo.kv_get(f"task:last:{name}")
if not raw:
return wall
try:
record = json.loads(raw)
except (TypeError, ValueError, json.JSONDecodeError):
return wall
if not isinstance(record, dict) or record.get("finished_at") is None:
return wall
finished_at = float(record["finished_at"])
summary = record.get("summary") if isinstance(record.get("summary"), dict) else {}
skipped = bool(record.get("skipped") or summary.get("skipped"))
failed = record.get("ok") is False
if skipped or failed:
gap = (
_SEED_RETRY_SECONDS if name == "snapshot_seed"
else float(_TASK_RETRY_SECONDS if retry_seconds is None else retry_seconds)
)
retry_at = finished_at + gap
return retry_at if retry_at > current else wall
scheduled = next_daily_run(finished_at, at)
# 成功记录推到的下一档若已过,退回墙钟下一档(严格 > now)
return scheduled if scheduled > current else wall
def _reached_daily_slot(at: str, now: float | None = None) -> bool:
"""当前上海时间是否已到达(含)今日 ``at`` 墙钟点。"""
current = shanghai_datetime(now)
hour, minute = parse_hhmm(at)
slot = current.replace(hour=hour, minute=minute, second=0, microsecond=0)
return current >= slot
def _client(credentials, logger: JsonLogger, *, pacer=None, retries: int = 3) -> BilibiliClient:
return BilibiliClient(
cookie=credentials.cookie,
@@ -769,11 +817,13 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
started = int(time.time())
_mark_task_running(repo, "snapshot", started_at=started)
try:
# 晚间全量:若当日是午夜 seed 壳,必须 force 打穿 ok=1 克隆行。
# 仅晚间 snapshot_at 及之后:seed 壳才 force 打穿 ok=1 克隆行。
# 白天/重启不得因 seeded 标记立刻全量强采。
today = shanghai_date()
force_seed = repo.kv_get(
SnapshotService.seeded_key(today)
) == "1"
force_seed = (
repo.kv_get(SnapshotService.seeded_key(today)) == "1"
and _reached_daily_slot(snapshot_at, float(started))
)
summary = snapshot_service.run_if_needed(force=force_seed)
except Exception as exc: # noqa: BLE001
finished = int(time.time())
@@ -842,9 +892,10 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
_mark_task_running(repo, "fans", started_at=started)
try:
today = shanghai_date()
force_seed = repo.kv_get(
SnapshotService.fans_seeded_key(today)
) == "1"
force_seed = (
repo.kv_get(SnapshotService.fans_seeded_key(today)) == "1"
and _reached_daily_slot(snapshot_at, float(started))
)
summary = snapshot_service.run_fans_if_needed(force=force_seed)
except Exception as exc: # noqa: BLE001
finished = int(time.time())
@@ -927,25 +978,39 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
logger.info("config.reloaded", revision=store.revision)
return {"revision": store.revision}
# --once:保持 next_run=0 立刻跑到期逻辑;常驻:日更对齐墙钟,避免重启即强采。
now_ts = time.time()
if args.once:
seed_next = snapshot_next = fans_next = 0.0
else:
seed_next = _initial_daily_next_run(
repo, "snapshot_seed", seed_at, now=now_ts,
)
snapshot_next = _initial_daily_next_run(
repo, "snapshot", snapshot_at, now=now_ts,
)
fans_next = _initial_daily_next_run(
repo, "fans", snapshot_at, now=now_ts,
)
scheduler = Scheduler(
[
ScheduledTask(
"snapshot_seed",
86400.0,
run_snapshot_seed,
next_run=0.0,
next_run=seed_next,
),
ScheduledTask(
"snapshot",
86400.0,
run_snapshot,
next_run=0.0,
next_run=snapshot_next,
),
ScheduledTask(
"fans",
86400.0,
run_fans,
next_run=0.0,
next_run=fans_next,
),
ScheduledTask(
"snapshot_recent",