fix(scheduler): do not force full snapshot on post-seed restart
Schedule snapshot_seed/snapshot/fans to the next wall-clock seed_at / snapshot_at on process start (optionally from last task record), instead of next_run=0. Force-seal of seed shells only after today's snapshot_at (or CLI --force), so a daytime restart cannot scrape the whole site.
This commit is contained in:
1 parent
1697573d95
commit
51ffa44577
3 files changed
+241
-10
No files matched your search
+75
-10
@@ -48,6 +48,7 @@ from .timeutil import (
|
||||
next_daily_run,
|
||||
parse_hhmm,
|
||||
shanghai_date,
|
||||
shanghai_datetime,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
@@ -297,6 +298,53 @@ def _defer_snapshot_seed(
|
||||
)
|
||||
|
||||
|
||||
def _initial_daily_next_run(
|
||||
repo: Repository,
|
||||
name: str,
|
||||
at: str,
|
||||
*,
|
||||
now: float | None = None,
|
||||
retry_seconds: float | None = None,
|
||||
) -> float:
|
||||
"""进程启动时日更任务的下次运行:墙钟下一档,或与系统页一致的上次记录推导。
|
||||
|
||||
不得返回 ``0`` / 过去时间来“立刻跑一遍”——午夜 seed 后重启否则会白天全量强采。
|
||||
"""
|
||||
current = time.time() if now is None else float(now)
|
||||
wall = next_daily_run(current, at)
|
||||
raw = repo.kv_get(f"task:last:{name}")
|
||||
if not raw:
|
||||
return wall
|
||||
try:
|
||||
record = json.loads(raw)
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
return wall
|
||||
if not isinstance(record, dict) or record.get("finished_at") is None:
|
||||
return wall
|
||||
finished_at = float(record["finished_at"])
|
||||
summary = record.get("summary") if isinstance(record.get("summary"), dict) else {}
|
||||
skipped = bool(record.get("skipped") or summary.get("skipped"))
|
||||
failed = record.get("ok") is False
|
||||
if skipped or failed:
|
||||
gap = (
|
||||
_SEED_RETRY_SECONDS if name == "snapshot_seed"
|
||||
else float(_TASK_RETRY_SECONDS if retry_seconds is None else retry_seconds)
|
||||
)
|
||||
retry_at = finished_at + gap
|
||||
return retry_at if retry_at > current else wall
|
||||
scheduled = next_daily_run(finished_at, at)
|
||||
# 成功记录推到的下一档若已过,退回墙钟下一档(严格 > now)
|
||||
return scheduled if scheduled > current else wall
|
||||
|
||||
|
||||
def _reached_daily_slot(at: str, now: float | None = None) -> bool:
|
||||
"""当前上海时间是否已到达(含)今日 ``at`` 墙钟点。"""
|
||||
current = shanghai_datetime(now)
|
||||
hour, minute = parse_hhmm(at)
|
||||
slot = current.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
||||
return current >= slot
|
||||
|
||||
|
||||
def _client(credentials, logger: JsonLogger, *, pacer=None, retries: int = 3) -> BilibiliClient:
|
||||
return BilibiliClient(
|
||||
cookie=credentials.cookie,
|
||||
@@ -769,11 +817,13 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
started = int(time.time())
|
||||
_mark_task_running(repo, "snapshot", started_at=started)
|
||||
try:
|
||||
# 晚间全量:若当日是午夜 seed 壳,必须 force 打穿 ok=1 克隆行。
|
||||
# 仅晚间 snapshot_at 及之后:seed 壳才 force 打穿 ok=1 克隆行。
|
||||
# 白天/重启不得因 seeded 标记立刻全量强采。
|
||||
today = shanghai_date()
|
||||
force_seed = repo.kv_get(
|
||||
SnapshotService.seeded_key(today)
|
||||
) == "1"
|
||||
force_seed = (
|
||||
repo.kv_get(SnapshotService.seeded_key(today)) == "1"
|
||||
and _reached_daily_slot(snapshot_at, float(started))
|
||||
)
|
||||
summary = snapshot_service.run_if_needed(force=force_seed)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
@@ -842,9 +892,10 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
_mark_task_running(repo, "fans", started_at=started)
|
||||
try:
|
||||
today = shanghai_date()
|
||||
force_seed = repo.kv_get(
|
||||
SnapshotService.fans_seeded_key(today)
|
||||
) == "1"
|
||||
force_seed = (
|
||||
repo.kv_get(SnapshotService.fans_seeded_key(today)) == "1"
|
||||
and _reached_daily_slot(snapshot_at, float(started))
|
||||
)
|
||||
summary = snapshot_service.run_fans_if_needed(force=force_seed)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
@@ -927,25 +978,39 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
logger.info("config.reloaded", revision=store.revision)
|
||||
return {"revision": store.revision}
|
||||
|
||||
# --once:保持 next_run=0 立刻跑到期逻辑;常驻:日更对齐墙钟,避免重启即强采。
|
||||
now_ts = time.time()
|
||||
if args.once:
|
||||
seed_next = snapshot_next = fans_next = 0.0
|
||||
else:
|
||||
seed_next = _initial_daily_next_run(
|
||||
repo, "snapshot_seed", seed_at, now=now_ts,
|
||||
)
|
||||
snapshot_next = _initial_daily_next_run(
|
||||
repo, "snapshot", snapshot_at, now=now_ts,
|
||||
)
|
||||
fans_next = _initial_daily_next_run(
|
||||
repo, "fans", snapshot_at, now=now_ts,
|
||||
)
|
||||
scheduler = Scheduler(
|
||||
[
|
||||
ScheduledTask(
|
||||
"snapshot_seed",
|
||||
86400.0,
|
||||
run_snapshot_seed,
|
||||
next_run=0.0,
|
||||
next_run=seed_next,
|
||||
),
|
||||
ScheduledTask(
|
||||
"snapshot",
|
||||
86400.0,
|
||||
run_snapshot,
|
||||
next_run=0.0,
|
||||
next_run=snapshot_next,
|
||||
),
|
||||
ScheduledTask(
|
||||
"fans",
|
||||
86400.0,
|
||||
run_fans,
|
||||
next_run=0.0,
|
||||
next_run=fans_next,
|
||||
),
|
||||
ScheduledTask(
|
||||
"snapshot_recent",
|
||||
|
||||
Reference in new issue
Block a user