feat(scheduler): persist task state/progress/due_at across restart (#8)
Replace process-start next_run=0 guesses with kv task:state records (status, due_at, stat_date, progress). Startup recovers stale running, honors future due_at for delayed retries, and resumes the same stat_date. Seed only enqueues evening seal; seal force keeps non-seed ok=1 as progress (keep_scraped). Keeps !7 fail-logging and cleanup retry.
This commit is contained in:
1 parent
51ffa44577
commit
f408da93c2
7 files changed
+1793
-902
No files matched your search
+417
-115
@@ -40,6 +40,7 @@ from .services.library import LibraryService
|
||||
from .services.snapshot import SnapshotService
|
||||
from .services.telegram_poller import TelegramPoller
|
||||
from .services.up_profile import UpProfileService
|
||||
from . import task_state as task_state_mod
|
||||
from .timeutil import (
|
||||
in_snapshot_quiet_window,
|
||||
interval_next_run,
|
||||
@@ -164,17 +165,29 @@ def _task_running_key(name: str) -> str:
|
||||
return f"task:running:{name}"
|
||||
|
||||
|
||||
def _mark_task_running(repo: Repository, name: str, *, started_at: int | None = None) -> None:
|
||||
"""任务开始时写入 running 标记,供 /system 显示「运行中」。"""
|
||||
now = int(time.time())
|
||||
def _mark_task_running(
|
||||
repo: Repository,
|
||||
name: str,
|
||||
*,
|
||||
started_at: int | None = None,
|
||||
stat_date: str | None = None,
|
||||
) -> None:
|
||||
"""任务开始:running 标记 + 持久化 status=running(跨重启可续跑)。"""
|
||||
now = int(started_at if started_at is not None else time.time())
|
||||
record = {
|
||||
"name": name,
|
||||
"started_at": int(started_at if started_at is not None else now),
|
||||
"started_at": now,
|
||||
}
|
||||
try:
|
||||
repo.kv_set(_task_running_key(name), json.dumps(record, ensure_ascii=False))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
try:
|
||||
task_state_mod.mark_running(
|
||||
repo, name, stat_date=stat_date, now=float(now),
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
def _clear_task_running(repo: Repository, name: str) -> None:
|
||||
@@ -202,20 +215,26 @@ def _record_task(
|
||||
payload=None,
|
||||
error: str | None = None,
|
||||
skipped: bool = False,
|
||||
next_due_at: float | None = None,
|
||||
stat_date: str | None = None,
|
||||
progress=None,
|
||||
) -> None:
|
||||
"""把最近一次任务结果写入 kv,并清除 running 标记。
|
||||
|
||||
``skipped=True`` 表示主动延后/避让,不得记成成功,也不推进「当日已完成」。
|
||||
同步写入 ``task:state``:有 ``next_due_at`` 用其排程;否则结束残留
|
||||
``running`` 并保留原 ``due_at``(CLI 一锤子路径)。
|
||||
"""
|
||||
now = int(time.time())
|
||||
if skipped:
|
||||
ok = False
|
||||
finished = int(finished_at if finished_at is not None else now)
|
||||
record = {
|
||||
"name": name,
|
||||
"ok": bool(ok),
|
||||
"skipped": bool(skipped),
|
||||
"started_at": int(started_at if started_at is not None else now),
|
||||
"finished_at": int(finished_at if finished_at is not None else now),
|
||||
"finished_at": finished,
|
||||
"error": error,
|
||||
"summary": payload,
|
||||
}
|
||||
@@ -224,6 +243,52 @@ def _record_task(
|
||||
except Exception: # noqa: BLE001 任务状态记录失败不影响主流程
|
||||
pass
|
||||
_clear_task_running(repo, name)
|
||||
try:
|
||||
if next_due_at is not None:
|
||||
due = float(next_due_at)
|
||||
else:
|
||||
# CLI 等非调度路径:结束 running,保留原 due_at,避免残留 running。
|
||||
prev = task_state_mod.load_state(repo, name)
|
||||
if prev is None or prev.status != task_state_mod.STATUS_RUNNING:
|
||||
return
|
||||
due = (
|
||||
float(prev.due_at)
|
||||
if prev.due_at and float(prev.due_at) > 0
|
||||
else float(finished)
|
||||
)
|
||||
if stat_date is None:
|
||||
stat_date = prev.stat_date
|
||||
if skipped:
|
||||
task_state_mod.mark_deferred(
|
||||
repo,
|
||||
name,
|
||||
due_at=due,
|
||||
now=float(finished),
|
||||
stat_date=stat_date,
|
||||
progress=progress,
|
||||
error=error,
|
||||
)
|
||||
elif ok:
|
||||
task_state_mod.mark_done(
|
||||
repo,
|
||||
name,
|
||||
due_at=due,
|
||||
now=float(finished),
|
||||
stat_date=stat_date,
|
||||
progress=progress,
|
||||
)
|
||||
else:
|
||||
task_state_mod.mark_failed(
|
||||
repo,
|
||||
name,
|
||||
due_at=due,
|
||||
now=float(finished),
|
||||
stat_date=stat_date,
|
||||
progress=progress,
|
||||
error=error,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
# 全站快照 / 粉丝失败后的补跑间隔(秒)
|
||||
@@ -245,6 +310,7 @@ def _quiet_window_defer(
|
||||
"""静默窗内不执行:记 skip(非成功),并排到 01:00 之后。"""
|
||||
finished = int(time.time())
|
||||
logger.info(f"{name}.skip", reason=_QUIET_SKIP_REASON)
|
||||
next_run = next_after_snapshot_quiet_window(float(finished))
|
||||
_record_task(
|
||||
repo,
|
||||
name,
|
||||
@@ -254,10 +320,11 @@ def _quiet_window_defer(
|
||||
finished_at=finished,
|
||||
payload={"skipped": True, "reason": _QUIET_SKIP_REASON},
|
||||
error=_QUIET_SKIP_REASON,
|
||||
next_due_at=next_run,
|
||||
)
|
||||
return TaskControl(
|
||||
ok=False,
|
||||
next_run=next_after_snapshot_quiet_window(float(finished)),
|
||||
next_run=next_run,
|
||||
error=_QUIET_SKIP_REASON,
|
||||
)
|
||||
|
||||
@@ -281,6 +348,7 @@ def _defer_snapshot_seed(
|
||||
"""seed 因 snapshot 占用 / no_source 延后:记 skip(非成功),10 分钟后再试。"""
|
||||
finished = int(finished_at if finished_at is not None else time.time())
|
||||
logger.info("snapshot_seed.skip", reason=reason)
|
||||
next_run = _seed_retry_soon(float(finished))
|
||||
_record_task(
|
||||
repo,
|
||||
"snapshot_seed",
|
||||
@@ -290,10 +358,11 @@ def _defer_snapshot_seed(
|
||||
finished_at=finished,
|
||||
payload={"skipped": True, "reason": reason},
|
||||
error=reason,
|
||||
next_due_at=next_run,
|
||||
)
|
||||
return TaskControl(
|
||||
ok=False,
|
||||
next_run=_seed_retry_soon(float(finished)),
|
||||
next_run=next_run,
|
||||
error=reason,
|
||||
)
|
||||
|
||||
@@ -306,35 +375,17 @@ def _initial_daily_next_run(
|
||||
now: float | None = None,
|
||||
retry_seconds: float | None = None,
|
||||
) -> float:
|
||||
"""进程启动时日更任务的下次运行:墙钟下一档,或与系统页一致的上次记录推导。
|
||||
"""进程启动时日更任务的下次运行:优先 ``task:state.due_at``。
|
||||
|
||||
不得返回 ``0`` / 过去时间来“立刻跑一遍”——午夜 seed 后重启否则会白天全量强采。
|
||||
不得返回 ``0`` / 用「无状态」以外的过去时间去误伤——无状态时对齐墙钟。
|
||||
失败且 ``due_at`` 已过:返回该 ``due_at``(调度器会立刻补跑),不退回次日墙钟。
|
||||
"""
|
||||
del retry_seconds # 间隔已落在 task:state.due_at
|
||||
current = time.time() if now is None else float(now)
|
||||
wall = next_daily_run(current, at)
|
||||
raw = repo.kv_get(f"task:last:{name}")
|
||||
if not raw:
|
||||
return wall
|
||||
try:
|
||||
record = json.loads(raw)
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
return wall
|
||||
if not isinstance(record, dict) or record.get("finished_at") is None:
|
||||
return wall
|
||||
finished_at = float(record["finished_at"])
|
||||
summary = record.get("summary") if isinstance(record.get("summary"), dict) else {}
|
||||
skipped = bool(record.get("skipped") or summary.get("skipped"))
|
||||
failed = record.get("ok") is False
|
||||
if skipped or failed:
|
||||
gap = (
|
||||
_SEED_RETRY_SECONDS if name == "snapshot_seed"
|
||||
else float(_TASK_RETRY_SECONDS if retry_seconds is None else retry_seconds)
|
||||
)
|
||||
retry_at = finished_at + gap
|
||||
return retry_at if retry_at > current else wall
|
||||
scheduled = next_daily_run(finished_at, at)
|
||||
# 成功记录推到的下一档若已过,退回墙钟下一档(严格 > now)
|
||||
return scheduled if scheduled > current else wall
|
||||
return task_state_mod.startup_next_run(
|
||||
repo, name, wall_due_at=wall, now=current,
|
||||
)
|
||||
|
||||
|
||||
def _reached_daily_slot(at: str, now: float | None = None) -> bool:
|
||||
@@ -345,6 +396,38 @@ def _reached_daily_slot(at: str, now: float | None = None) -> bool:
|
||||
return current >= slot
|
||||
|
||||
|
||||
def _evening_due_at(now: float, snapshot_at: str) -> float:
|
||||
"""尚未到当日 snapshot_at 则挂今晚,否则下一档墙钟。"""
|
||||
hour, minute = parse_hhmm(snapshot_at)
|
||||
dt = shanghai_datetime(now)
|
||||
slot = dt.replace(hour=hour, minute=minute, second=0, microsecond=0).timestamp()
|
||||
if slot > now:
|
||||
return float(slot)
|
||||
return float(next_daily_run(now, snapshot_at))
|
||||
|
||||
|
||||
def _enqueue_evening_seal(
|
||||
repo: Repository,
|
||||
*,
|
||||
stat_date: str,
|
||||
snapshot_at: str,
|
||||
now: float,
|
||||
) -> None:
|
||||
"""seed 成功后:当晚 snapshot / fans 记 pending,due_at=当日 snapshot_at。"""
|
||||
current = shanghai_datetime(now)
|
||||
hour, minute = parse_hhmm(snapshot_at)
|
||||
slot_ts = current.replace(
|
||||
hour=hour, minute=minute, second=0, microsecond=0,
|
||||
).timestamp()
|
||||
due = slot_ts if slot_ts > now else now
|
||||
task_state_mod.ensure_pending(
|
||||
repo, "snapshot", due_at=due, stat_date=stat_date, now=now,
|
||||
)
|
||||
task_state_mod.ensure_pending(
|
||||
repo, "fans", due_at=due, stat_date=stat_date, now=now,
|
||||
)
|
||||
|
||||
|
||||
def _client(credentials, logger: JsonLogger, *, pacer=None, retries: int = 3) -> BilibiliClient:
|
||||
return BilibiliClient(
|
||||
cookie=credentials.cookie,
|
||||
@@ -445,51 +528,120 @@ def cmd_snapshot(args, logger: JsonLogger) -> int:
|
||||
)
|
||||
service = SnapshotService(store, repo, client, logger=logger)
|
||||
started = int(time.time())
|
||||
_mark_task_running(repo, "snapshot", started_at=started)
|
||||
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
|
||||
snapshot_at = f"{hh:02d}:{mm:02d}"
|
||||
_mark_task_running(
|
||||
repo, "snapshot", started_at=started,
|
||||
stat_date=shanghai_date(float(started)),
|
||||
)
|
||||
try:
|
||||
video_summary = service.run_if_needed(force=args.force)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
_record_task(
|
||||
repo, "snapshot", ok=False, started_at=started,
|
||||
finished_at=int(time.time()), error=str(exc),
|
||||
finished_at=finished, error=str(exc),
|
||||
next_due_at=_retry_soon(float(finished)),
|
||||
stat_date=shanghai_date(float(started)),
|
||||
)
|
||||
raise
|
||||
video_payload = video_summary.as_dict()
|
||||
video_payload["database"] = str(cfg.storage.database_path)
|
||||
video_payload["snapshot_counts"] = repo.snapshot_counts(video_summary.stat_date)
|
||||
video_payload["counts"] = repo.counts()
|
||||
video_finished = int(video_summary.finished_at or time.time())
|
||||
video_ok = bool(video_summary.completed and not video_summary.backup_error)
|
||||
_record_task(
|
||||
repo, "snapshot", ok=video_ok,
|
||||
started_at=int(video_summary.started_at or started),
|
||||
finished_at=int(video_summary.finished_at or time.time()),
|
||||
payload=video_payload,
|
||||
error=video_summary.backup_error,
|
||||
)
|
||||
if video_summary.reason == "seeded_pending_seal":
|
||||
evening = _evening_due_at(float(started), snapshot_at)
|
||||
_enqueue_evening_seal(
|
||||
repo,
|
||||
stat_date=video_summary.stat_date,
|
||||
snapshot_at=snapshot_at,
|
||||
now=float(video_finished),
|
||||
)
|
||||
_record_task(
|
||||
repo, "snapshot", ok=False, skipped=True,
|
||||
started_at=int(video_summary.started_at or started),
|
||||
finished_at=video_finished,
|
||||
payload=video_payload,
|
||||
error="seeded_pending_seal",
|
||||
next_due_at=evening,
|
||||
stat_date=video_summary.stat_date,
|
||||
)
|
||||
elif video_ok:
|
||||
_record_task(
|
||||
repo, "snapshot", ok=True,
|
||||
started_at=int(video_summary.started_at or started),
|
||||
finished_at=video_finished,
|
||||
payload=video_payload,
|
||||
error=video_summary.backup_error,
|
||||
next_due_at=next_daily_run(float(video_finished), snapshot_at),
|
||||
stat_date=video_summary.stat_date,
|
||||
)
|
||||
else:
|
||||
_record_task(
|
||||
repo, "snapshot", ok=False,
|
||||
started_at=int(video_summary.started_at or started),
|
||||
finished_at=video_finished,
|
||||
payload=video_payload,
|
||||
error=video_summary.backup_error or "snapshot_incomplete",
|
||||
next_due_at=_retry_soon(float(video_finished)),
|
||||
stat_date=video_summary.stat_date,
|
||||
)
|
||||
|
||||
fans_started = int(time.time())
|
||||
_mark_task_running(repo, "fans", started_at=fans_started)
|
||||
_mark_task_running(
|
||||
repo, "fans", started_at=fans_started,
|
||||
stat_date=video_summary.stat_date,
|
||||
)
|
||||
try:
|
||||
fans_summary = service.run_fans_if_needed(
|
||||
date=video_summary.stat_date, force=args.force,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
_record_task(
|
||||
repo, "fans", ok=False, started_at=fans_started,
|
||||
finished_at=int(time.time()), error=str(exc),
|
||||
finished_at=finished, error=str(exc),
|
||||
next_due_at=_retry_soon(float(finished)),
|
||||
stat_date=video_summary.stat_date,
|
||||
)
|
||||
raise
|
||||
fans_payload = fans_summary.as_dict()
|
||||
fans_finished = int(fans_summary.finished_at or time.time())
|
||||
fans_ok = fans_summary.fans_failed == 0
|
||||
_record_task(
|
||||
repo, "fans", ok=fans_ok,
|
||||
started_at=int(fans_summary.started_at or fans_started),
|
||||
finished_at=int(fans_summary.finished_at or time.time()),
|
||||
payload=fans_payload,
|
||||
error=";".join(
|
||||
err for err in fans_summary.errors if str(err).startswith("fans ")
|
||||
) or None,
|
||||
)
|
||||
if fans_summary.reason == "seeded_pending_seal":
|
||||
evening = _evening_due_at(float(fans_started), snapshot_at)
|
||||
_record_task(
|
||||
repo, "fans", ok=False, skipped=True,
|
||||
started_at=int(fans_summary.started_at or fans_started),
|
||||
finished_at=fans_finished,
|
||||
payload=fans_payload,
|
||||
error="seeded_pending_seal",
|
||||
next_due_at=evening,
|
||||
stat_date=video_summary.stat_date,
|
||||
)
|
||||
elif fans_ok:
|
||||
_record_task(
|
||||
repo, "fans", ok=True,
|
||||
started_at=int(fans_summary.started_at or fans_started),
|
||||
finished_at=fans_finished,
|
||||
payload=fans_payload,
|
||||
next_due_at=next_daily_run(float(fans_finished), snapshot_at),
|
||||
stat_date=video_summary.stat_date,
|
||||
)
|
||||
else:
|
||||
_record_task(
|
||||
repo, "fans", ok=False,
|
||||
started_at=int(fans_summary.started_at or fans_started),
|
||||
finished_at=fans_finished,
|
||||
payload=fans_payload,
|
||||
error=";".join(
|
||||
err for err in fans_summary.errors if str(err).startswith("fans ")
|
||||
) or None,
|
||||
next_due_at=_retry_soon(float(fans_finished)),
|
||||
stat_date=video_summary.stat_date,
|
||||
)
|
||||
payload = {
|
||||
**video_payload,
|
||||
"fans_ok": fans_summary.fans_ok,
|
||||
@@ -525,32 +677,45 @@ def cmd_snapshot_seed(args, logger: JsonLogger) -> int:
|
||||
"stat_date": target or shanghai_date(),
|
||||
"database": str(cfg.storage.database_path),
|
||||
}
|
||||
finished = int(time.time())
|
||||
_record_task(
|
||||
repo,
|
||||
"snapshot_seed",
|
||||
ok=False,
|
||||
skipped=True,
|
||||
started_at=started,
|
||||
finished_at=int(time.time()),
|
||||
finished_at=finished,
|
||||
payload=payload,
|
||||
error=_SEED_SKIP_SNAPSHOT_RUNNING,
|
||||
next_due_at=_seed_retry_soon(float(finished)),
|
||||
stat_date=payload["stat_date"],
|
||||
)
|
||||
_print(payload)
|
||||
return 1
|
||||
# seed 只读本地库,不调 B 站;占位 client 满足构造约束。
|
||||
service = SnapshotService(store, repo, object(), logger=logger)
|
||||
_mark_task_running(repo, "snapshot_seed", started_at=started)
|
||||
seed_date = target or shanghai_date()
|
||||
_mark_task_running(
|
||||
repo, "snapshot_seed", started_at=started, stat_date=seed_date,
|
||||
)
|
||||
try:
|
||||
summary = service.run_seed_from_last_completed(date=target)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
_record_task(
|
||||
repo, "snapshot_seed", ok=False, started_at=started,
|
||||
finished_at=int(time.time()), error=str(exc),
|
||||
finished_at=finished, error=str(exc),
|
||||
next_due_at=_seed_retry_soon(float(finished)),
|
||||
stat_date=seed_date,
|
||||
)
|
||||
raise
|
||||
payload = summary.as_dict()
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
finished = int(summary.finished_at or time.time())
|
||||
hh, mm = parse_hhmm(cfg.schedule.seed_at)
|
||||
seed_at = f"{hh:02d}:{mm:02d}"
|
||||
sh, sm = parse_hhmm(cfg.schedule.snapshot_at)
|
||||
snapshot_at = f"{sh:02d}:{sm:02d}"
|
||||
if summary.reason == _SEED_SKIP_NO_SOURCE:
|
||||
_record_task(
|
||||
repo,
|
||||
@@ -561,9 +726,12 @@ def cmd_snapshot_seed(args, logger: JsonLogger) -> int:
|
||||
finished_at=finished,
|
||||
payload={**payload, "skipped": True, "reason": _SEED_SKIP_NO_SOURCE},
|
||||
error=_SEED_SKIP_NO_SOURCE,
|
||||
next_due_at=_seed_retry_soon(float(finished)),
|
||||
stat_date=summary.stat_date or seed_date,
|
||||
)
|
||||
_print(payload)
|
||||
return 1
|
||||
next_due = next_daily_run(float(finished), seed_at)
|
||||
_record_task(
|
||||
repo,
|
||||
"snapshot_seed",
|
||||
@@ -571,7 +739,16 @@ def cmd_snapshot_seed(args, logger: JsonLogger) -> int:
|
||||
started_at=int(summary.started_at or started),
|
||||
finished_at=finished,
|
||||
payload=payload,
|
||||
next_due_at=next_due,
|
||||
stat_date=summary.stat_date or seed_date,
|
||||
)
|
||||
if summary.reason == "seeded":
|
||||
_enqueue_evening_seal(
|
||||
repo,
|
||||
stat_date=summary.stat_date or seed_date,
|
||||
snapshot_at=snapshot_at,
|
||||
now=float(finished),
|
||||
)
|
||||
_print(payload)
|
||||
return 0
|
||||
finally:
|
||||
@@ -609,24 +786,34 @@ def cmd_discover(args, logger: JsonLogger) -> int:
|
||||
logger=logger,
|
||||
)
|
||||
started = int(time.time())
|
||||
interval = float(cfg.schedule.discover_minutes * 60)
|
||||
_mark_task_running(repo, "discovery", started_at=started)
|
||||
try:
|
||||
summary = service.run()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
_record_task(
|
||||
repo, "discovery", ok=False, started_at=started,
|
||||
finished_at=int(time.time()), error=str(exc),
|
||||
finished_at=finished, error=str(exc),
|
||||
next_due_at=_retry_soon(float(finished)),
|
||||
)
|
||||
raise
|
||||
payload = summary.as_dict()
|
||||
payload["candidate_total"] = len(repo.list_candidates())
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
finished = int(summary.finished_at or time.time())
|
||||
ok = not summary.errors
|
||||
next_due = (
|
||||
interval_next_run(float(finished), interval) if ok
|
||||
else _retry_soon(float(finished))
|
||||
)
|
||||
_record_task(
|
||||
repo, "discovery", ok=not summary.errors,
|
||||
repo, "discovery", ok=ok,
|
||||
started_at=int(summary.started_at or started),
|
||||
finished_at=int(summary.finished_at or time.time()),
|
||||
finished_at=finished,
|
||||
payload=payload,
|
||||
error=";".join(summary.errors) if summary.errors else None,
|
||||
next_due_at=next_due,
|
||||
)
|
||||
_print(payload)
|
||||
return 0 if not summary.errors else 1
|
||||
@@ -747,29 +934,32 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
summary = discovery_service.run()
|
||||
except Exception as exc: # noqa: BLE001 记录后补跑,不抛给次日
|
||||
finished = int(time.time())
|
||||
next_run = _retry_soon(float(finished))
|
||||
_record_task(
|
||||
repo, "discovery", ok=False, started_at=started,
|
||||
finished_at=finished, error=str(exc),
|
||||
next_due_at=next_run,
|
||||
)
|
||||
return TaskControl(
|
||||
ok=False, next_run=_retry_soon(float(finished)), error=str(exc),
|
||||
ok=False, next_run=next_run, error=str(exc),
|
||||
)
|
||||
payload = summary.as_dict()
|
||||
ok = not summary.errors
|
||||
finished = int(summary.finished_at or time.time())
|
||||
next_run = (
|
||||
interval_next_run(float(finished), interval) if ok
|
||||
else _retry_soon(float(finished))
|
||||
)
|
||||
_record_task(
|
||||
repo, "discovery", ok=ok,
|
||||
started_at=summary.started_at,
|
||||
finished_at=finished,
|
||||
payload=payload,
|
||||
error=";".join(summary.errors) if summary.errors else None,
|
||||
next_due_at=next_run,
|
||||
)
|
||||
last["discover"] = summary
|
||||
logger.info("discovery.summary", **payload)
|
||||
next_run = (
|
||||
interval_next_run(float(finished), interval) if ok
|
||||
else _retry_soon(float(finished))
|
||||
)
|
||||
return TaskControl(ok=ok, next_run=next_run)
|
||||
|
||||
def run_cleanup():
|
||||
@@ -785,12 +975,14 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
expired = repo.expire_fav_actions()
|
||||
except Exception as exc: # noqa: BLE001 记录后约 5 分钟补跑
|
||||
finished = int(time.time())
|
||||
next_run = _retry_soon(float(finished))
|
||||
_record_task(
|
||||
repo, "candidate_cleanup", ok=False, started_at=started,
|
||||
finished_at=finished, error=str(exc),
|
||||
next_due_at=next_run,
|
||||
)
|
||||
return TaskControl(
|
||||
ok=False, next_run=_retry_soon(float(finished)), error=str(exc),
|
||||
ok=False, next_run=next_run, error=str(exc),
|
||||
)
|
||||
if removed or expired:
|
||||
repo.bump_cache_version("videos")
|
||||
@@ -799,6 +991,7 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
"cleanup.done",
|
||||
candidates_removed=removed, fav_actions_expired=expired,
|
||||
)
|
||||
next_run = interval_next_run(float(finished), 86400.0)
|
||||
_record_task(
|
||||
repo, "candidate_cleanup", ok=True, started_at=started,
|
||||
finished_at=finished,
|
||||
@@ -806,48 +999,90 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
"candidates_removed": removed,
|
||||
"fav_actions_expired": expired,
|
||||
},
|
||||
next_due_at=next_run,
|
||||
)
|
||||
return TaskControl(
|
||||
ok=True,
|
||||
next_run=interval_next_run(float(finished), 86400.0),
|
||||
)
|
||||
return TaskControl(ok=True, next_run=next_run)
|
||||
|
||||
def run_snapshot():
|
||||
"""全站快照:不因其它任务 running 而 skip;失败/未完成则短间隔补跑。"""
|
||||
started = int(time.time())
|
||||
_mark_task_running(repo, "snapshot", started_at=started)
|
||||
today = shanghai_date(float(started))
|
||||
stat_date = task_state_mod.resume_stat_date(
|
||||
repo, "snapshot", fallback=today,
|
||||
)
|
||||
_mark_task_running(
|
||||
repo, "snapshot", started_at=started, stat_date=stat_date,
|
||||
)
|
||||
try:
|
||||
# 仅晚间 snapshot_at 及之后:seed 壳才 force 打穿 ok=1 克隆行。
|
||||
# 白天/重启不得因 seeded 标记立刻全量强采。
|
||||
today = shanghai_date()
|
||||
force_seed = (
|
||||
repo.kv_get(SnapshotService.seeded_key(today)) == "1"
|
||||
and _reached_daily_slot(snapshot_at, float(started))
|
||||
# seed 壳 ≠ 白天强采:仅到达当日 snapshot_at(或显式续跑已到期的
|
||||
# seal pending)才 force。due_at 门闩负责「会不会跑」;这里是
|
||||
# force 的第二道闸。
|
||||
seeded = repo.kv_get(SnapshotService.seeded_key(stat_date)) == "1"
|
||||
force_seed = seeded and _reached_daily_slot(
|
||||
snapshot_at, float(started),
|
||||
)
|
||||
summary = snapshot_service.run_if_needed(
|
||||
date=stat_date, force=force_seed,
|
||||
)
|
||||
summary = snapshot_service.run_if_needed(force=force_seed)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
next_run = _retry_soon(float(finished))
|
||||
_record_task(
|
||||
repo, "snapshot", ok=False, started_at=started,
|
||||
finished_at=finished, error=str(exc),
|
||||
next_due_at=next_run, stat_date=stat_date,
|
||||
)
|
||||
return TaskControl(
|
||||
ok=False, next_run=_retry_soon(float(finished)), error=str(exc),
|
||||
ok=False, next_run=next_run, error=str(exc),
|
||||
)
|
||||
payload = summary.as_dict()
|
||||
ok = bool(summary.completed and not summary.backup_error)
|
||||
finished = int(summary.finished_at or time.time())
|
||||
_record_task(
|
||||
repo, "snapshot", ok=ok, started_at=summary.started_at,
|
||||
finished_at=finished,
|
||||
payload=payload, error=summary.backup_error,
|
||||
)
|
||||
last["snapshot"] = summary
|
||||
logger.info("snapshot.summary", **payload)
|
||||
# 白天误碰到 seed 壳:延到当晚 seal,禁止 5 分钟空转。
|
||||
if (
|
||||
not ok
|
||||
and getattr(summary, "reason", None) == "seeded_pending_seal"
|
||||
):
|
||||
current = shanghai_datetime(float(finished))
|
||||
hour, minute = parse_hhmm(snapshot_at)
|
||||
slot_ts = current.replace(
|
||||
hour=hour, minute=minute, second=0, microsecond=0,
|
||||
).timestamp()
|
||||
next_run = slot_ts if slot_ts > float(finished) else _retry_soon(
|
||||
float(finished),
|
||||
)
|
||||
_record_task(
|
||||
repo, "snapshot", ok=False, skipped=True,
|
||||
started_at=summary.started_at, finished_at=finished,
|
||||
payload=payload, error="seeded_pending_seal",
|
||||
next_due_at=next_run,
|
||||
stat_date=summary.stat_date or stat_date,
|
||||
)
|
||||
last["snapshot"] = summary
|
||||
logger.info("snapshot.summary", **payload)
|
||||
return TaskControl(
|
||||
ok=False, next_run=next_run, error="seeded_pending_seal",
|
||||
)
|
||||
next_run = (
|
||||
next_daily_run(float(finished), snapshot_at) if ok
|
||||
else _retry_soon(float(finished))
|
||||
)
|
||||
progress = {
|
||||
"total": summary.total,
|
||||
"success": summary.success,
|
||||
"failed": summary.failed,
|
||||
"skipped": summary.skipped,
|
||||
}
|
||||
_record_task(
|
||||
repo, "snapshot", ok=ok, started_at=summary.started_at,
|
||||
finished_at=finished,
|
||||
payload=payload, error=summary.backup_error,
|
||||
next_due_at=next_run,
|
||||
stat_date=summary.stat_date or stat_date,
|
||||
progress=progress,
|
||||
)
|
||||
last["snapshot"] = summary
|
||||
logger.info("snapshot.summary", **payload)
|
||||
return TaskControl(ok=ok, next_run=next_run)
|
||||
|
||||
def run_snapshot_recent():
|
||||
@@ -862,71 +1097,109 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
summary = snapshot_service.run_recent_videos()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
next_run = _retry_soon(float(finished))
|
||||
_record_task(
|
||||
repo, "snapshot_recent", ok=False, started_at=started,
|
||||
finished_at=finished, error=str(exc),
|
||||
next_due_at=next_run,
|
||||
)
|
||||
return TaskControl(
|
||||
ok=False, next_run=_retry_soon(float(finished)), error=str(exc),
|
||||
ok=False, next_run=next_run, error=str(exc),
|
||||
)
|
||||
payload = summary.as_dict()
|
||||
ok = summary.failed == 0
|
||||
error = ";".join(summary.errors) if summary.errors else None
|
||||
finished = int(summary.finished_at or time.time())
|
||||
_record_task(
|
||||
repo, "snapshot_recent", ok=ok, started_at=summary.started_at,
|
||||
finished_at=finished,
|
||||
payload=payload, error=error,
|
||||
)
|
||||
last["snapshot_recent"] = summary
|
||||
logger.info("snapshot.recent_summary", **payload)
|
||||
next_run = (
|
||||
interval_next_run(float(finished), interval) if ok
|
||||
else _retry_soon(float(finished))
|
||||
)
|
||||
_record_task(
|
||||
repo, "snapshot_recent", ok=ok, started_at=summary.started_at,
|
||||
finished_at=finished,
|
||||
payload=payload, error=error,
|
||||
next_due_at=next_run,
|
||||
)
|
||||
last["snapshot_recent"] = summary
|
||||
logger.info("snapshot.recent_summary", **payload)
|
||||
return TaskControl(ok=ok, next_run=next_run)
|
||||
|
||||
def run_fans():
|
||||
"""粉丝更新短耗时:不避开任何任务,可与全量并行。"""
|
||||
started = int(time.time())
|
||||
_mark_task_running(repo, "fans", started_at=started)
|
||||
today = shanghai_date(float(started))
|
||||
stat_date = task_state_mod.resume_stat_date(
|
||||
repo, "fans", fallback=today,
|
||||
)
|
||||
_mark_task_running(
|
||||
repo, "fans", started_at=started, stat_date=stat_date,
|
||||
)
|
||||
try:
|
||||
today = shanghai_date()
|
||||
force_seed = (
|
||||
repo.kv_get(SnapshotService.fans_seeded_key(today)) == "1"
|
||||
and _reached_daily_slot(snapshot_at, float(started))
|
||||
seeded = (
|
||||
repo.kv_get(SnapshotService.fans_seeded_key(stat_date)) == "1"
|
||||
)
|
||||
force_seed = seeded and _reached_daily_slot(
|
||||
snapshot_at, float(started),
|
||||
)
|
||||
summary = snapshot_service.run_fans_if_needed(
|
||||
date=stat_date, force=force_seed,
|
||||
)
|
||||
summary = snapshot_service.run_fans_if_needed(force=force_seed)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
next_run = _retry_soon(float(finished))
|
||||
_record_task(
|
||||
repo, "fans", ok=False, started_at=started,
|
||||
finished_at=finished, error=str(exc),
|
||||
next_due_at=next_run, stat_date=stat_date,
|
||||
)
|
||||
return TaskControl(
|
||||
ok=False, next_run=_retry_soon(float(finished)), error=str(exc),
|
||||
ok=False, next_run=next_run, error=str(exc),
|
||||
)
|
||||
payload = summary.as_dict()
|
||||
finished = int(summary.finished_at or time.time())
|
||||
if getattr(summary, "reason", None) == "seeded_pending_seal":
|
||||
current = shanghai_datetime(float(finished))
|
||||
hour, minute = parse_hhmm(snapshot_at)
|
||||
slot_ts = current.replace(
|
||||
hour=hour, minute=minute, second=0, microsecond=0,
|
||||
).timestamp()
|
||||
next_run = slot_ts if slot_ts > float(finished) else _retry_soon(
|
||||
float(finished),
|
||||
)
|
||||
_record_task(
|
||||
repo, "fans", ok=False, skipped=True,
|
||||
started_at=summary.started_at, finished_at=finished,
|
||||
payload=payload, error="seeded_pending_seal",
|
||||
next_due_at=next_run,
|
||||
stat_date=summary.stat_date or stat_date,
|
||||
)
|
||||
last["fans"] = summary
|
||||
logger.info("fans.summary", **payload)
|
||||
return TaskControl(
|
||||
ok=False, next_run=next_run, error="seeded_pending_seal",
|
||||
)
|
||||
ok = summary.fans_failed == 0
|
||||
error = ";".join(
|
||||
err for err in summary.errors if str(err).startswith("fans ")
|
||||
) or None
|
||||
finished = int(summary.finished_at or time.time())
|
||||
_record_task(
|
||||
repo, "fans", ok=ok, started_at=summary.started_at,
|
||||
finished_at=finished,
|
||||
payload=payload, error=error,
|
||||
)
|
||||
last["fans"] = summary
|
||||
logger.info("fans.summary", **payload)
|
||||
next_run = (
|
||||
next_daily_run(float(finished), snapshot_at) if ok
|
||||
else _retry_soon(float(finished))
|
||||
)
|
||||
_record_task(
|
||||
repo, "fans", ok=ok, started_at=summary.started_at,
|
||||
finished_at=finished,
|
||||
payload=payload, error=error,
|
||||
next_due_at=next_run,
|
||||
stat_date=summary.stat_date or stat_date,
|
||||
)
|
||||
last["fans"] = summary
|
||||
logger.info("fans.summary", **payload)
|
||||
return TaskControl(ok=ok, next_run=next_run)
|
||||
|
||||
def run_snapshot_seed():
|
||||
started = int(time.time())
|
||||
today = shanghai_date(float(started))
|
||||
# 全站快照占用时不执行、不记成功;10 分钟后重试直到真正 seed 成功。
|
||||
if _is_task_running(repo, "snapshot"):
|
||||
return _defer_snapshot_seed(
|
||||
@@ -935,18 +1208,22 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
started_at=started,
|
||||
reason=_SEED_SKIP_SNAPSHOT_RUNNING,
|
||||
)
|
||||
_mark_task_running(repo, "snapshot_seed", started_at=started)
|
||||
_mark_task_running(
|
||||
repo, "snapshot_seed", started_at=started, stat_date=today,
|
||||
)
|
||||
try:
|
||||
summary = snapshot_service.run_seed_from_last_completed()
|
||||
summary = snapshot_service.run_seed_from_last_completed(date=today)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
finished = int(time.time())
|
||||
next_run = _seed_retry_soon(float(finished))
|
||||
_record_task(
|
||||
repo, "snapshot_seed", ok=False, started_at=started,
|
||||
finished_at=finished, error=str(exc),
|
||||
next_due_at=next_run, stat_date=today,
|
||||
)
|
||||
return TaskControl(
|
||||
ok=False,
|
||||
next_run=_seed_retry_soon(float(finished)),
|
||||
next_run=next_run,
|
||||
error=str(exc),
|
||||
)
|
||||
payload = summary.as_dict()
|
||||
@@ -962,14 +1239,22 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
finished_at=finished,
|
||||
)
|
||||
# already_completed / already_seeded / seeded → 成功,推到次日 seed_at
|
||||
next_run = next_daily_run(float(finished), seed_at)
|
||||
_record_task(
|
||||
repo, "snapshot_seed", ok=True, started_at=summary.started_at,
|
||||
finished_at=finished,
|
||||
payload=payload,
|
||||
next_due_at=next_run,
|
||||
stat_date=summary.stat_date or today,
|
||||
)
|
||||
return TaskControl(
|
||||
ok=True, next_run=next_daily_run(float(finished), seed_at),
|
||||
)
|
||||
if summary.reason == "seeded":
|
||||
_enqueue_evening_seal(
|
||||
repo,
|
||||
stat_date=summary.stat_date or today,
|
||||
snapshot_at=snapshot_at,
|
||||
now=float(finished),
|
||||
)
|
||||
return TaskControl(ok=True, next_run=next_run)
|
||||
|
||||
def run_config_watch():
|
||||
before = store.revision
|
||||
@@ -978,10 +1263,11 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
logger.info("config.reloaded", revision=store.revision)
|
||||
return {"revision": store.revision}
|
||||
|
||||
# --once:保持 next_run=0 立刻跑到期逻辑;常驻:日更对齐墙钟,避免重启即强采。
|
||||
# --once:next_run=0 立刻跑;常驻:一律从 task:state.due_at 恢复,禁止日更 next_run=0。
|
||||
now_ts = time.time()
|
||||
if args.once:
|
||||
seed_next = snapshot_next = fans_next = 0.0
|
||||
recent_next = discover_next = cleanup_next = 0.0
|
||||
else:
|
||||
seed_next = _initial_daily_next_run(
|
||||
repo, "snapshot_seed", seed_at, now=now_ts,
|
||||
@@ -992,6 +1278,15 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
fans_next = _initial_daily_next_run(
|
||||
repo, "fans", snapshot_at, now=now_ts,
|
||||
)
|
||||
recent_next = task_state_mod.startup_next_run(
|
||||
repo, "snapshot_recent", wall_due_at=now_ts, now=now_ts,
|
||||
)
|
||||
discover_next = task_state_mod.startup_next_run(
|
||||
repo, "discovery", wall_due_at=now_ts, now=now_ts,
|
||||
)
|
||||
cleanup_next = task_state_mod.startup_next_run(
|
||||
repo, "candidate_cleanup", wall_due_at=now_ts, now=now_ts,
|
||||
)
|
||||
scheduler = Scheduler(
|
||||
[
|
||||
ScheduledTask(
|
||||
@@ -1016,13 +1311,20 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
"snapshot_recent",
|
||||
float(recent_minutes * 60),
|
||||
run_snapshot_recent,
|
||||
next_run=recent_next,
|
||||
),
|
||||
ScheduledTask(
|
||||
"discovery",
|
||||
float(discover_minutes * 60),
|
||||
run_discover,
|
||||
next_run=discover_next,
|
||||
),
|
||||
ScheduledTask(
|
||||
"candidate_cleanup",
|
||||
86400.0,
|
||||
run_cleanup,
|
||||
next_run=cleanup_next,
|
||||
),
|
||||
ScheduledTask("candidate_cleanup", 86400.0, run_cleanup),
|
||||
ScheduledTask("config_watch", 60.0, run_config_watch),
|
||||
],
|
||||
logger=logger,
|
||||
|
||||
Reference in new issue
Block a user