feat(scheduler): add snapshot-seed CLI and 10m seed deferral
Add `python app.py snapshot-seed [--date]` (offline clone, Debian / docker compose exec friendly). When seed fires while full-site snapshot is running, or last_completed is missing (no_source), record skip as non-success and retry in 10 minutes until a real seed succeeds; fans and quiet-window rules unchanged.
This commit is contained in:
1 parent
9969e65202
commit
bb18d232f6
7 files changed
+374
-8
No files matched your search
+157
-3
@@ -4,6 +4,7 @@
|
||||
python app.py init # 建库 / 迁移 / 写入黑名单种子(离线)
|
||||
python app.py ingest # 手动入库 B 站视频链接 / BV 号 / av 号(只写本地库)
|
||||
python app.py snapshot # 采集当日七项快照与粉丝数
|
||||
python app.py snapshot-seed # 用上一完成日克隆当日壳(离线,可 --date)
|
||||
python app.py discover # 执行一次搜索发现并写入候选池
|
||||
python app.py candidates # 查看候选池(离线)
|
||||
python app.py auth-check # 检查 Cookie 登录态
|
||||
@@ -42,6 +43,7 @@ from .services.up_profile import UpProfileService
|
||||
from .timeutil import (
|
||||
in_snapshot_quiet_window,
|
||||
interval_next_run,
|
||||
is_date,
|
||||
next_after_snapshot_quiet_window,
|
||||
next_daily_run,
|
||||
parse_hhmm,
|
||||
@@ -78,6 +80,17 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
snapshot.add_argument(
|
||||
"--force", action="store_true", help="忽略当日完成标记,重新采集"
|
||||
)
|
||||
snapshot_seed = sub.add_parser(
|
||||
"snapshot-seed",
|
||||
parents=[common],
|
||||
help="用上一完成日克隆当日快照壳(离线,不调 B 站)",
|
||||
)
|
||||
snapshot_seed.add_argument(
|
||||
"--date",
|
||||
default=None,
|
||||
metavar="YYYY-MM-DD",
|
||||
help="目标统计日(默认上海时区今天)",
|
||||
)
|
||||
discover = sub.add_parser(
|
||||
"discover", parents=[common], help="执行一次搜索发现并写入候选池"
|
||||
)
|
||||
@@ -170,6 +183,14 @@ def _clear_task_running(repo: Repository, name: str) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _is_task_running(repo: Repository, name: str) -> bool:
|
||||
"""供 seed 等任务检测全站快照是否在跑(避让,而非互斥跳过成功)。"""
|
||||
try:
|
||||
return bool(repo.kv_get(_task_running_key(name)))
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
|
||||
def _record_task(
|
||||
repo: Repository,
|
||||
name: str,
|
||||
@@ -206,7 +227,11 @@ def _record_task(
|
||||
|
||||
# 全站快照 / 粉丝失败后的补跑间隔(秒)
|
||||
_TASK_RETRY_SECONDS = 300.0
|
||||
# snapshot_seed:全站快照占用 / 尚无 last_completed 时的补跑间隔(秒)
|
||||
_SEED_RETRY_SECONDS = 600.0
|
||||
_QUIET_SKIP_REASON = "snapshot_quiet_window"
|
||||
_SEED_SKIP_SNAPSHOT_RUNNING = "snapshot_running"
|
||||
_SEED_SKIP_NO_SOURCE = "no_source"
|
||||
|
||||
|
||||
def _quiet_window_defer(
|
||||
@@ -240,6 +265,38 @@ def _retry_soon(now: float | None = None) -> float:
|
||||
return (time.time() if now is None else now) + _TASK_RETRY_SECONDS
|
||||
|
||||
|
||||
def _seed_retry_soon(now: float | None = None) -> float:
|
||||
return (time.time() if now is None else now) + _SEED_RETRY_SECONDS
|
||||
|
||||
|
||||
def _defer_snapshot_seed(
|
||||
repo: Repository,
|
||||
logger: JsonLogger,
|
||||
*,
|
||||
started_at: int,
|
||||
reason: str,
|
||||
finished_at: int | None = None,
|
||||
) -> TaskControl:
|
||||
"""seed 因 snapshot 占用 / no_source 延后:记 skip(非成功),10 分钟后再试。"""
|
||||
finished = int(finished_at if finished_at is not None else time.time())
|
||||
logger.info("snapshot_seed.skip", reason=reason)
|
||||
_record_task(
|
||||
repo,
|
||||
"snapshot_seed",
|
||||
ok=False,
|
||||
skipped=True,
|
||||
started_at=started_at,
|
||||
finished_at=finished,
|
||||
payload={"skipped": True, "reason": reason},
|
||||
error=reason,
|
||||
)
|
||||
return TaskControl(
|
||||
ok=False,
|
||||
next_run=_seed_retry_soon(float(finished)),
|
||||
error=reason,
|
||||
)
|
||||
|
||||
|
||||
def _client(credentials, logger: JsonLogger, *, pacer=None, retries: int = 3) -> BilibiliClient:
|
||||
return BilibiliClient(
|
||||
cookie=credentials.cookie,
|
||||
@@ -397,6 +454,82 @@ def cmd_snapshot(args, logger: JsonLogger) -> int:
|
||||
db.close()
|
||||
|
||||
|
||||
def cmd_snapshot_seed(args, logger: JsonLogger) -> int:
|
||||
"""离线克隆:``python app.py snapshot-seed [--date YYYY-MM-DD]``。
|
||||
|
||||
与调度器同一套规则:全站 ``snapshot`` 运行中或 ``no_source`` 时不记成功,
|
||||
返回码非 0,便于 ``docker compose exec re-xxm python app.py snapshot-seed`` 排查。
|
||||
"""
|
||||
store = ConfigStore(args.config, require_auth=False)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
target = args.date
|
||||
if target is not None and not is_date(target):
|
||||
raise ConfigError(f"日期格式必须是 YYYY-MM-DD: {target!r}")
|
||||
started = int(time.time())
|
||||
if _is_task_running(repo, "snapshot"):
|
||||
payload = {
|
||||
"ok": False,
|
||||
"skipped": True,
|
||||
"reason": _SEED_SKIP_SNAPSHOT_RUNNING,
|
||||
"stat_date": target or shanghai_date(),
|
||||
"database": str(cfg.storage.database_path),
|
||||
}
|
||||
_record_task(
|
||||
repo,
|
||||
"snapshot_seed",
|
||||
ok=False,
|
||||
skipped=True,
|
||||
started_at=started,
|
||||
finished_at=int(time.time()),
|
||||
payload=payload,
|
||||
error=_SEED_SKIP_SNAPSHOT_RUNNING,
|
||||
)
|
||||
_print(payload)
|
||||
return 1
|
||||
# seed 只读本地库,不调 B 站;占位 client 满足构造约束。
|
||||
service = SnapshotService(store, repo, object(), logger=logger)
|
||||
_mark_task_running(repo, "snapshot_seed", started_at=started)
|
||||
try:
|
||||
summary = service.run_seed_from_last_completed(date=target)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_record_task(
|
||||
repo, "snapshot_seed", ok=False, started_at=started,
|
||||
finished_at=int(time.time()), error=str(exc),
|
||||
)
|
||||
raise
|
||||
payload = summary.as_dict()
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
finished = int(summary.finished_at or time.time())
|
||||
if summary.reason == _SEED_SKIP_NO_SOURCE:
|
||||
_record_task(
|
||||
repo,
|
||||
"snapshot_seed",
|
||||
ok=False,
|
||||
skipped=True,
|
||||
started_at=int(summary.started_at or started),
|
||||
finished_at=finished,
|
||||
payload={**payload, "skipped": True, "reason": _SEED_SKIP_NO_SOURCE},
|
||||
error=_SEED_SKIP_NO_SOURCE,
|
||||
)
|
||||
_print(payload)
|
||||
return 1
|
||||
_record_task(
|
||||
repo,
|
||||
"snapshot_seed",
|
||||
ok=True,
|
||||
started_at=int(summary.started_at or started),
|
||||
finished_at=finished,
|
||||
payload=payload,
|
||||
)
|
||||
_print(payload)
|
||||
return 0
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _telegram(cfg, logger: JsonLogger):
|
||||
if not cfg.telegram.enabled:
|
||||
return None
|
||||
@@ -733,6 +866,14 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
|
||||
def run_snapshot_seed():
|
||||
started = int(time.time())
|
||||
# 全站快照占用时不执行、不记成功;10 分钟后重试直到真正 seed 成功。
|
||||
if _is_task_running(repo, "snapshot"):
|
||||
return _defer_snapshot_seed(
|
||||
repo,
|
||||
logger,
|
||||
started_at=started,
|
||||
reason=_SEED_SKIP_SNAPSHOT_RUNNING,
|
||||
)
|
||||
_mark_task_running(repo, "snapshot_seed", started_at=started)
|
||||
try:
|
||||
summary = snapshot_service.run_seed_from_last_completed()
|
||||
@@ -743,17 +884,28 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
|
||||
finished_at=finished, error=str(exc),
|
||||
)
|
||||
return TaskControl(
|
||||
ok=False, next_run=_retry_soon(float(finished)), error=str(exc),
|
||||
ok=False,
|
||||
next_run=_seed_retry_soon(float(finished)),
|
||||
error=str(exc),
|
||||
)
|
||||
payload = summary.as_dict()
|
||||
finished = int(summary.finished_at or time.time())
|
||||
last["snapshot_seed"] = summary
|
||||
logger.info("snapshot.seed_summary", **payload)
|
||||
if summary.reason == _SEED_SKIP_NO_SOURCE:
|
||||
return _defer_snapshot_seed(
|
||||
repo,
|
||||
logger,
|
||||
started_at=int(summary.started_at or started),
|
||||
reason=_SEED_SKIP_NO_SOURCE,
|
||||
finished_at=finished,
|
||||
)
|
||||
# already_completed / already_seeded / seeded → 成功,推到次日 seed_at
|
||||
_record_task(
|
||||
repo, "snapshot_seed", ok=True, started_at=summary.started_at,
|
||||
finished_at=finished,
|
||||
payload=payload,
|
||||
)
|
||||
last["snapshot_seed"] = summary
|
||||
logger.info("snapshot.seed_summary", **payload)
|
||||
return TaskControl(
|
||||
ok=True, next_run=next_daily_run(float(finished), seed_at),
|
||||
)
|
||||
@@ -942,6 +1094,8 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
return cmd_init(args, logger)
|
||||
if args.command == "snapshot":
|
||||
return cmd_snapshot(args, logger)
|
||||
if args.command == "snapshot-seed":
|
||||
return cmd_snapshot_seed(args, logger)
|
||||
if args.command == "ingest":
|
||||
return cmd_ingest(args, logger)
|
||||
if args.command == "backfill-followers":
|
||||
|
||||
@@ -51,6 +51,7 @@ class SnapshotSummary:
|
||||
completed: bool = False
|
||||
already_completed: bool = False
|
||||
finished_at: int = 0
|
||||
reason: str | None = None
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
@@ -73,6 +74,7 @@ class SnapshotSummary:
|
||||
"backup_error": self.backup_error,
|
||||
"completed": self.completed,
|
||||
"already_completed": self.already_completed,
|
||||
"reason": self.reason,
|
||||
"errors": self.errors,
|
||||
}
|
||||
|
||||
@@ -543,7 +545,8 @@ class SnapshotService:
|
||||
stat_date=stat_date, reason="no_source", source=source,
|
||||
)
|
||||
summary.finished_at = self._now()
|
||||
summary.completed = True
|
||||
summary.completed = False
|
||||
summary.reason = "no_source"
|
||||
return summary
|
||||
if self.repo.kv_get(self.completed_key(stat_date)) == "1":
|
||||
self._log(
|
||||
@@ -553,6 +556,7 @@ class SnapshotService:
|
||||
summary.finished_at = self._now()
|
||||
summary.completed = True
|
||||
summary.already_completed = True
|
||||
summary.reason = "already_completed"
|
||||
return summary
|
||||
if self.repo.kv_get(self.seeded_key(stat_date)) == "1":
|
||||
self._log(
|
||||
@@ -562,6 +566,7 @@ class SnapshotService:
|
||||
summary.finished_at = self._now()
|
||||
summary.completed = True
|
||||
summary.already_completed = True
|
||||
summary.reason = "already_seeded"
|
||||
return summary
|
||||
|
||||
copied = self.repo.copy_ok_snapshots(
|
||||
@@ -583,6 +588,7 @@ class SnapshotService:
|
||||
self.repo.bump_cache_version("fans")
|
||||
summary.finished_at = self._now()
|
||||
summary.completed = True # seed 任务本身成功;不等于快照落盘完成
|
||||
summary.reason = "seeded"
|
||||
self._log(
|
||||
"info", "snapshot.seed_done",
|
||||
stat_date=stat_date, source=source,
|
||||
|
||||
@@ -1982,6 +1982,7 @@ class DashboardQueries:
|
||||
"snapshot_recent": int(interval_next_run(now, float(recent_interval))),
|
||||
}
|
||||
retry_seconds = 300
|
||||
seed_retry_seconds = 600
|
||||
for name, key, at in (
|
||||
("fans", "fans", cfg.schedule.snapshot_at),
|
||||
("snapshot", "snapshot", cfg.schedule.snapshot_at),
|
||||
@@ -2015,7 +2016,12 @@ class DashboardQueries:
|
||||
else:
|
||||
if skipped or failed:
|
||||
# 日更任务失败/skip:短间隔补跑,绝不按成功推到次日。
|
||||
next_runs[name] = int(finished_at + retry_seconds)
|
||||
# snapshot_seed:全站快照占用 / no_source 时 10 分钟补跑。
|
||||
gap = (
|
||||
seed_retry_seconds if name == "snapshot_seed"
|
||||
else retry_seconds
|
||||
)
|
||||
next_runs[name] = int(finished_at + gap)
|
||||
else:
|
||||
next_runs[name] = int(next_daily_run(finished_at, at))
|
||||
errors = [
|
||||
|
||||
Reference in new issue
Block a user