feat(scheduler): add snapshot-seed CLI and 10m seed deferral

Add `python app.py snapshot-seed [--date]` (offline clone, Debian /
docker compose exec friendly). When seed fires while full-site snapshot
is running, or last_completed is missing (no_source), record skip as
non-success and retry in 10 minutes until a real seed succeeds; fans
and quiet-window rules unchanged.
This commit is contained in:
bot-ci committed 2026-11-03 00:22:11 +08:00
1 parent 9969e65202
commit bb18d232f6
7 files changed
+374 -8

No files matched your search

+157 -3
View File
@@ -4,6 +4,7 @@
python app.py init # 建库 / 迁移 / 写入黑名单种子(离线)
python app.py ingest # 手动入库 B 站视频链接 / BV 号 / av 号(只写本地库)
python app.py snapshot # 采集当日七项快照与粉丝数
python app.py snapshot-seed # 用上一完成日克隆当日壳(离线,可 --date)
python app.py discover # 执行一次搜索发现并写入候选池
python app.py candidates # 查看候选池(离线)
python app.py auth-check # 检查 Cookie 登录态
@@ -42,6 +43,7 @@ from .services.up_profile import UpProfileService
from .timeutil import (
in_snapshot_quiet_window,
interval_next_run,
is_date,
next_after_snapshot_quiet_window,
next_daily_run,
parse_hhmm,
@@ -78,6 +80,17 @@ def _build_parser() -> argparse.ArgumentParser:
snapshot.add_argument(
"--force", action="store_true", help="忽略当日完成标记,重新采集"
)
snapshot_seed = sub.add_parser(
"snapshot-seed",
parents=[common],
help="用上一完成日克隆当日快照壳(离线,不调 B 站)",
)
snapshot_seed.add_argument(
"--date",
default=None,
metavar="YYYY-MM-DD",
help="目标统计日(默认上海时区今天)",
)
discover = sub.add_parser(
"discover", parents=[common], help="执行一次搜索发现并写入候选池"
)
@@ -170,6 +183,14 @@ def _clear_task_running(repo: Repository, name: str) -> None:
pass
def _is_task_running(repo: Repository, name: str) -> bool:
"""供 seed 等任务检测全站快照是否在跑(避让,而非互斥跳过成功)。"""
try:
return bool(repo.kv_get(_task_running_key(name)))
except Exception: # noqa: BLE001
return False
def _record_task(
repo: Repository,
name: str,
@@ -206,7 +227,11 @@ def _record_task(
# 全站快照 / 粉丝失败后的补跑间隔(秒)
_TASK_RETRY_SECONDS = 300.0
# snapshot_seed:全站快照占用 / 尚无 last_completed 时的补跑间隔(秒)
_SEED_RETRY_SECONDS = 600.0
_QUIET_SKIP_REASON = "snapshot_quiet_window"
_SEED_SKIP_SNAPSHOT_RUNNING = "snapshot_running"
_SEED_SKIP_NO_SOURCE = "no_source"
def _quiet_window_defer(
@@ -240,6 +265,38 @@ def _retry_soon(now: float | None = None) -> float:
return (time.time() if now is None else now) + _TASK_RETRY_SECONDS
def _seed_retry_soon(now: float | None = None) -> float:
return (time.time() if now is None else now) + _SEED_RETRY_SECONDS
def _defer_snapshot_seed(
repo: Repository,
logger: JsonLogger,
*,
started_at: int,
reason: str,
finished_at: int | None = None,
) -> TaskControl:
"""seed 因 snapshot 占用 / no_source 延后:记 skip(非成功),10 分钟后再试。"""
finished = int(finished_at if finished_at is not None else time.time())
logger.info("snapshot_seed.skip", reason=reason)
_record_task(
repo,
"snapshot_seed",
ok=False,
skipped=True,
started_at=started_at,
finished_at=finished,
payload={"skipped": True, "reason": reason},
error=reason,
)
return TaskControl(
ok=False,
next_run=_seed_retry_soon(float(finished)),
error=reason,
)
def _client(credentials, logger: JsonLogger, *, pacer=None, retries: int = 3) -> BilibiliClient:
return BilibiliClient(
cookie=credentials.cookie,
@@ -397,6 +454,82 @@ def cmd_snapshot(args, logger: JsonLogger) -> int:
db.close()
def cmd_snapshot_seed(args, logger: JsonLogger) -> int:
"""离线克隆:``python app.py snapshot-seed [--date YYYY-MM-DD]``。
与调度器同一套规则:全站 ``snapshot`` 运行中或 ``no_source`` 时不记成功,
返回码非 0,便于 ``docker compose exec re-xxm python app.py snapshot-seed`` 排查。
"""
store = ConfigStore(args.config, require_auth=False)
cfg = store.get()
db = _open_db(cfg, logger)
try:
repo = Repository(db)
target = args.date
if target is not None and not is_date(target):
raise ConfigError(f"日期格式必须是 YYYY-MM-DD: {target!r}")
started = int(time.time())
if _is_task_running(repo, "snapshot"):
payload = {
"ok": False,
"skipped": True,
"reason": _SEED_SKIP_SNAPSHOT_RUNNING,
"stat_date": target or shanghai_date(),
"database": str(cfg.storage.database_path),
}
_record_task(
repo,
"snapshot_seed",
ok=False,
skipped=True,
started_at=started,
finished_at=int(time.time()),
payload=payload,
error=_SEED_SKIP_SNAPSHOT_RUNNING,
)
_print(payload)
return 1
# seed 只读本地库,不调 B 站;占位 client 满足构造约束。
service = SnapshotService(store, repo, object(), logger=logger)
_mark_task_running(repo, "snapshot_seed", started_at=started)
try:
summary = service.run_seed_from_last_completed(date=target)
except Exception as exc: # noqa: BLE001
_record_task(
repo, "snapshot_seed", ok=False, started_at=started,
finished_at=int(time.time()), error=str(exc),
)
raise
payload = summary.as_dict()
payload["database"] = str(cfg.storage.database_path)
finished = int(summary.finished_at or time.time())
if summary.reason == _SEED_SKIP_NO_SOURCE:
_record_task(
repo,
"snapshot_seed",
ok=False,
skipped=True,
started_at=int(summary.started_at or started),
finished_at=finished,
payload={**payload, "skipped": True, "reason": _SEED_SKIP_NO_SOURCE},
error=_SEED_SKIP_NO_SOURCE,
)
_print(payload)
return 1
_record_task(
repo,
"snapshot_seed",
ok=True,
started_at=int(summary.started_at or started),
finished_at=finished,
payload=payload,
)
_print(payload)
return 0
finally:
db.close()
def _telegram(cfg, logger: JsonLogger):
if not cfg.telegram.enabled:
return None
@@ -733,6 +866,14 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
def run_snapshot_seed():
started = int(time.time())
# 全站快照占用时不执行、不记成功;10 分钟后重试直到真正 seed 成功。
if _is_task_running(repo, "snapshot"):
return _defer_snapshot_seed(
repo,
logger,
started_at=started,
reason=_SEED_SKIP_SNAPSHOT_RUNNING,
)
_mark_task_running(repo, "snapshot_seed", started_at=started)
try:
summary = snapshot_service.run_seed_from_last_completed()
@@ -743,17 +884,28 @@ def cmd_run(args, logger: JsonLogger, *, serve_web: bool = False) -> int:
finished_at=finished, error=str(exc),
)
return TaskControl(
ok=False, next_run=_retry_soon(float(finished)), error=str(exc),
ok=False,
next_run=_seed_retry_soon(float(finished)),
error=str(exc),
)
payload = summary.as_dict()
finished = int(summary.finished_at or time.time())
last["snapshot_seed"] = summary
logger.info("snapshot.seed_summary", **payload)
if summary.reason == _SEED_SKIP_NO_SOURCE:
return _defer_snapshot_seed(
repo,
logger,
started_at=int(summary.started_at or started),
reason=_SEED_SKIP_NO_SOURCE,
finished_at=finished,
)
# already_completed / already_seeded / seeded → 成功,推到次日 seed_at
_record_task(
repo, "snapshot_seed", ok=True, started_at=summary.started_at,
finished_at=finished,
payload=payload,
)
last["snapshot_seed"] = summary
logger.info("snapshot.seed_summary", **payload)
return TaskControl(
ok=True, next_run=next_daily_run(float(finished), seed_at),
)
@@ -942,6 +1094,8 @@ def main(argv: Sequence[str] | None = None) -> int:
return cmd_init(args, logger)
if args.command == "snapshot":
return cmd_snapshot(args, logger)
if args.command == "snapshot-seed":
return cmd_snapshot_seed(args, logger)
if args.command == "ingest":
return cmd_ingest(args, logger)
if args.command == "backfill-followers":