fix(system): three next-runs, completed-date data_date, running mark
Split /system schedule into fans / snapshot+UP / discovery next-runs. Drive data_date and snapshot_days from fully completed video snapshots only, keep prior success date on failure, show task failure records, and mark in-progress tasks as 运行中 without wall-clock next-run jumps.
This commit is contained in:
bot-ci committed
1 parent
0b92e10317
commit
af90ce8f87
9 files changed
+411
-52
No files matched your search
@@ -7,8 +7,8 @@
|
||||
* 成功写入七项指标；``ok=1`` 的同日快照直接跳过（断点续跑）；
|
||||
* 视频不可访问时标记 ``removed``；
|
||||
* 其他错误最多重试 3 次，仍失败写 ``ok=0`` 占位，不写假零值；
|
||||
* 对 ``track_fans`` 逐个请求 ``relation/stat``；
|
||||
* 完成后写入当日完成标记并生成日备份。
|
||||
* 对 ``track_fans`` 逐个请求 ``relation/stat``（独立任务，不影响视频完成日）；
|
||||
* 视频采集全部成功后写入当日完成标记并生成日备份。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -131,12 +131,13 @@ class SnapshotService:
|
||||
def run_if_needed(
|
||||
self, *, date: str | None = None, force: bool = False
|
||||
) -> SnapshotSummary:
|
||||
"""视频快照（含 UP 头像补齐）；粉丝采集请用 ``run_fans_if_needed``。"""
|
||||
now = self._now()
|
||||
stat_date = date or shanghai_date(now)
|
||||
if not is_date(stat_date):
|
||||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||||
if force:
|
||||
return self.run(date=stat_date, force=True)
|
||||
return self.run_videos(date=stat_date, force=True)
|
||||
|
||||
completed = self.repo.kv_get(self.completed_key(stat_date)) == "1"
|
||||
pending = self.repo.pending_snapshot_aids(stat_date)
|
||||
@@ -151,7 +152,7 @@ class SnapshotService:
|
||||
completed=True,
|
||||
already_completed=True,
|
||||
)
|
||||
return self.run(date=stat_date, force=False)
|
||||
return self.run_videos(date=stat_date, force=False)
|
||||
|
||||
def backup_only(self, stat_date: str) -> SnapshotSummary:
|
||||
"""采集已完成但备份缺失时，只重试备份，不请求任何外部接口。"""
|
||||
@@ -170,6 +171,12 @@ class SnapshotService:
|
||||
return summary
|
||||
|
||||
def run(self, *, date: str | None = None, force: bool = False) -> SnapshotSummary:
|
||||
"""兼容 CLI：先跑视频快照，再跑粉丝采集，合并进同一摘要。"""
|
||||
summary = self.run_videos(date=date, force=force)
|
||||
return self.run_fans(date=summary.stat_date, force=force, summary=summary)
|
||||
|
||||
def run_videos(self, *, date: str | None = None, force: bool = False) -> SnapshotSummary:
|
||||
"""采集视频七项指标并在全部成功后标记完成日 / 备份。"""
|
||||
cfg = self.config_store.get()
|
||||
client = self._client()
|
||||
now = self._now()
|
||||
@@ -240,6 +247,77 @@ class SnapshotService:
|
||||
skipped=summary.skipped, removed=summary.removed,
|
||||
)
|
||||
|
||||
# 完成日只看视频；粉丝是独立调度任务。
|
||||
summary.completed = summary.failed == 0
|
||||
self.repo.kv_set(self.completed_key(stat_date), "1" if summary.completed else "0")
|
||||
self.repo.kv_set("snapshot:last_attempt", stat_date)
|
||||
if summary.completed:
|
||||
self.repo.kv_set("snapshot:last_completed", stat_date)
|
||||
elif self.repo.kv_get("snapshot:last_completed") == stat_date:
|
||||
self.repo.kv_set("snapshot:last_completed", "")
|
||||
|
||||
self._run_backup(cfg, stat_date, summary)
|
||||
|
||||
if summary.success or summary.removed or summary.failed:
|
||||
self.repo.bump_cache_version("videos")
|
||||
|
||||
summary.finished_at = self._now()
|
||||
self._log(
|
||||
"info" if summary.completed else "warning",
|
||||
"snapshot.done",
|
||||
**summary.as_dict(),
|
||||
)
|
||||
return summary
|
||||
|
||||
def run_fans_if_needed(
|
||||
self, *, date: str | None = None, force: bool = False
|
||||
) -> SnapshotSummary:
|
||||
now = self._now()
|
||||
stat_date = date or shanghai_date(now)
|
||||
if not is_date(stat_date):
|
||||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||||
cfg = self.config_store.get()
|
||||
if not cfg.track_fans:
|
||||
summary = SnapshotSummary(
|
||||
stat_date=stat_date, started_at=now, finished_at=now, completed=True,
|
||||
)
|
||||
self._log("info", "fans.skip", stat_date=stat_date, reason="no_track_fans")
|
||||
return summary
|
||||
if not force and not self._pending_fan_mids(stat_date, cfg.track_fans):
|
||||
self._log("info", "fans.skip", stat_date=stat_date, reason="already_completed")
|
||||
return SnapshotSummary(
|
||||
stat_date=stat_date,
|
||||
started_at=now,
|
||||
finished_at=now,
|
||||
completed=True,
|
||||
already_completed=True,
|
||||
fans_skipped=len(cfg.track_fans),
|
||||
)
|
||||
return self.run_fans(date=stat_date, force=force)
|
||||
|
||||
def run_fans(
|
||||
self,
|
||||
*,
|
||||
date: str | None = None,
|
||||
force: bool = False,
|
||||
summary: SnapshotSummary | None = None,
|
||||
) -> SnapshotSummary:
|
||||
"""采集 track_fans 粉丝数；成败不影响视频完成日。"""
|
||||
cfg = self.config_store.get()
|
||||
client = self._client()
|
||||
now = self._now()
|
||||
stat_date = date or shanghai_date(now)
|
||||
if not is_date(stat_date):
|
||||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||||
|
||||
standalone = summary is None
|
||||
if summary is None:
|
||||
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
|
||||
else:
|
||||
# CLI 合并跑：保留视频摘要，只追加粉丝段耗时终点
|
||||
summary.stat_date = stat_date
|
||||
|
||||
self._log("info", "fans.start", stat_date=stat_date, force=force, total=len(cfg.track_fans))
|
||||
for mid in cfg.track_fans:
|
||||
existing_fan = self.repo.get_up_stat(stat_date, mid)
|
||||
if (
|
||||
@@ -276,29 +354,33 @@ class SnapshotService:
|
||||
stat_date=stat_date, mid=mid, error=error,
|
||||
)
|
||||
|
||||
summary.completed = summary.failed == 0 and summary.fans_failed == 0
|
||||
self.repo.kv_set(self.completed_key(stat_date), "1" if summary.completed else "0")
|
||||
self.repo.kv_set("snapshot:last_attempt", stat_date)
|
||||
if summary.completed:
|
||||
self.repo.kv_set("snapshot:last_completed", stat_date)
|
||||
elif self.repo.kv_get("snapshot:last_completed") == stat_date:
|
||||
self.repo.kv_set("snapshot:last_completed", "")
|
||||
|
||||
self._run_backup(cfg, stat_date, summary)
|
||||
|
||||
if summary.success or summary.removed or summary.failed:
|
||||
self.repo.bump_cache_version("videos")
|
||||
if summary.fans_ok or summary.fans_failed:
|
||||
self.repo.bump_cache_version("fans")
|
||||
|
||||
fans_ok = summary.fans_failed == 0
|
||||
# 独立粉丝任务用 completed 表示粉丝段成败；合并进视频摘要时不改动视频完成标记。
|
||||
if standalone:
|
||||
summary.completed = fans_ok
|
||||
summary.finished_at = self._now()
|
||||
self._log(
|
||||
"info" if summary.completed else "warning",
|
||||
"snapshot.done",
|
||||
**summary.as_dict(),
|
||||
"info" if fans_ok else "warning",
|
||||
"fans.done",
|
||||
stat_date=stat_date,
|
||||
fans_ok=summary.fans_ok,
|
||||
fans_failed=summary.fans_failed,
|
||||
fans_skipped=summary.fans_skipped,
|
||||
completed=summary.completed,
|
||||
)
|
||||
return summary
|
||||
|
||||
def _pending_fan_mids(self, stat_date: str, mids: tuple[int, ...] | list[int]) -> list[int]:
|
||||
pending: list[int] = []
|
||||
for mid in mids:
|
||||
existing = self.repo.get_up_stat(stat_date, int(mid))
|
||||
if existing is None or int(existing.get("ok") or 0) != 1:
|
||||
pending.append(int(mid))
|
||||
return pending
|
||||
|
||||
# ------------------------------------------------------------ internals
|
||||
|
||||
def _run_backup(
|
||||
|
||||
Reference in new issue
Block a user