feat: EOD 23:15 seal + midnight seed shell for video/fans
Move daily snapshot/fans to 23:15 and add 00:01 seed that clones last_completed ok=1 rows into today without advancing completed. Evening runs force through seed markers; growth previous is per-aid adjacent. Home shows short-update clock + 全站数据 date, and 收录视频 counts.
This commit is contained in:
1 parent
374b138041
commit
8a9cb21408
17 files changed
+520
-39
No files matched your search
@@ -5,6 +5,8 @@
|
||||
|
||||
* 对 ``active`` / ``unfaved`` 视频逐条请求 ``view``;
|
||||
* 成功写入七项指标;``ok=1`` 的同日快照直接跳过(断点续跑;含小时任务 1 小时内刚刷新的);
|
||||
* 午夜 ``seed_at`` 用上一完成日 ok=1 行克隆出当日壳(``snapshot:seeded`` / ``fans:seeded``),不推进完成日;
|
||||
* 晚间 ``snapshot_at`` 全量必须 force 打穿 seed,成功后才写完成日并清除 seeded;
|
||||
* ``run_recent_videos`` 只刷新近 N 天新片的当日行,不改完成日 / 备份;
|
||||
* 视频不可访问时标记 ``removed``;
|
||||
* 其他错误最多重试 3 次,仍失败写 ``ok=0`` 占位,不写假零值;
|
||||
@@ -129,18 +131,43 @@ class SnapshotService:
|
||||
def backup_error_key(stat_date: str) -> str:
|
||||
return f"snapshot:backup_error:{stat_date}"
|
||||
|
||||
@staticmethod
|
||||
def seeded_key(stat_date: str) -> str:
|
||||
return f"snapshot:seeded:{stat_date}"
|
||||
|
||||
@staticmethod
|
||||
def fans_seeded_key(stat_date: str) -> str:
|
||||
return f"fans:seeded:{stat_date}"
|
||||
|
||||
def run_if_needed(
|
||||
self, *, date: str | None = None, force: bool = False
|
||||
) -> SnapshotSummary:
|
||||
"""视频快照(含 UP 头像补齐);粉丝采集请用 ``run_fans_if_needed``。"""
|
||||
"""视频快照(含 UP 头像补齐);粉丝采集请用 ``run_fans_if_needed``。
|
||||
|
||||
当日若仅有午夜 seed 壳且未 ``force``,直接返回等待晚间落盘,
|
||||
避免把「全员 skip」误标成 completed。
|
||||
"""
|
||||
now = self._now()
|
||||
stat_date = date or shanghai_date(now)
|
||||
if not is_date(stat_date):
|
||||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||||
seeded = self.repo.kv_get(self.seeded_key(stat_date)) == "1"
|
||||
if force:
|
||||
return self.run_videos(date=stat_date, force=True)
|
||||
|
||||
completed = self.repo.kv_get(self.completed_key(stat_date)) == "1"
|
||||
if seeded and not completed:
|
||||
self._log(
|
||||
"info", "snapshot.skip",
|
||||
stat_date=stat_date, reason="seeded_pending_seal",
|
||||
)
|
||||
return SnapshotSummary(
|
||||
stat_date=stat_date,
|
||||
started_at=now,
|
||||
finished_at=now,
|
||||
completed=False,
|
||||
already_completed=False,
|
||||
)
|
||||
pending = self.repo.pending_snapshot_aids(stat_date)
|
||||
if completed and not pending:
|
||||
if self.repo.kv_get(self.backup_key(stat_date)) != "1":
|
||||
@@ -256,6 +283,8 @@ class SnapshotService:
|
||||
self.repo.kv_set("snapshot:last_attempt", stat_date)
|
||||
if summary.completed:
|
||||
self.repo.kv_set("snapshot:last_completed", stat_date)
|
||||
# 真落盘后清掉午夜 seed 标记。
|
||||
self.repo.kv_delete(self.seeded_key(stat_date))
|
||||
elif self.repo.kv_get("snapshot:last_completed") == stat_date:
|
||||
self.repo.kv_set("snapshot:last_completed", "")
|
||||
|
||||
@@ -378,7 +407,24 @@ class SnapshotService:
|
||||
)
|
||||
self._log("info", "fans.skip", stat_date=stat_date, reason="no_track_fans")
|
||||
return summary
|
||||
if not force and not self._pending_fan_mids(stat_date, cfg.track_fans):
|
||||
seeded = self.repo.kv_get(self.fans_seeded_key(stat_date)) == "1"
|
||||
if force or seeded:
|
||||
# 晚间/显式 force,或存在 seed 壳时必须重采(seeded 时由调度传入 force)。
|
||||
if seeded and not force:
|
||||
self._log(
|
||||
"info", "fans.skip",
|
||||
stat_date=stat_date, reason="seeded_pending_seal",
|
||||
)
|
||||
return SnapshotSummary(
|
||||
stat_date=stat_date,
|
||||
started_at=now,
|
||||
finished_at=now,
|
||||
completed=False,
|
||||
already_completed=False,
|
||||
fans_skipped=len(cfg.track_fans),
|
||||
)
|
||||
return self.run_fans(date=stat_date, force=True)
|
||||
if not self._pending_fan_mids(stat_date, cfg.track_fans):
|
||||
self._log("info", "fans.skip", stat_date=stat_date, reason="already_completed")
|
||||
return SnapshotSummary(
|
||||
stat_date=stat_date,
|
||||
@@ -456,6 +502,11 @@ class SnapshotService:
|
||||
# 独立粉丝任务用 completed 表示粉丝段成败;合并进视频摘要时不改动视频完成标记。
|
||||
if standalone:
|
||||
summary.completed = fans_ok
|
||||
if fans_ok:
|
||||
self.repo.kv_delete(self.fans_seeded_key(stat_date))
|
||||
elif fans_ok:
|
||||
# CLI 合并跑:粉丝段成功也清 seed。
|
||||
self.repo.kv_delete(self.fans_seeded_key(stat_date))
|
||||
summary.finished_at = self._now()
|
||||
self._log(
|
||||
"info" if fans_ok else "warning",
|
||||
@@ -468,6 +519,76 @@ class SnapshotService:
|
||||
)
|
||||
return summary
|
||||
|
||||
|
||||
def run_seed_from_last_completed(
|
||||
self, *, date: str | None = None
|
||||
) -> SnapshotSummary:
|
||||
"""用上一完成日的 ok=1 行克隆出当日壳(视频 + 粉丝)。
|
||||
|
||||
不写 ``snapshot:completed`` / ``snapshot:last_completed``;
|
||||
仅打 ``snapshot:seeded`` / ``fans:seeded``,供白天展示与晚间 force 重采。
|
||||
"""
|
||||
cfg = self.config_store.get()
|
||||
now = self._now()
|
||||
stat_date = date or shanghai_date(now)
|
||||
if not is_date(stat_date):
|
||||
raise ValueError(f"日期格式必须是 YYYY-MM-DD: {stat_date!r}")
|
||||
|
||||
summary = SnapshotSummary(stat_date=stat_date, started_at=now)
|
||||
source = self.repo.kv_get("snapshot:last_completed")
|
||||
if not source or source >= stat_date:
|
||||
self._log(
|
||||
"info", "snapshot.seed_skip",
|
||||
stat_date=stat_date, reason="no_source", source=source,
|
||||
)
|
||||
summary.finished_at = self._now()
|
||||
summary.completed = True
|
||||
return summary
|
||||
if self.repo.kv_get(self.completed_key(stat_date)) == "1":
|
||||
self._log(
|
||||
"info", "snapshot.seed_skip",
|
||||
stat_date=stat_date, reason="already_completed",
|
||||
)
|
||||
summary.finished_at = self._now()
|
||||
summary.completed = True
|
||||
summary.already_completed = True
|
||||
return summary
|
||||
if self.repo.kv_get(self.seeded_key(stat_date)) == "1":
|
||||
self._log(
|
||||
"info", "snapshot.seed_skip",
|
||||
stat_date=stat_date, reason="already_seeded", source=source,
|
||||
)
|
||||
summary.finished_at = self._now()
|
||||
summary.completed = True
|
||||
summary.already_completed = True
|
||||
return summary
|
||||
|
||||
copied = self.repo.copy_ok_snapshots(
|
||||
source, stat_date, captured_at=now, src="seed",
|
||||
)
|
||||
fans_copied = 0
|
||||
if cfg.track_fans:
|
||||
fans_copied = self.repo.copy_ok_up_stats(
|
||||
source, stat_date, captured_at=now, mids=cfg.track_fans,
|
||||
)
|
||||
self.repo.kv_set(self.fans_seeded_key(stat_date), "1")
|
||||
self.repo.kv_set(self.seeded_key(stat_date), "1")
|
||||
summary.total = copied
|
||||
summary.success = copied
|
||||
summary.fans_ok = fans_copied
|
||||
if copied or fans_copied:
|
||||
self.repo.bump_cache_version("videos")
|
||||
if fans_copied:
|
||||
self.repo.bump_cache_version("fans")
|
||||
summary.finished_at = self._now()
|
||||
summary.completed = True # seed 任务本身成功;不等于快照落盘完成
|
||||
self._log(
|
||||
"info", "snapshot.seed_done",
|
||||
stat_date=stat_date, source=source,
|
||||
copied=copied, fans_copied=fans_copied,
|
||||
)
|
||||
return summary
|
||||
|
||||
def _pending_fan_mids(self, stat_date: str, mids: tuple[int, ...] | list[int]) -> list[int]:
|
||||
pending: list[int] = []
|
||||
for mid in mids:
|
||||
|
||||
Reference in new issue
Block a user