feat: EOD 23:15 seal + midnight seed shell (video/fans) #3

Merged
kairee merged 2 commits from auto/snapshot-eod-seed-4227 into main 2026-11-02 12:46:55 +08:00
7 changed files with 16 additions and 14 deletions
Showing only changes of commit 25cdea5287 - Show all commits

No files matched your search

+3 -3
View File
@@ -295,7 +295,7 @@ PRAGMA synchronous = NORMAL;
"schedule": {
"discover_minutes": 60,
"snapshot_at": "23:15",
"seed_at": "00:01",
"seed_at": "00:00",
"candidate_ttl_days": 7
},
"notify": { "min_score": 2, "max_items_per_message": 10 }
@@ -435,7 +435,7 @@ P3 实现说明:
### 7.4 每日快照与粉丝采集
每天 `23:15`(`Asia/Shanghai`)执行全量落盘;`00:01` 用上一完成日克隆当日壳(seed):
每天 `23:15`(`Asia/Shanghai`)执行全量落盘;`00:00` 用上一完成日克隆当日壳(seed):
- 对 `status IN ('active', 'unfaved')` 的视频逐条请求 `view?aid=`;
- 成功写入播放、点赞、收藏、投币、转发、评论、弹幕七项数据,并把 `view` 的 `owner.face` 写入 `video.up_face`(UP 头像,供 UP 列表/详情展示);
@@ -451,7 +451,7 @@ P3 实现说明:
P1 实现说明:
- 时区使用固定 UTC+8(中国无夏令时),避免 Windows 缺少 `tzdata`;
- 午夜 `seed_at`(默认 `00:01`)把 `snapshot:last_completed` 的 ok=1 行克隆为当日壳,写入 `snapshot:seeded:<date>=1` / `fans:seeded:<date>=1`,**不**推进完成日;
- 午夜 `seed_at`(默认 `00:00`)把 `snapshot:last_completed` 的 ok=1 行克隆为当日壳,写入 `snapshot:seeded:<date>=1` / `fans:seeded:<date>=1`,**不**推进完成日;
- 晚间 `snapshot_at`(默认 `23:15`)对 seeded 日 force 重采;成功后写 `snapshot:completed:<date>=1`、推进 `last_completed` 并清除 seeded;
- 当日全部视频与粉丝采集成功后在 `kv` 写入 `snapshot:completed:<date>=1`;启动时未完成会补跑(纯 seed 日不误标完成);
- 完成标记不是唯一门槛:每次检查 `pending_snapshot_aids(date)`,新入库或从 `dropped` 恢复的视频会立即补采当天快照;
+1 -1
View File
@@ -64,7 +64,7 @@
"schedule": {
"discover_minutes": 60,
"snapshot_at": "23:15",
"seed_at": "00:01",
"seed_at": "00:00",
"candidate_ttl_days": 7,
"recent_snapshot_minutes": 60,
"recent_video_days": 2
+1 -1
View File
@@ -74,7 +74,7 @@
"schedule": {
"discover_minutes": 60,
"snapshot_at": "23:15",
"seed_at": "00:01",
"seed_at": "00:00",
"candidate_ttl_days": 7,
"recent_snapshot_minutes": 60,
"recent_video_days": 2
+4 -4
View File
@@ -117,7 +117,7 @@ Telegram 私聊发链接/BV 号 ───┘
145178393, 66389997, 293976264, 3026109
],
"schedule": { "discover_minutes": 60,
"snapshot_at": "23:15", "seed_at": "00:01", "candidate_ttl_days": 7 },
"snapshot_at": "23:15", "seed_at": "00:00", "candidate_ttl_days": 7 },
"discover": { "keywords": ["咻咻满"], "pages": 3, "window_days": 7,
"candidate_min_score": 4 },
"notify": { "min_score": 4, "max_items_per_message": 10 },
@@ -377,7 +377,7 @@ for folder in 配置的年度夹(2025、2026、…):
### 5.1 任务
```
每天 23:15(Asia/Shanghai)全量落盘;00:01 克隆当日壳:
每天 23:15(Asia/Shanghai)全量落盘;00:00 克隆当日壳:
videos = db.where(status='active') # 跨所有年度夹
for v in videos:
if exists(snapshot[today, v.aid]) and ok: continue # 断点续跑
@@ -397,7 +397,7 @@ for folder in 配置的年度夹(2025、2026、…):
| 1600(2027 年末) | 1600 | 约 33 分钟 |
| 3000(2029 年) | 3000 | 约 60 分钟 |
- 23:15 启动全量落盘;00:01 仅本地克隆壳,不影响白天使用;
- 23:15 启动全量落盘;00:00 仅本地克隆壳,不影响白天使用;
- 若单条 0.1s + sleep 1.2s ≈ 1.3s,实际按 1.2s 估算偏保守。
### 5.3 幂等与口径
@@ -782,7 +782,7 @@ config.docker.example.json 服务器配置样例(0.0.0.0 + /data 绝对路径
**进程结构**
```
scheduler 线程:discovery(每 60min) / snapshot_seed(每日 00:01) / snapshot+fans(每日 23:15) / snapshot_recent(小时) / candidate_cleanup(每日)
scheduler 线程:discovery(每 60min) / snapshot_seed(每日 00:00) / snapshot+fans(每日 23:15) / snapshot_recent(小时) / candidate_cleanup(每日)
web 线程:ThreadingHTTPServer :8080
telegram:getUpdates long polling(回调 + 私聊文本入库)
storage:SQLite WAL,单写线程
+2 -2
View File
@@ -84,7 +84,7 @@ DEFAULT_CONFIG: dict[str, Any] = {
"schedule": {
"discover_minutes": 60,
"snapshot_at": "23:15",
"seed_at": "00:01",
"seed_at": "00:00",
"candidate_ttl_days": 7,
"recent_snapshot_minutes": 60,
"recent_video_days": 2,
@@ -454,7 +454,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
snapshot_at = _as_str(sched_raw.get("snapshot_at", "23:15"), "schedule.snapshot_at")
if not _SNAPSHOT_AT_RE.match(snapshot_at):
raise ConfigError("schedule.snapshot_at 必须是 HH:MM(24 小时制)")
seed_at = _as_str(sched_raw.get("seed_at", "00:01"), "schedule.seed_at")
seed_at = _as_str(sched_raw.get("seed_at", "00:00"), "schedule.seed_at")
if not _SNAPSHOT_AT_RE.match(seed_at):
raise ConfigError("schedule.seed_at 必须是 HH:MM(24 小时制)")
schedule = ScheduleConfig(
+4 -2
View File
@@ -142,6 +142,8 @@ def test_analytics_defaults_and_validation():
assert cfg.analytics.home_growth_days == "auto"
assert cfg.schedule.recent_snapshot_minutes == 60
assert cfg.schedule.recent_video_days == 2
assert cfg.schedule.snapshot_at == "23:15"
assert cfg.schedule.seed_at == "00:00"
data = base_config()
data["analytics"] = {
@@ -155,7 +157,7 @@ def test_analytics_defaults_and_validation():
data["schedule"] = {
"discover_minutes": 30,
"snapshot_at": "23:15",
"seed_at": "00:01",
"seed_at": "00:00",
"candidate_ttl_days": 7,
"recent_snapshot_minutes": 15,
"recent_video_days": 3,
@@ -278,7 +280,7 @@ def test_schedule_recent_validation():
data["schedule"] = {
"discover_minutes": 60,
"snapshot_at": "23:15",
"seed_at": "00:01",
"seed_at": "00:00",
"candidate_ttl_days": 7,
**bad,
}
+1 -1
View File
@@ -49,7 +49,7 @@ def write_config(
"schedule": {
"discover_minutes": 60,
"snapshot_at": "23:15",
"seed_at": "00:01",
"seed_at": "00:00",
"candidate_ttl_days": 7,
"recent_snapshot_minutes": 60,
"recent_video_days": 2,