feat: EOD 23:15 seal + midnight seed shell (video/fans) #3
No files matched your search
+3
-3
@@ -295,7 +295,7 @@ PRAGMA synchronous = NORMAL;
|
|||||||
"schedule": {
|
"schedule": {
|
||||||
"discover_minutes": 60,
|
"discover_minutes": 60,
|
||||||
"snapshot_at": "23:15",
|
"snapshot_at": "23:15",
|
||||||
"seed_at": "00:01",
|
"seed_at": "00:00",
|
||||||
"candidate_ttl_days": 7
|
"candidate_ttl_days": 7
|
||||||
},
|
},
|
||||||
"notify": { "min_score": 2, "max_items_per_message": 10 }
|
"notify": { "min_score": 2, "max_items_per_message": 10 }
|
||||||
@@ -435,7 +435,7 @@ P3 实现说明:
|
|||||||
|
|
||||||
### 7.4 每日快照与粉丝采集
|
### 7.4 每日快照与粉丝采集
|
||||||
|
|
||||||
每天 `23:15`(`Asia/Shanghai`)执行全量落盘;`00:01` 用上一完成日克隆当日壳(seed):
|
每天 `23:15`(`Asia/Shanghai`)执行全量落盘;`00:00` 用上一完成日克隆当日壳(seed):
|
||||||
|
|
||||||
- 对 `status IN ('active', 'unfaved')` 的视频逐条请求 `view?aid=`;
|
- 对 `status IN ('active', 'unfaved')` 的视频逐条请求 `view?aid=`;
|
||||||
- 成功写入播放、点赞、收藏、投币、转发、评论、弹幕七项数据,并把 `view` 的 `owner.face` 写入 `video.up_face`(UP 头像,供 UP 列表/详情展示);
|
- 成功写入播放、点赞、收藏、投币、转发、评论、弹幕七项数据,并把 `view` 的 `owner.face` 写入 `video.up_face`(UP 头像,供 UP 列表/详情展示);
|
||||||
@@ -451,7 +451,7 @@ P3 实现说明:
|
|||||||
P1 实现说明:
|
P1 实现说明:
|
||||||
|
|
||||||
- 时区使用固定 UTC+8(中国无夏令时),避免 Windows 缺少 `tzdata`;
|
- 时区使用固定 UTC+8(中国无夏令时),避免 Windows 缺少 `tzdata`;
|
||||||
- 午夜 `seed_at`(默认 `00:01`)把 `snapshot:last_completed` 的 ok=1 行克隆为当日壳,写入 `snapshot:seeded:<date>=1` / `fans:seeded:<date>=1`,**不**推进完成日;
|
- 午夜 `seed_at`(默认 `00:00`)把 `snapshot:last_completed` 的 ok=1 行克隆为当日壳,写入 `snapshot:seeded:<date>=1` / `fans:seeded:<date>=1`,**不**推进完成日;
|
||||||
- 晚间 `snapshot_at`(默认 `23:15`)对 seeded 日 force 重采;成功后写 `snapshot:completed:<date>=1`、推进 `last_completed` 并清除 seeded;
|
- 晚间 `snapshot_at`(默认 `23:15`)对 seeded 日 force 重采;成功后写 `snapshot:completed:<date>=1`、推进 `last_completed` 并清除 seeded;
|
||||||
- 当日全部视频与粉丝采集成功后在 `kv` 写入 `snapshot:completed:<date>=1`;启动时未完成会补跑(纯 seed 日不误标完成);
|
- 当日全部视频与粉丝采集成功后在 `kv` 写入 `snapshot:completed:<date>=1`;启动时未完成会补跑(纯 seed 日不误标完成);
|
||||||
- 完成标记不是唯一门槛:每次检查 `pending_snapshot_aids(date)`,新入库或从 `dropped` 恢复的视频会立即补采当天快照;
|
- 完成标记不是唯一门槛:每次检查 `pending_snapshot_aids(date)`,新入库或从 `dropped` 恢复的视频会立即补采当天快照;
|
||||||
|
|||||||
@@ -64,7 +64,7 @@
|
|||||||
"schedule": {
|
"schedule": {
|
||||||
"discover_minutes": 60,
|
"discover_minutes": 60,
|
||||||
"snapshot_at": "23:15",
|
"snapshot_at": "23:15",
|
||||||
"seed_at": "00:01",
|
"seed_at": "00:00",
|
||||||
"candidate_ttl_days": 7,
|
"candidate_ttl_days": 7,
|
||||||
"recent_snapshot_minutes": 60,
|
"recent_snapshot_minutes": 60,
|
||||||
"recent_video_days": 2
|
"recent_video_days": 2
|
||||||
|
|||||||
+1
-1
@@ -74,7 +74,7 @@
|
|||||||
"schedule": {
|
"schedule": {
|
||||||
"discover_minutes": 60,
|
"discover_minutes": 60,
|
||||||
"snapshot_at": "23:15",
|
"snapshot_at": "23:15",
|
||||||
"seed_at": "00:01",
|
"seed_at": "00:00",
|
||||||
"candidate_ttl_days": 7,
|
"candidate_ttl_days": 7,
|
||||||
"recent_snapshot_minutes": 60,
|
"recent_snapshot_minutes": 60,
|
||||||
"recent_video_days": 2
|
"recent_video_days": 2
|
||||||
|
|||||||
+4
-4
@@ -117,7 +117,7 @@ Telegram 私聊发链接/BV 号 ───┘
|
|||||||
145178393, 66389997, 293976264, 3026109
|
145178393, 66389997, 293976264, 3026109
|
||||||
],
|
],
|
||||||
"schedule": { "discover_minutes": 60,
|
"schedule": { "discover_minutes": 60,
|
||||||
"snapshot_at": "23:15", "seed_at": "00:01", "candidate_ttl_days": 7 },
|
"snapshot_at": "23:15", "seed_at": "00:00", "candidate_ttl_days": 7 },
|
||||||
"discover": { "keywords": ["咻咻满"], "pages": 3, "window_days": 7,
|
"discover": { "keywords": ["咻咻满"], "pages": 3, "window_days": 7,
|
||||||
"candidate_min_score": 4 },
|
"candidate_min_score": 4 },
|
||||||
"notify": { "min_score": 4, "max_items_per_message": 10 },
|
"notify": { "min_score": 4, "max_items_per_message": 10 },
|
||||||
@@ -377,7 +377,7 @@ for folder in 配置的年度夹(2025、2026、…):
|
|||||||
### 5.1 任务
|
### 5.1 任务
|
||||||
|
|
||||||
```
|
```
|
||||||
每天 23:15(Asia/Shanghai)全量落盘;00:01 克隆当日壳:
|
每天 23:15(Asia/Shanghai)全量落盘;00:00 克隆当日壳:
|
||||||
videos = db.where(status='active') # 跨所有年度夹
|
videos = db.where(status='active') # 跨所有年度夹
|
||||||
for v in videos:
|
for v in videos:
|
||||||
if exists(snapshot[today, v.aid]) and ok: continue # 断点续跑
|
if exists(snapshot[today, v.aid]) and ok: continue # 断点续跑
|
||||||
@@ -397,7 +397,7 @@ for folder in 配置的年度夹(2025、2026、…):
|
|||||||
| 1600(2027 年末) | 1600 | 约 33 分钟 |
|
| 1600(2027 年末) | 1600 | 约 33 分钟 |
|
||||||
| 3000(2029 年) | 3000 | 约 60 分钟 |
|
| 3000(2029 年) | 3000 | 约 60 分钟 |
|
||||||
|
|
||||||
- 23:15 启动全量落盘;00:01 仅本地克隆壳,不影响白天使用;
|
- 23:15 启动全量落盘;00:00 仅本地克隆壳,不影响白天使用;
|
||||||
- 若单条 0.1s + sleep 1.2s ≈ 1.3s,实际按 1.2s 估算偏保守。
|
- 若单条 0.1s + sleep 1.2s ≈ 1.3s,实际按 1.2s 估算偏保守。
|
||||||
|
|
||||||
### 5.3 幂等与口径
|
### 5.3 幂等与口径
|
||||||
@@ -782,7 +782,7 @@ config.docker.example.json 服务器配置样例(0.0.0.0 + /data 绝对路径
|
|||||||
**进程结构**
|
**进程结构**
|
||||||
|
|
||||||
```
|
```
|
||||||
scheduler 线程:discovery(每 60min) / snapshot_seed(每日 00:01) / snapshot+fans(每日 23:15) / snapshot_recent(小时) / candidate_cleanup(每日)
|
scheduler 线程:discovery(每 60min) / snapshot_seed(每日 00:00) / snapshot+fans(每日 23:15) / snapshot_recent(小时) / candidate_cleanup(每日)
|
||||||
web 线程:ThreadingHTTPServer :8080
|
web 线程:ThreadingHTTPServer :8080
|
||||||
telegram:getUpdates long polling(回调 + 私聊文本入库)
|
telegram:getUpdates long polling(回调 + 私聊文本入库)
|
||||||
storage:SQLite WAL,单写线程
|
storage:SQLite WAL,单写线程
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ DEFAULT_CONFIG: dict[str, Any] = {
|
|||||||
"schedule": {
|
"schedule": {
|
||||||
"discover_minutes": 60,
|
"discover_minutes": 60,
|
||||||
"snapshot_at": "23:15",
|
"snapshot_at": "23:15",
|
||||||
"seed_at": "00:01",
|
"seed_at": "00:00",
|
||||||
"candidate_ttl_days": 7,
|
"candidate_ttl_days": 7,
|
||||||
"recent_snapshot_minutes": 60,
|
"recent_snapshot_minutes": 60,
|
||||||
"recent_video_days": 2,
|
"recent_video_days": 2,
|
||||||
@@ -454,7 +454,7 @@ def _build_config(data: Mapping[str, Any], path: Path) -> AppConfig:
|
|||||||
snapshot_at = _as_str(sched_raw.get("snapshot_at", "23:15"), "schedule.snapshot_at")
|
snapshot_at = _as_str(sched_raw.get("snapshot_at", "23:15"), "schedule.snapshot_at")
|
||||||
if not _SNAPSHOT_AT_RE.match(snapshot_at):
|
if not _SNAPSHOT_AT_RE.match(snapshot_at):
|
||||||
raise ConfigError("schedule.snapshot_at 必须是 HH:MM(24 小时制)")
|
raise ConfigError("schedule.snapshot_at 必须是 HH:MM(24 小时制)")
|
||||||
seed_at = _as_str(sched_raw.get("seed_at", "00:01"), "schedule.seed_at")
|
seed_at = _as_str(sched_raw.get("seed_at", "00:00"), "schedule.seed_at")
|
||||||
if not _SNAPSHOT_AT_RE.match(seed_at):
|
if not _SNAPSHOT_AT_RE.match(seed_at):
|
||||||
raise ConfigError("schedule.seed_at 必须是 HH:MM(24 小时制)")
|
raise ConfigError("schedule.seed_at 必须是 HH:MM(24 小时制)")
|
||||||
schedule = ScheduleConfig(
|
schedule = ScheduleConfig(
|
||||||
|
|||||||
@@ -142,6 +142,8 @@ def test_analytics_defaults_and_validation():
|
|||||||
assert cfg.analytics.home_growth_days == "auto"
|
assert cfg.analytics.home_growth_days == "auto"
|
||||||
assert cfg.schedule.recent_snapshot_minutes == 60
|
assert cfg.schedule.recent_snapshot_minutes == 60
|
||||||
assert cfg.schedule.recent_video_days == 2
|
assert cfg.schedule.recent_video_days == 2
|
||||||
|
assert cfg.schedule.snapshot_at == "23:15"
|
||||||
|
assert cfg.schedule.seed_at == "00:00"
|
||||||
|
|
||||||
data = base_config()
|
data = base_config()
|
||||||
data["analytics"] = {
|
data["analytics"] = {
|
||||||
@@ -155,7 +157,7 @@ def test_analytics_defaults_and_validation():
|
|||||||
data["schedule"] = {
|
data["schedule"] = {
|
||||||
"discover_minutes": 30,
|
"discover_minutes": 30,
|
||||||
"snapshot_at": "23:15",
|
"snapshot_at": "23:15",
|
||||||
"seed_at": "00:01",
|
"seed_at": "00:00",
|
||||||
"candidate_ttl_days": 7,
|
"candidate_ttl_days": 7,
|
||||||
"recent_snapshot_minutes": 15,
|
"recent_snapshot_minutes": 15,
|
||||||
"recent_video_days": 3,
|
"recent_video_days": 3,
|
||||||
@@ -278,7 +280,7 @@ def test_schedule_recent_validation():
|
|||||||
data["schedule"] = {
|
data["schedule"] = {
|
||||||
"discover_minutes": 60,
|
"discover_minutes": 60,
|
||||||
"snapshot_at": "23:15",
|
"snapshot_at": "23:15",
|
||||||
"seed_at": "00:01",
|
"seed_at": "00:00",
|
||||||
"candidate_ttl_days": 7,
|
"candidate_ttl_days": 7,
|
||||||
**bad,
|
**bad,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ def write_config(
|
|||||||
"schedule": {
|
"schedule": {
|
||||||
"discover_minutes": 60,
|
"discover_minutes": 60,
|
||||||
"snapshot_at": "23:15",
|
"snapshot_at": "23:15",
|
||||||
"seed_at": "00:01",
|
"seed_at": "00:00",
|
||||||
"candidate_ttl_days": 7,
|
"candidate_ttl_days": 7,
|
||||||
"recent_snapshot_minutes": 60,
|
"recent_snapshot_minutes": 60,
|
||||||
"recent_video_days": 2,
|
"recent_video_days": 2,
|
||||||
|
|||||||
Reference in new issue
Block a user