feat(p1): add daily snapshots and fan tracking

Implement P1: seven-metric view snapshots, relation/stat fan snapshots, failure placeholders, resume by daily completion markers, Cookie login warnings, daily SQLite backups, Shanghai daily scheduling, and CLI snapshot/auth-check commands. Real read-only run captured 795/795 videos successfully over 1124s with a verified daily backup.
This commit is contained in:
kairee committed 2026-10-31 19:19:24 +08:00
1 parent ee289bfa56
commit 5c2ab7b187
20 files changed
+1680 -54

No files matched your search

+121 -31
View File
@@ -1,10 +1,12 @@
# -*- coding: utf-8 -*-
"""P0 命令行入口。
"""看板命令行入口。
python app.py init # 建库 / 迁移 / 写入黑名单种子(离线)
python app.py sync # 收藏夹只读同步(首次导入或增量)
python app.py run # 按 folder_sync_minutes 常驻周期同步(--once 只跑一次)
python app.py status # 查看库内状态(离线)
python app.py init # 建库 / 迁移 / 写入黑名单种子(离线)
python app.py sync # 收藏夹只读同步(首次导入或增量)
python app.py snapshot # 采集当日七项快照与粉丝数
python app.py auth-check # 检查 Cookie 登录态
python app.py run # 按配置周期常驻运行(--once 只跑一次)
python app.py status # 查看库内状态(离线)
P2 起 ``python app.py`` 将在此基础上启动本地看板服务。
"""
@@ -17,13 +19,15 @@ import sys
from pathlib import Path
from typing import Sequence
from .bilibili.client import BilibiliClient
from .bilibili.client import BilibiliClient, Pacer
from .config import ConfigError, ConfigStore
from .database import Database
from .logging_json import JsonLogger
from .repository import Repository
from .scheduler import ScheduledTask, Scheduler
from .services.folder_sync import FolderSyncService
from .services.snapshot import SnapshotService
from .timeutil import next_daily_run, parse_hhmm, shanghai_date
ROOT = Path(__file__).resolve().parent.parent
DEFAULT_CONFIG = ROOT / "config.json"
@@ -39,7 +43,7 @@ def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="app.py",
description="咻咻满 Reaction 数据看板 —— P0 数据底座",
description="咻咻满 Reaction 数据看板 —— 数据底座、同步与每日快照",
)
sub = parser.add_subparsers(dest="command", required=True)
@@ -48,7 +52,14 @@ def _build_parser() -> argparse.ArgumentParser:
sync.add_argument(
"--full", action="store_true", help="强制全量刷新元数据(忽略 6 小时周期)"
)
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行收藏夹同步")
snapshot = sub.add_parser(
"snapshot", parents=[common], help="采集当日七项快照与粉丝数"
)
snapshot.add_argument(
"--force", action="store_true", help="忽略当日完成标记,重新采集"
)
sub.add_parser("auth-check", parents=[common], help="检查 Cookie 登录态")
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行同步与快照")
run.add_argument(
"--once", action="store_true", help="只执行一次到期任务后退出"
)
@@ -71,6 +82,17 @@ def _print(payload) -> None:
print(json.dumps(payload, ensure_ascii=False, indent=2), flush=True)
def _client(cfg, logger: JsonLogger, *, pacer=None, retries: int = 3) -> BilibiliClient:
return BilibiliClient(
cookie=cfg.bilibili.cookie,
csrf=cfg.bilibili.csrf,
mid=cfg.bilibili.mid,
pacer=pacer,
retries=retries,
logger=logger,
)
def cmd_init(args, logger: JsonLogger) -> int:
store = ConfigStore(args.config, require_auth=False)
cfg = store.get()
@@ -98,12 +120,7 @@ def cmd_sync(args, logger: JsonLogger) -> int:
db = _open_db(cfg, logger)
try:
repo = Repository(db)
client = BilibiliClient(
cookie=cfg.bilibili.cookie,
csrf=cfg.bilibili.csrf,
mid=cfg.bilibili.mid,
logger=logger,
)
client = _client(cfg, logger)
service = FolderSyncService(store, repo, client, logger=logger)
summary = service.run(full_refresh=args.full)
payload = summary.as_dict()
@@ -115,6 +132,46 @@ def cmd_sync(args, logger: JsonLogger) -> int:
db.close()
def cmd_snapshot(args, logger: JsonLogger) -> int:
store = ConfigStore(args.config, require_auth=False)
cfg = store.get()
db = _open_db(cfg, logger)
try:
repo = Repository(db)
client = _client(
cfg, logger, pacer=Pacer(1.0, 1.6), retries=1
)
service = SnapshotService(store, repo, client, logger=logger)
summary = service.run_if_needed(force=args.force)
payload = summary.as_dict()
payload["database"] = str(cfg.storage.database_path)
payload["snapshot_counts"] = repo.snapshot_counts(summary.stat_date)
payload["counts"] = repo.counts()
_print(payload)
return 0 if summary.completed else 1
finally:
db.close()
def cmd_auth_check(args, logger: JsonLogger) -> int:
store = ConfigStore(args.config, require_auth=False)
cfg = store.get()
if not cfg.bilibili.cookie:
_print({"ok": False, "is_login": False, "message": "未配置 cookie"})
return 1
client = _client(cfg, logger, retries=1)
state = client.check_login()
_print({
"ok": state.is_login,
"is_login": state.is_login,
"mid": state.mid,
"uname": state.uname,
"code": state.code,
"message": state.message,
})
return 0 if state.is_login else 1
def cmd_run(args, logger: JsonLogger) -> int:
store = ConfigStore(args.config, require_auth=True)
cfg = store.get()
@@ -122,26 +179,40 @@ def cmd_run(args, logger: JsonLogger) -> int:
last: dict[str, object] = {}
try:
repo = Repository(db)
client = BilibiliClient(
cookie=cfg.bilibili.cookie,
csrf=cfg.bilibili.csrf,
mid=cfg.bilibili.mid,
logger=logger,
)
service = FolderSyncService(store, repo, client, logger=logger)
folder_client = _client(cfg, logger)
folder_service = FolderSyncService(store, repo, folder_client, logger=logger)
snapshot_client = _client(cfg, logger, pacer=Pacer(1.0, 1.6), retries=1)
snapshot_service = SnapshotService(store, repo, snapshot_client, logger=logger)
def run_sync():
summary = service.run()
last["summary"] = summary
summary = folder_service.run()
last["folder"] = summary
logger.info("folder_sync.summary", **summary.as_dict())
return summary
def run_snapshot():
summary = snapshot_service.run_if_needed()
last["snapshot"] = summary
logger.info("snapshot.summary", **summary.as_dict())
return summary
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
snapshot_at = f"{hh:02d}:{mm:02d}"
scheduler = Scheduler(
[ScheduledTask(
"folder_sync",
float(cfg.schedule.folder_sync_minutes * 60),
run_sync,
)],
[
ScheduledTask(
"folder_sync",
float(cfg.schedule.folder_sync_minutes * 60),
run_sync,
),
ScheduledTask(
"snapshot",
86400.0,
run_snapshot,
next_run=0.0,
next_run_fn=lambda started: next_daily_run(started, snapshot_at),
),
],
logger=logger,
)
if args.once:
@@ -150,11 +221,16 @@ def cmd_run(args, logger: JsonLogger) -> int:
else:
logger.info(
"scheduler.start",
interval_seconds=cfg.schedule.folder_sync_minutes * 60,
folder_sync_seconds=cfg.schedule.folder_sync_minutes * 60,
snapshot_at=snapshot_at,
)
scheduler.run_forever()
summary = last.get("summary")
return 0 if summary is None or getattr(summary, "ok", True) else 1
folder_summary = last.get("folder")
snapshot_summary = last.get("snapshot")
folder_ok = folder_summary is None or getattr(folder_summary, "ok", True)
snapshot_ok = snapshot_summary is None or getattr(snapshot_summary, "completed", True)
return 0 if folder_ok and snapshot_ok else 1
finally:
db.close()
@@ -165,6 +241,15 @@ def cmd_status(args, logger: JsonLogger) -> int:
db = _open_db(cfg, logger)
try:
repo = Repository(db)
today = shanghai_date()
snapshot = repo.snapshot_counts(today)
snapshot.update({
"date": today,
"last_attempt": repo.kv_get("snapshot:last_attempt"),
"last_completed": repo.kv_get("snapshot:last_completed"),
"auth_ok": repo.kv_get("auth:ok"),
"auth_message": repo.kv_get("auth:message"),
})
_print({
"ok": True,
"schema_version": db.schema_version,
@@ -181,6 +266,7 @@ def cmd_status(args, logger: JsonLogger) -> int:
}
for row in repo.list_folders()
],
"snapshot": snapshot,
**repo.counts(),
})
finally:
@@ -196,6 +282,10 @@ def main(argv: Sequence[str] | None = None) -> int:
return cmd_init(args, logger)
if args.command == "sync":
return cmd_sync(args, logger)
if args.command == "snapshot":
return cmd_snapshot(args, logger)
if args.command == "auth-check":
return cmd_auth_check(args, logger)
if args.command == "run":
return cmd_run(args, logger)
if args.command == "status":