feat(p1): add daily snapshots and fan tracking
Implement P1: seven-metric view snapshots, relation/stat fan snapshots, failure placeholders, resume by daily completion markers, Cookie login warnings, daily SQLite backups, Shanghai daily scheduling, and CLI snapshot/auth-check commands. Real read-only run captured 795/795 videos successfully over 1124s with a verified daily backup.
This commit is contained in:
1 parent
ee289bfa56
commit
5c2ab7b187
20 files changed
+1680
-54
No files matched your search
+121
-31
@@ -1,10 +1,12 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""P0 命令行入口。
|
||||
"""看板命令行入口。
|
||||
|
||||
python app.py init # 建库 / 迁移 / 写入黑名单种子(离线)
|
||||
python app.py sync # 收藏夹只读同步(首次导入或增量)
|
||||
python app.py run # 按 folder_sync_minutes 常驻周期同步(--once 只跑一次)
|
||||
python app.py status # 查看库内状态(离线)
|
||||
python app.py init # 建库 / 迁移 / 写入黑名单种子(离线)
|
||||
python app.py sync # 收藏夹只读同步(首次导入或增量)
|
||||
python app.py snapshot # 采集当日七项快照与粉丝数
|
||||
python app.py auth-check # 检查 Cookie 登录态
|
||||
python app.py run # 按配置周期常驻运行(--once 只跑一次)
|
||||
python app.py status # 查看库内状态(离线)
|
||||
|
||||
P2 起 ``python app.py`` 将在此基础上启动本地看板服务。
|
||||
"""
|
||||
@@ -17,13 +19,15 @@ import sys
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
from .bilibili.client import BilibiliClient
|
||||
from .bilibili.client import BilibiliClient, Pacer
|
||||
from .config import ConfigError, ConfigStore
|
||||
from .database import Database
|
||||
from .logging_json import JsonLogger
|
||||
from .repository import Repository
|
||||
from .scheduler import ScheduledTask, Scheduler
|
||||
from .services.folder_sync import FolderSyncService
|
||||
from .services.snapshot import SnapshotService
|
||||
from .timeutil import next_daily_run, parse_hhmm, shanghai_date
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_CONFIG = ROOT / "config.json"
|
||||
@@ -39,7 +43,7 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="app.py",
|
||||
description="咻咻满 Reaction 数据看板 —— P0 数据底座",
|
||||
description="咻咻满 Reaction 数据看板 —— 数据底座、同步与每日快照",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
@@ -48,7 +52,14 @@ def _build_parser() -> argparse.ArgumentParser:
|
||||
sync.add_argument(
|
||||
"--full", action="store_true", help="强制全量刷新元数据(忽略 6 小时周期)"
|
||||
)
|
||||
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行收藏夹同步")
|
||||
snapshot = sub.add_parser(
|
||||
"snapshot", parents=[common], help="采集当日七项快照与粉丝数"
|
||||
)
|
||||
snapshot.add_argument(
|
||||
"--force", action="store_true", help="忽略当日完成标记,重新采集"
|
||||
)
|
||||
sub.add_parser("auth-check", parents=[common], help="检查 Cookie 登录态")
|
||||
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行同步与快照")
|
||||
run.add_argument(
|
||||
"--once", action="store_true", help="只执行一次到期任务后退出"
|
||||
)
|
||||
@@ -71,6 +82,17 @@ def _print(payload) -> None:
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2), flush=True)
|
||||
|
||||
|
||||
def _client(cfg, logger: JsonLogger, *, pacer=None, retries: int = 3) -> BilibiliClient:
|
||||
return BilibiliClient(
|
||||
cookie=cfg.bilibili.cookie,
|
||||
csrf=cfg.bilibili.csrf,
|
||||
mid=cfg.bilibili.mid,
|
||||
pacer=pacer,
|
||||
retries=retries,
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
|
||||
def cmd_init(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=False)
|
||||
cfg = store.get()
|
||||
@@ -98,12 +120,7 @@ def cmd_sync(args, logger: JsonLogger) -> int:
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
client = BilibiliClient(
|
||||
cookie=cfg.bilibili.cookie,
|
||||
csrf=cfg.bilibili.csrf,
|
||||
mid=cfg.bilibili.mid,
|
||||
logger=logger,
|
||||
)
|
||||
client = _client(cfg, logger)
|
||||
service = FolderSyncService(store, repo, client, logger=logger)
|
||||
summary = service.run(full_refresh=args.full)
|
||||
payload = summary.as_dict()
|
||||
@@ -115,6 +132,46 @@ def cmd_sync(args, logger: JsonLogger) -> int:
|
||||
db.close()
|
||||
|
||||
|
||||
def cmd_snapshot(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=False)
|
||||
cfg = store.get()
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
client = _client(
|
||||
cfg, logger, pacer=Pacer(1.0, 1.6), retries=1
|
||||
)
|
||||
service = SnapshotService(store, repo, client, logger=logger)
|
||||
summary = service.run_if_needed(force=args.force)
|
||||
payload = summary.as_dict()
|
||||
payload["database"] = str(cfg.storage.database_path)
|
||||
payload["snapshot_counts"] = repo.snapshot_counts(summary.stat_date)
|
||||
payload["counts"] = repo.counts()
|
||||
_print(payload)
|
||||
return 0 if summary.completed else 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def cmd_auth_check(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=False)
|
||||
cfg = store.get()
|
||||
if not cfg.bilibili.cookie:
|
||||
_print({"ok": False, "is_login": False, "message": "未配置 cookie"})
|
||||
return 1
|
||||
client = _client(cfg, logger, retries=1)
|
||||
state = client.check_login()
|
||||
_print({
|
||||
"ok": state.is_login,
|
||||
"is_login": state.is_login,
|
||||
"mid": state.mid,
|
||||
"uname": state.uname,
|
||||
"code": state.code,
|
||||
"message": state.message,
|
||||
})
|
||||
return 0 if state.is_login else 1
|
||||
|
||||
|
||||
def cmd_run(args, logger: JsonLogger) -> int:
|
||||
store = ConfigStore(args.config, require_auth=True)
|
||||
cfg = store.get()
|
||||
@@ -122,26 +179,40 @@ def cmd_run(args, logger: JsonLogger) -> int:
|
||||
last: dict[str, object] = {}
|
||||
try:
|
||||
repo = Repository(db)
|
||||
client = BilibiliClient(
|
||||
cookie=cfg.bilibili.cookie,
|
||||
csrf=cfg.bilibili.csrf,
|
||||
mid=cfg.bilibili.mid,
|
||||
logger=logger,
|
||||
)
|
||||
service = FolderSyncService(store, repo, client, logger=logger)
|
||||
folder_client = _client(cfg, logger)
|
||||
folder_service = FolderSyncService(store, repo, folder_client, logger=logger)
|
||||
snapshot_client = _client(cfg, logger, pacer=Pacer(1.0, 1.6), retries=1)
|
||||
snapshot_service = SnapshotService(store, repo, snapshot_client, logger=logger)
|
||||
|
||||
def run_sync():
|
||||
summary = service.run()
|
||||
last["summary"] = summary
|
||||
summary = folder_service.run()
|
||||
last["folder"] = summary
|
||||
logger.info("folder_sync.summary", **summary.as_dict())
|
||||
return summary
|
||||
|
||||
def run_snapshot():
|
||||
summary = snapshot_service.run_if_needed()
|
||||
last["snapshot"] = summary
|
||||
logger.info("snapshot.summary", **summary.as_dict())
|
||||
return summary
|
||||
|
||||
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
|
||||
snapshot_at = f"{hh:02d}:{mm:02d}"
|
||||
scheduler = Scheduler(
|
||||
[ScheduledTask(
|
||||
"folder_sync",
|
||||
float(cfg.schedule.folder_sync_minutes * 60),
|
||||
run_sync,
|
||||
)],
|
||||
[
|
||||
ScheduledTask(
|
||||
"folder_sync",
|
||||
float(cfg.schedule.folder_sync_minutes * 60),
|
||||
run_sync,
|
||||
),
|
||||
ScheduledTask(
|
||||
"snapshot",
|
||||
86400.0,
|
||||
run_snapshot,
|
||||
next_run=0.0,
|
||||
next_run_fn=lambda started: next_daily_run(started, snapshot_at),
|
||||
),
|
||||
],
|
||||
logger=logger,
|
||||
)
|
||||
if args.once:
|
||||
@@ -150,11 +221,16 @@ def cmd_run(args, logger: JsonLogger) -> int:
|
||||
else:
|
||||
logger.info(
|
||||
"scheduler.start",
|
||||
interval_seconds=cfg.schedule.folder_sync_minutes * 60,
|
||||
folder_sync_seconds=cfg.schedule.folder_sync_minutes * 60,
|
||||
snapshot_at=snapshot_at,
|
||||
)
|
||||
scheduler.run_forever()
|
||||
summary = last.get("summary")
|
||||
return 0 if summary is None or getattr(summary, "ok", True) else 1
|
||||
|
||||
folder_summary = last.get("folder")
|
||||
snapshot_summary = last.get("snapshot")
|
||||
folder_ok = folder_summary is None or getattr(folder_summary, "ok", True)
|
||||
snapshot_ok = snapshot_summary is None or getattr(snapshot_summary, "completed", True)
|
||||
return 0 if folder_ok and snapshot_ok else 1
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -165,6 +241,15 @@ def cmd_status(args, logger: JsonLogger) -> int:
|
||||
db = _open_db(cfg, logger)
|
||||
try:
|
||||
repo = Repository(db)
|
||||
today = shanghai_date()
|
||||
snapshot = repo.snapshot_counts(today)
|
||||
snapshot.update({
|
||||
"date": today,
|
||||
"last_attempt": repo.kv_get("snapshot:last_attempt"),
|
||||
"last_completed": repo.kv_get("snapshot:last_completed"),
|
||||
"auth_ok": repo.kv_get("auth:ok"),
|
||||
"auth_message": repo.kv_get("auth:message"),
|
||||
})
|
||||
_print({
|
||||
"ok": True,
|
||||
"schema_version": db.schema_version,
|
||||
@@ -181,6 +266,7 @@ def cmd_status(args, logger: JsonLogger) -> int:
|
||||
}
|
||||
for row in repo.list_folders()
|
||||
],
|
||||
"snapshot": snapshot,
|
||||
**repo.counts(),
|
||||
})
|
||||
finally:
|
||||
@@ -196,6 +282,10 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
return cmd_init(args, logger)
|
||||
if args.command == "sync":
|
||||
return cmd_sync(args, logger)
|
||||
if args.command == "snapshot":
|
||||
return cmd_snapshot(args, logger)
|
||||
if args.command == "auth-check":
|
||||
return cmd_auth_check(args, logger)
|
||||
if args.command == "run":
|
||||
return cmd_run(args, logger)
|
||||
if args.command == "status":
|
||||
|
||||
Reference in new issue
Block a user