kairee
5c2ab7b187
Implement P1: seven-metric view snapshots, relation/stat fan snapshots, failure placeholders, resume by daily completion markers, Cookie login warnings, daily SQLite backups, Shanghai daily scheduling, and CLI snapshot/auth-check commands. Real read-only run captured 795/795 videos successfully over 1124s with a verified daily backup.
308 lines
10 KiB
Python
308 lines
10 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""看板命令行入口。
|
||
|
||
python app.py init # 建库 / 迁移 / 写入黑名单种子（离线）
|
||
python app.py sync # 收藏夹只读同步（首次导入或增量）
|
||
python app.py snapshot # 采集当日七项快照与粉丝数
|
||
python app.py auth-check # 检查 Cookie 登录态
|
||
python app.py run # 按配置周期常驻运行（--once 只跑一次）
|
||
python app.py status # 查看库内状态（离线）
|
||
|
||
P2 起 ``python app.py`` 将在此基础上启动本地看板服务。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
from typing import Sequence
|
||
|
||
from .bilibili.client import BilibiliClient, Pacer
|
||
from .config import ConfigError, ConfigStore
|
||
from .database import Database
|
||
from .logging_json import JsonLogger
|
||
from .repository import Repository
|
||
from .scheduler import ScheduledTask, Scheduler
|
||
from .services.folder_sync import FolderSyncService
|
||
from .services.snapshot import SnapshotService
|
||
from .timeutil import next_daily_run, parse_hhmm, shanghai_date
|
||
|
||
ROOT = Path(__file__).resolve().parent.parent
|
||
DEFAULT_CONFIG = ROOT / "config.json"
|
||
|
||
|
||
def _build_parser() -> argparse.ArgumentParser:
|
||
common = argparse.ArgumentParser(add_help=False)
|
||
common.add_argument(
|
||
"--config",
|
||
default=str(DEFAULT_CONFIG),
|
||
help="配置文件路径（默认项目根目录 config.json）",
|
||
)
|
||
|
||
parser = argparse.ArgumentParser(
|
||
prog="app.py",
|
||
description="咻咻满 Reaction 数据看板 —— 数据底座、同步与每日快照",
|
||
)
|
||
sub = parser.add_subparsers(dest="command", required=True)
|
||
|
||
sub.add_parser("init", parents=[common], help="建库、迁移并写入黑名单种子")
|
||
sync = sub.add_parser("sync", parents=[common], help="同步已配置的年度收藏夹")
|
||
sync.add_argument(
|
||
"--full", action="store_true", help="强制全量刷新元数据（忽略 6 小时周期）"
|
||
)
|
||
snapshot = sub.add_parser(
|
||
"snapshot", parents=[common], help="采集当日七项快照与粉丝数"
|
||
)
|
||
snapshot.add_argument(
|
||
"--force", action="store_true", help="忽略当日完成标记，重新采集"
|
||
)
|
||
sub.add_parser("auth-check", parents=[common], help="检查 Cookie 登录态")
|
||
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行同步与快照")
|
||
run.add_argument(
|
||
"--once", action="store_true", help="只执行一次到期任务后退出"
|
||
)
|
||
sub.add_parser("status", parents=[common], help="查看数据库状态（离线）")
|
||
return parser
|
||
|
||
|
||
def _open_db(cfg, logger: JsonLogger) -> Database:
|
||
db = Database(
|
||
cfg.storage.database_path,
|
||
backup_dir=cfg.storage.backup_dir,
|
||
backup_keep_days=cfg.storage.backup_keep_days,
|
||
logger=logger,
|
||
)
|
||
db.initialize()
|
||
return db
|
||
|
||
|
||
def _print(payload) -> None:
|
||
print(json.dumps(payload, ensure_ascii=False, indent=2), flush=True)
|
||
|
||
|
||
def _client(cfg, logger: JsonLogger, *, pacer=None, retries: int = 3) -> BilibiliClient:
|
||
return BilibiliClient(
|
||
cookie=cfg.bilibili.cookie,
|
||
csrf=cfg.bilibili.csrf,
|
||
mid=cfg.bilibili.mid,
|
||
pacer=pacer,
|
||
retries=retries,
|
||
logger=logger,
|
||
)
|
||
|
||
|
||
def cmd_init(args, logger: JsonLogger) -> int:
|
||
store = ConfigStore(args.config, require_auth=False)
|
||
cfg = store.get()
|
||
db = _open_db(cfg, logger)
|
||
try:
|
||
repo = Repository(db)
|
||
for folder in cfg.folders:
|
||
repo.ensure_folder(folder)
|
||
seeded = repo.seed_blacklist(cfg.up_blacklist_seed)
|
||
_print({
|
||
"ok": True,
|
||
"schema_version": db.schema_version,
|
||
"database": str(cfg.storage.database_path),
|
||
"blacklist_seeded": seeded,
|
||
**repo.counts(),
|
||
})
|
||
finally:
|
||
db.close()
|
||
return 0
|
||
|
||
|
||
def cmd_sync(args, logger: JsonLogger) -> int:
|
||
store = ConfigStore(args.config, require_auth=True)
|
||
cfg = store.get()
|
||
db = _open_db(cfg, logger)
|
||
try:
|
||
repo = Repository(db)
|
||
client = _client(cfg, logger)
|
||
service = FolderSyncService(store, repo, client, logger=logger)
|
||
summary = service.run(full_refresh=args.full)
|
||
payload = summary.as_dict()
|
||
payload["database"] = str(cfg.storage.database_path)
|
||
payload["counts"] = repo.counts()
|
||
_print(payload)
|
||
return 0 if summary.ok else 1
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def cmd_snapshot(args, logger: JsonLogger) -> int:
|
||
store = ConfigStore(args.config, require_auth=False)
|
||
cfg = store.get()
|
||
db = _open_db(cfg, logger)
|
||
try:
|
||
repo = Repository(db)
|
||
client = _client(
|
||
cfg, logger, pacer=Pacer(1.0, 1.6), retries=1
|
||
)
|
||
service = SnapshotService(store, repo, client, logger=logger)
|
||
summary = service.run_if_needed(force=args.force)
|
||
payload = summary.as_dict()
|
||
payload["database"] = str(cfg.storage.database_path)
|
||
payload["snapshot_counts"] = repo.snapshot_counts(summary.stat_date)
|
||
payload["counts"] = repo.counts()
|
||
_print(payload)
|
||
return 0 if summary.completed else 1
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def cmd_auth_check(args, logger: JsonLogger) -> int:
|
||
store = ConfigStore(args.config, require_auth=False)
|
||
cfg = store.get()
|
||
if not cfg.bilibili.cookie:
|
||
_print({"ok": False, "is_login": False, "message": "未配置 cookie"})
|
||
return 1
|
||
client = _client(cfg, logger, retries=1)
|
||
state = client.check_login()
|
||
_print({
|
||
"ok": state.is_login,
|
||
"is_login": state.is_login,
|
||
"mid": state.mid,
|
||
"uname": state.uname,
|
||
"code": state.code,
|
||
"message": state.message,
|
||
})
|
||
return 0 if state.is_login else 1
|
||
|
||
|
||
def cmd_run(args, logger: JsonLogger) -> int:
|
||
store = ConfigStore(args.config, require_auth=True)
|
||
cfg = store.get()
|
||
db = _open_db(cfg, logger)
|
||
last: dict[str, object] = {}
|
||
try:
|
||
repo = Repository(db)
|
||
folder_client = _client(cfg, logger)
|
||
folder_service = FolderSyncService(store, repo, folder_client, logger=logger)
|
||
snapshot_client = _client(cfg, logger, pacer=Pacer(1.0, 1.6), retries=1)
|
||
snapshot_service = SnapshotService(store, repo, snapshot_client, logger=logger)
|
||
|
||
def run_sync():
|
||
summary = folder_service.run()
|
||
last["folder"] = summary
|
||
logger.info("folder_sync.summary", **summary.as_dict())
|
||
return summary
|
||
|
||
def run_snapshot():
|
||
summary = snapshot_service.run_if_needed()
|
||
last["snapshot"] = summary
|
||
logger.info("snapshot.summary", **summary.as_dict())
|
||
return summary
|
||
|
||
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
|
||
snapshot_at = f"{hh:02d}:{mm:02d}"
|
||
scheduler = Scheduler(
|
||
[
|
||
ScheduledTask(
|
||
"folder_sync",
|
||
float(cfg.schedule.folder_sync_minutes * 60),
|
||
run_sync,
|
||
),
|
||
ScheduledTask(
|
||
"snapshot",
|
||
86400.0,
|
||
run_snapshot,
|
||
next_run=0.0,
|
||
next_run_fn=lambda started: next_daily_run(started, snapshot_at),
|
||
),
|
||
],
|
||
logger=logger,
|
||
)
|
||
if args.once:
|
||
results = scheduler.run_due()
|
||
_print([result.as_dict() for result in results])
|
||
else:
|
||
logger.info(
|
||
"scheduler.start",
|
||
folder_sync_seconds=cfg.schedule.folder_sync_minutes * 60,
|
||
snapshot_at=snapshot_at,
|
||
)
|
||
scheduler.run_forever()
|
||
|
||
folder_summary = last.get("folder")
|
||
snapshot_summary = last.get("snapshot")
|
||
folder_ok = folder_summary is None or getattr(folder_summary, "ok", True)
|
||
snapshot_ok = snapshot_summary is None or getattr(snapshot_summary, "completed", True)
|
||
return 0 if folder_ok and snapshot_ok else 1
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def cmd_status(args, logger: JsonLogger) -> int:
|
||
store = ConfigStore(args.config, require_auth=False)
|
||
cfg = store.get()
|
||
db = _open_db(cfg, logger)
|
||
try:
|
||
repo = Repository(db)
|
||
today = shanghai_date()
|
||
snapshot = repo.snapshot_counts(today)
|
||
snapshot.update({
|
||
"date": today,
|
||
"last_attempt": repo.kv_get("snapshot:last_attempt"),
|
||
"last_completed": repo.kv_get("snapshot:last_completed"),
|
||
"auth_ok": repo.kv_get("auth:ok"),
|
||
"auth_message": repo.kv_get("auth:message"),
|
||
})
|
||
_print({
|
||
"ok": True,
|
||
"schema_version": db.schema_version,
|
||
"database": str(cfg.storage.database_path),
|
||
"configured_folders": [
|
||
{
|
||
"media_id": row.media_id,
|
||
"year": row.year,
|
||
"title": row.title,
|
||
"media_count": row.media_count,
|
||
"last_ids_at": row.last_ids_at,
|
||
"last_full_at": row.last_full_at,
|
||
"enabled": row.enabled,
|
||
}
|
||
for row in repo.list_folders()
|
||
],
|
||
"snapshot": snapshot,
|
||
**repo.counts(),
|
||
})
|
||
finally:
|
||
db.close()
|
||
return 0
|
||
|
||
|
||
def main(argv: Sequence[str] | None = None) -> int:
|
||
args = _build_parser().parse_args(list(argv) if argv is not None else None)
|
||
logger = JsonLogger()
|
||
try:
|
||
if args.command == "init":
|
||
return cmd_init(args, logger)
|
||
if args.command == "sync":
|
||
return cmd_sync(args, logger)
|
||
if args.command == "snapshot":
|
||
return cmd_snapshot(args, logger)
|
||
if args.command == "auth-check":
|
||
return cmd_auth_check(args, logger)
|
||
if args.command == "run":
|
||
return cmd_run(args, logger)
|
||
if args.command == "status":
|
||
return cmd_status(args, logger)
|
||
except ConfigError as exc:
|
||
logger.error("config.error", error=str(exc))
|
||
return 2
|
||
except KeyboardInterrupt:
|
||
logger.warning("app.interrupted")
|
||
return 130
|
||
except Exception as exc: # noqa: BLE001 CLI 边界统一记录
|
||
logger.error("app.error", error=str(exc))
|
||
return 2
|
||
logger.error("app.unknown_command", command=getattr(args, "command", None))
|
||
return 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|