kairee
6a21b5d905
Backfill same-day snapshots for newly added or restored videos, preserve successful fan snapshots on resume, persist incomplete state before forced re-runs, and split collection completion from backup completion so failed backups can retry without API calls. Reject missing data payloads instead of treating them as empty folders, guard against mass-unfav on suspicious empty remote folders, rebuild Bilibili clients when credentials change, recompute primary folder ownership from active relations with per-folder fav_time, preserve original titles for dropped invalid videos, poll config every 60 seconds, and fix favsort verify interval and mixed resource types.
382 lines
13 KiB
Python
382 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""看板命令行入口。
|
||
|
||
python app.py init # 建库 / 迁移 / 写入黑名单种子（离线）
|
||
python app.py sync # 收藏夹只读同步（首次导入或增量）
|
||
python app.py snapshot # 采集当日七项快照与粉丝数
|
||
python app.py auth-check # 检查 Cookie 登录态
|
||
python app.py run # 按配置周期常驻运行（--once 只跑一次）
|
||
python app.py status # 查看库内状态（离线）
|
||
|
||
P2 起 ``python app.py`` 将在此基础上启动本地看板服务。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
from typing import Sequence
|
||
|
||
from .bilibili.auth_flow import AuthService
|
||
from .bilibili.client import BilibiliClient, Pacer
|
||
from .bilibili.provider import ClientProvider
|
||
from .config import ConfigError, ConfigStore
|
||
from .database import Database
|
||
from .logging_json import JsonLogger
|
||
from .repository import Repository
|
||
from .scheduler import ScheduledTask, Scheduler
|
||
from .services.folder_sync import FolderSyncService
|
||
from .services.snapshot import SnapshotService
|
||
from .timeutil import next_daily_run, parse_hhmm, shanghai_date
|
||
|
||
ROOT = Path(__file__).resolve().parent.parent
|
||
DEFAULT_CONFIG = ROOT / "config.json"
|
||
|
||
|
||
def _build_parser() -> argparse.ArgumentParser:
|
||
common = argparse.ArgumentParser(add_help=False)
|
||
common.add_argument(
|
||
"--config",
|
||
default=str(DEFAULT_CONFIG),
|
||
help="配置文件路径（默认项目根目录 config.json）",
|
||
)
|
||
|
||
parser = argparse.ArgumentParser(
|
||
prog="app.py",
|
||
description="咻咻满 Reaction 数据看板 —— 数据底座、同步与每日快照",
|
||
)
|
||
sub = parser.add_subparsers(dest="command", required=True)
|
||
|
||
sub.add_parser("init", parents=[common], help="建库、迁移并写入黑名单种子")
|
||
sync = sub.add_parser("sync", parents=[common], help="同步已配置的年度收藏夹")
|
||
sync.add_argument(
|
||
"--full", action="store_true", help="强制全量刷新元数据（忽略 6 小时周期）"
|
||
)
|
||
sync.add_argument(
|
||
"--allow-empty-folder",
|
||
action="store_true",
|
||
help="允许把远端 0 条视为真实清空（默认按异常跳过差集）",
|
||
)
|
||
snapshot = sub.add_parser(
|
||
"snapshot", parents=[common], help="采集当日七项快照与粉丝数"
|
||
)
|
||
snapshot.add_argument(
|
||
"--force", action="store_true", help="忽略当日完成标记，重新采集"
|
||
)
|
||
auth = sub.add_parser("auth", parents=[common], help="扫码授权并更新指定账号 Cookie")
|
||
auth.add_argument("--account", choices=("read", "write"), default="read",
|
||
help="要授权的账号角色（默认 read）")
|
||
auth.add_argument("--no-browser", action="store_true", help="不自动打开二维码图片")
|
||
auth.add_argument("--timeout", type=int, default=180, metavar="SEC",
|
||
help="等待扫码的超时时间（默认 180 秒）")
|
||
auth_check = sub.add_parser("auth-check", parents=[common], help="检查账号登录态")
|
||
auth_check.add_argument("--account", choices=("read", "write", "all"), default="read",
|
||
help="要检查的账号角色（默认 read）")
|
||
run = sub.add_parser("run", parents=[common], help="按配置周期常驻运行同步与快照")
|
||
run.add_argument(
|
||
"--once", action="store_true", help="只执行一次到期任务后退出"
|
||
)
|
||
run.add_argument(
|
||
"--allow-empty-folder",
|
||
action="store_true",
|
||
help="允许把远端 0 条视为真实清空（默认按异常跳过差集）",
|
||
)
|
||
sub.add_parser("status", parents=[common], help="查看数据库状态（离线）")
|
||
return parser
|
||
|
||
|
||
def _open_db(cfg, logger: JsonLogger) -> Database:
|
||
db = Database(
|
||
cfg.storage.database_path,
|
||
backup_dir=cfg.storage.backup_dir,
|
||
backup_keep_days=cfg.storage.backup_keep_days,
|
||
logger=logger,
|
||
)
|
||
db.initialize()
|
||
return db
|
||
|
||
|
||
def _print(payload) -> None:
|
||
print(json.dumps(payload, ensure_ascii=False, indent=2), flush=True)
|
||
|
||
|
||
def _client(credentials, logger: JsonLogger, *, pacer=None, retries: int = 3) -> BilibiliClient:
|
||
return BilibiliClient(
|
||
cookie=credentials.cookie,
|
||
csrf=credentials.csrf,
|
||
mid=credentials.mid,
|
||
pacer=pacer,
|
||
retries=retries,
|
||
logger=logger,
|
||
)
|
||
|
||
|
||
def cmd_init(args, logger: JsonLogger) -> int:
|
||
store = ConfigStore(args.config, require_auth=False)
|
||
cfg = store.get()
|
||
db = _open_db(cfg, logger)
|
||
try:
|
||
repo = Repository(db)
|
||
for folder in cfg.folders:
|
||
repo.ensure_folder(folder)
|
||
seeded = repo.seed_blacklist(cfg.up_blacklist_seed)
|
||
_print({
|
||
"ok": True,
|
||
"schema_version": db.schema_version,
|
||
"database": str(cfg.storage.database_path),
|
||
"blacklist_seeded": seeded,
|
||
**repo.counts(),
|
||
})
|
||
finally:
|
||
db.close()
|
||
return 0
|
||
|
||
|
||
def cmd_sync(args, logger: JsonLogger) -> int:
|
||
store = ConfigStore(args.config, require_auth=True)
|
||
cfg = store.get()
|
||
db = _open_db(cfg, logger)
|
||
try:
|
||
repo = Repository(db)
|
||
client = _client(cfg.bilibili.read, logger)
|
||
service = FolderSyncService(
|
||
store,
|
||
repo,
|
||
client,
|
||
logger=logger,
|
||
allow_empty_folders=args.allow_empty_folder,
|
||
)
|
||
summary = service.run(full_refresh=args.full)
|
||
payload = summary.as_dict()
|
||
payload["database"] = str(cfg.storage.database_path)
|
||
payload["counts"] = repo.counts()
|
||
_print(payload)
|
||
return 0 if summary.ok else 1
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def cmd_snapshot(args, logger: JsonLogger) -> int:
|
||
store = ConfigStore(args.config, require_auth=False)
|
||
cfg = store.get()
|
||
db = _open_db(cfg, logger)
|
||
try:
|
||
repo = Repository(db)
|
||
client = _client(
|
||
cfg.bilibili.read, logger, pacer=Pacer(1.0, 1.6), retries=1
|
||
)
|
||
service = SnapshotService(store, repo, client, logger=logger)
|
||
summary = service.run_if_needed(force=args.force)
|
||
payload = summary.as_dict()
|
||
payload["database"] = str(cfg.storage.database_path)
|
||
payload["snapshot_counts"] = repo.snapshot_counts(summary.stat_date)
|
||
payload["counts"] = repo.counts()
|
||
_print(payload)
|
||
return 0 if summary.completed and not summary.backup_error else 1
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def cmd_auth(args, logger: JsonLogger) -> int:
|
||
if args.timeout < 1:
|
||
raise ConfigError("auth --timeout 必须至少为 1 秒")
|
||
return AuthService(Path(args.config), account=args.account, logger=logger).run(
|
||
timeout=args.timeout, no_browser=args.no_browser
|
||
)
|
||
|
||
|
||
def _auth_check(cfg, account: str, logger: JsonLogger) -> dict[str, object]:
|
||
credentials = getattr(cfg.bilibili, account)
|
||
if not credentials.cookie:
|
||
return {"account": account, "ok": False, "is_login": False, "message": "未配置 cookie"}
|
||
state = _client(credentials, logger, retries=1).check_login()
|
||
return {
|
||
"account": account,
|
||
"ok": state.is_login,
|
||
"is_login": state.is_login,
|
||
"mid": state.mid,
|
||
"uname": state.uname,
|
||
"code": state.code,
|
||
"message": state.message,
|
||
}
|
||
|
||
|
||
def cmd_auth_check(args, logger: JsonLogger) -> int:
|
||
store = ConfigStore(args.config, require_auth=False)
|
||
cfg = store.get()
|
||
if args.account != "all":
|
||
result = _auth_check(cfg, args.account, logger)
|
||
_print(result)
|
||
return 0 if result["ok"] else 1
|
||
results = [_auth_check(cfg, account, logger) for account in ("read", "write")]
|
||
_print({"ok": all(result["ok"] for result in results), "accounts": results})
|
||
return 0 if all(result["ok"] for result in results) else 1
|
||
|
||
|
||
def cmd_run(args, logger: JsonLogger) -> int:
|
||
store = ConfigStore(args.config, require_auth=True)
|
||
cfg = store.get()
|
||
db = _open_db(cfg, logger)
|
||
last: dict[str, object] = {}
|
||
try:
|
||
repo = Repository(db)
|
||
folder_provider = ClientProvider(
|
||
store, "read", lambda credentials: _client(credentials, logger)
|
||
)
|
||
snapshot_provider = ClientProvider(
|
||
store,
|
||
"read",
|
||
lambda credentials: _client(
|
||
credentials, logger, pacer=Pacer(1.0, 1.6), retries=1
|
||
),
|
||
)
|
||
folder_service = FolderSyncService(
|
||
store,
|
||
repo,
|
||
client_provider=folder_provider,
|
||
logger=logger,
|
||
allow_empty_folders=args.allow_empty_folder,
|
||
)
|
||
snapshot_service = SnapshotService(
|
||
store, repo, client_provider=snapshot_provider, logger=logger
|
||
)
|
||
|
||
def run_snapshot():
|
||
summary = snapshot_service.run_if_needed()
|
||
last["snapshot"] = summary
|
||
logger.info("snapshot.summary", **summary.as_dict())
|
||
return summary
|
||
|
||
def run_sync():
|
||
summary = folder_service.run()
|
||
last["folder"] = summary
|
||
logger.info("folder_sync.summary", **summary.as_dict())
|
||
# 新入库/恢复的视频立即补当天快照，而不是等到次日。
|
||
if summary.added > 0:
|
||
run_snapshot()
|
||
return summary
|
||
|
||
def run_config_watch():
|
||
before = store.revision
|
||
store.get()
|
||
if store.revision != before:
|
||
logger.info("config.reloaded", revision=store.revision)
|
||
return {"revision": store.revision}
|
||
|
||
hh, mm = parse_hhmm(cfg.schedule.snapshot_at)
|
||
snapshot_at = f"{hh:02d}:{mm:02d}"
|
||
scheduler = Scheduler(
|
||
[
|
||
ScheduledTask(
|
||
"folder_sync",
|
||
float(cfg.schedule.folder_sync_minutes * 60),
|
||
run_sync,
|
||
),
|
||
ScheduledTask(
|
||
"snapshot",
|
||
86400.0,
|
||
run_snapshot,
|
||
next_run=0.0,
|
||
next_run_fn=lambda started: next_daily_run(started, snapshot_at),
|
||
),
|
||
ScheduledTask("config_watch", 60.0, run_config_watch),
|
||
],
|
||
logger=logger,
|
||
)
|
||
if args.once:
|
||
results = scheduler.run_due()
|
||
_print([result.as_dict() for result in results])
|
||
else:
|
||
logger.info(
|
||
"scheduler.start",
|
||
folder_sync_seconds=cfg.schedule.folder_sync_minutes * 60,
|
||
snapshot_at=snapshot_at,
|
||
)
|
||
scheduler.run_forever()
|
||
|
||
folder_summary = last.get("folder")
|
||
snapshot_summary = last.get("snapshot")
|
||
folder_ok = folder_summary is None or getattr(folder_summary, "ok", True)
|
||
snapshot_ok = snapshot_summary is None or getattr(snapshot_summary, "completed", True)
|
||
return 0 if folder_ok and snapshot_ok else 1
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def cmd_status(args, logger: JsonLogger) -> int:
|
||
store = ConfigStore(args.config, require_auth=False)
|
||
cfg = store.get()
|
||
db = _open_db(cfg, logger)
|
||
try:
|
||
repo = Repository(db)
|
||
today = shanghai_date()
|
||
snapshot = repo.snapshot_counts(today)
|
||
snapshot.update({
|
||
"date": today,
|
||
"last_attempt": repo.kv_get("snapshot:last_attempt"),
|
||
"last_completed": repo.kv_get("snapshot:last_completed"),
|
||
"auth_ok": repo.kv_get("auth:ok"),
|
||
"auth_message": repo.kv_get("auth:message"),
|
||
"backup_ok": repo.kv_get(f"snapshot:backup:{today}"),
|
||
"backup_error": repo.kv_get(f"snapshot:backup_error:{today}"),
|
||
})
|
||
_print({
|
||
"ok": True,
|
||
"schema_version": db.schema_version,
|
||
"database": str(cfg.storage.database_path),
|
||
"configured_folders": [
|
||
{
|
||
"media_id": row.media_id,
|
||
"year": row.year,
|
||
"title": row.title,
|
||
"media_count": row.media_count,
|
||
"last_ids_at": row.last_ids_at,
|
||
"last_full_at": row.last_full_at,
|
||
"enabled": row.enabled,
|
||
}
|
||
for row in repo.list_folders()
|
||
],
|
||
"snapshot": snapshot,
|
||
**repo.counts(),
|
||
})
|
||
finally:
|
||
db.close()
|
||
return 0
|
||
|
||
|
||
def main(argv: Sequence[str] | None = None) -> int:
|
||
args = _build_parser().parse_args(list(argv) if argv is not None else None)
|
||
logger = JsonLogger()
|
||
try:
|
||
if args.command == "init":
|
||
return cmd_init(args, logger)
|
||
if args.command == "sync":
|
||
return cmd_sync(args, logger)
|
||
if args.command == "snapshot":
|
||
return cmd_snapshot(args, logger)
|
||
if args.command == "auth":
|
||
return cmd_auth(args, logger)
|
||
if args.command == "auth-check":
|
||
return cmd_auth_check(args, logger)
|
||
if args.command == "run":
|
||
return cmd_run(args, logger)
|
||
if args.command == "status":
|
||
return cmd_status(args, logger)
|
||
except ConfigError as exc:
|
||
logger.error("config.error", error=str(exc))
|
||
return 2
|
||
except KeyboardInterrupt:
|
||
logger.warning("app.interrupted")
|
||
return 130
|
||
except Exception as exc: # noqa: BLE001 CLI 边界统一记录
|
||
logger.error("app.error", error=str(exc))
|
||
return 2
|
||
logger.error("app.unknown_command", command=getattr(args, "command", None))
|
||
return 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|